diff --git a/backend/Dockerfile b/backend/Dockerfile index a9be671e6..f1e150ec6 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,8 +1,7 @@ -FROM --platform=linux/amd64 python:3.9 +FROM python:3.12.0-slim-bookworm ENV PYTHONUNBUFFERED=1 WORKDIR /api COPY requirements.txt /api/ -RUN pip install --upgrade pip==24.0 RUN pip install -r requirements.txt diff --git a/backend/api/apps.py b/backend/api/apps.py index f051eb5e7..2bccbd09b 100644 --- a/backend/api/apps.py +++ b/backend/api/apps.py @@ -1,19 +1,14 @@ import inspect import logging -import pika import pkgutil -import smtplib import sys from collections import namedtuple -from amqp import AMQPError from django.apps import AppConfig from django.db.models.signals import post_migrate from db_comments.db_actions import create_db_comments, \ create_db_comments_from_models -from zeva.settings import RUNSERVER, AMQP_CONNECTION_PARAMETERS, KEYCLOAK, \ - EMAIL, DEBUG logger = logging.getLogger('zeva.apps') @@ -30,32 +25,6 @@ def ready(self): post_migrate.connect(post_migration_callback, sender=self) sys.modules['api.models.account_balance'] = None - if RUNSERVER: - try: - check_external_services() - except RuntimeError as error: - logger.critical('Startup checks failed.', error) - if not DEBUG: - logger.critical( - 'Aborting startup due to failed startup check.', error - ) - exit(-1) - - -def check_external_services(): - """Called after initialization. Use it to validate settings""" - - logger.info('Checking AMQP connection') - - try: - parameters = AMQP_CONNECTION_PARAMETERS - connection = pika.BlockingConnection(parameters) - connection.channel() - connection.close() - except AMQPError as _error: - logger.error(_error) - raise RuntimeError('AMQP connection failed') - def post_migration_callback(sender, **kwargs): """ @@ -114,12 +83,13 @@ def get_all_model_classes(): # Has to be a local import. Must be loaded late. import api.models + models_path = api.models.__path__ classes = set() ModuleInfo = namedtuple('ModuleInfo', ['module_finder', 'name', 'ispkg']) for (module_finder, name, ispkg) in pkgutil.walk_packages( - api.models.__path__, + models_path, prefix='api.models.' ): @@ -130,9 +100,8 @@ def get_all_model_classes(): mod = sys.modules[sub_module.name] else: # load the module - mod = sub_module.module_finder.find_module( - sub_module.name - ).load_module() + spec = sub_module.module_finder.find_spec(sub_module.name, models_path) + mod = spec.loader.load_module(spec.name) for name, obj in inspect.getmembers(mod): if inspect.getmodule(obj) is not None and \ diff --git a/backend/requirements.txt b/backend/requirements.txt index 32f52cf96..7f2f1bb40 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,8 +1,6 @@ -amqp==2.5.2 asgiref==3.3.2 astroid==2.3.3 billiard==3.6.1.0 -celery==4.4.0 certifi==2022.12.7 cffi chardet==3.0.4 @@ -10,7 +8,6 @@ configparser==5.0.0 coverage==5.0.3 cryptography==42.0.4 Django==3.2.25 -django-celery-beat==1.5.0 django-cors-headers==3.2.1 django-enumfields==2.1.1 django-filter==2.4.0 @@ -26,10 +23,10 @@ lazy-object-proxy==1.4.3 Markdown==3.1.1 mccabe==0.6.1 minio==5.0.10 -numpy==1.21.6 -pandas==1.1.5 +numpy==1.26.0 +pandas==2.2.3 pika==1.1.0 -psycopg2-binary==2.9.3 +psycopg2-binary==2.9.9 pycodestyle==2.5.0 pycparser==2.19 pyjwt==2.1.0 @@ -37,15 +34,14 @@ pylint==2.4.4 pylint-django==2.0.13 pylint-plugin-utils==0.6 python-crontab==2.4.0 -python-dateutil==2.8.1 +python-dateutil==2.9.0 python-dotenv==0.10.5 python-magic==0.4.18 pytz==2022.2.1 requests==2.32.0 -six==1.13.0 +six==1.16.0 sqlparse==0.5.0 -typed-ast -urllib3==1.25.11 +urllib3==1.26.20 vine==1.3.0 virtualenv==16.0.0 wrapt==1.11.2 diff --git a/backend/zeva/amqp.py b/backend/zeva/amqp.py deleted file mode 100644 index 37f8f6add..000000000 --- a/backend/zeva/amqp.py +++ /dev/null @@ -1,12 +0,0 @@ -import os - - -def config(): - return { - 'ENGINE': 'rabbitmq', - 'VHOST': os.getenv('RABBITMQ_VHOST', '/'), - 'USER': os.getenv('RABBITMQ_USER', 'guest'), - 'PASSWORD': os.getenv('RABBITMQ_PASSWORD', 'guest'), - 'HOST': os.getenv('RABBITMQ_HOST', 'localhost'), - 'PORT': os.getenv('RABBITMQ_PORT', '5672'), - } diff --git a/backend/zeva/celery.py b/backend/zeva/celery.py deleted file mode 100644 index 75d6a1557..000000000 --- a/backend/zeva/celery.py +++ /dev/null @@ -1,25 +0,0 @@ -from __future__ import absolute_import, unicode_literals -import os -from celery import Celery - -from zeva.settings import AMQP - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'zeva.settings') - -app = Celery('zeva', broker='amqp://{}:{}@{}:{}/{}'.format( - AMQP['USER'], - AMQP['PASSWORD'], - AMQP['HOST'], - AMQP['PORT'], - AMQP['VHOST'])) - -app.conf.update({ - 'beat_scheduler': 'django_celery_beat.schedulers:DatabaseScheduler' -}) - -app.autodiscover_tasks() - -app.conf.update({ - 'beat_schedule': { - } -}) diff --git a/backend/zeva/settings.py b/backend/zeva/settings.py index 4282f1e73..693ab7b3d 100644 --- a/backend/zeva/settings.py +++ b/backend/zeva/settings.py @@ -13,10 +13,7 @@ import os import sys -from pika import ConnectionParameters, PlainCredentials -from django.db.models import BigAutoField - -from . import database, amqp, email, keycloak, minio +from . import database, email, keycloak, minio # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -50,7 +47,6 @@ INSTALLED_APPS = [ 'django.contrib.auth', 'django.contrib.contenttypes', - 'django_celery_beat', 'rest_framework', 'zeva', 'corsheaders', @@ -110,15 +106,6 @@ KEYCLOAK = keycloak.config() -AMQP = amqp.config() - -AMQP_CONNECTION_PARAMETERS = ConnectionParameters( - host=AMQP['HOST'], - port=AMQP['PORT'], - virtual_host=AMQP['VHOST'], - credentials=PlainCredentials(AMQP['USER'], AMQP['PASSWORD']) -) - FILE_UPLOAD_HANDLERS = [ 'django.core.files.uploadhandler.TemporaryFileUploadHandler' ] @@ -195,10 +182,6 @@ 'level': 'INFO', 'handlers': ['console'], }, - 'celery': { - 'level': 'WARNING', - 'handlers': ['console'], - }, 'zeva': { 'level': 'INFO', 'handlers': ['console'], diff --git a/docker-compose.yml b/docker-compose.yml index 03f136417..9c15cc992 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,19 +14,14 @@ services: interval: 5s timeout: 5s retries: 5 - volumes: - - postgres-data:/var/lib/postgresql/data api: build: ./backend command: > sh -c "python manage.py migrate && - python manage.py load_ops_data --directory api/fixtures/operational/ - python manage.py load_ops_data --directory api/fixtures/test/ python manage.py runserver 0.0.0.0:8000" env_file: - backend.env - minio.env - - rabbitmq.env volumes: - ./backend:/api ports: @@ -34,14 +29,8 @@ services: depends_on: db: condition: service_healthy - mailslurper: - build: ./mailslurper - ports: - - 2500:2500 - - 8081:8081 - - 8085:8085 minio: - image: minio/minio + image: minio/minio:RELEASE.2024-10-13T13-34-11Z-cpuv1 hostname: "minio" volumes: - ./minio/minio_files:/minio_files @@ -51,30 +40,13 @@ services: ports: - 9000:9000 - 9001:9001 - rabbitmq: - image: rabbitmq:3.7-management - hostname: "rabbitmq" - environment: - - RABBITMQ_DEFAULT_USER=rabbitmq - - RABBITMQ_DEFAULT_PASS=rabbitmq - - RABBITMQ_DEFAULT_VHOST=/zeva - - RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=-rabbit log_levels [{connection,error}] - ports: - - 15672:15672 - - 5672:5672 web: build: ./frontend command: npm start env_file: - frontend.env - - rabbitmq.env volumes: - ./frontend:/web - /web/node_modules - depends_on: - - rabbitmq ports: - 3000:3000 - - 8080:8080 -volumes: - postgres-data: diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 4ba8abc9d..ebe1c704f 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16.20.0 +FROM node:20.18.0-bookworm-slim WORKDIR /web diff --git a/frontend/notifications.js b/frontend/notifications.js deleted file mode 100644 index 63e3a22a2..000000000 --- a/frontend/notifications.js +++ /dev/null @@ -1,156 +0,0 @@ -const amqp = require('amqp') -const jwt = require('jsonwebtoken') -const jwksClient = require('jwks-rsa') - -const vhost = process.env.RABBITMQ_VHOST || '/' - -const user = process.env.RABBITMQ_USER || 'guest' -const password = process.env.RABBITMQ_PASSWORD || 'guest' -const amqpHost = process.env.RABBITMQ_HOST || 'localhost' -const amqpPort = process.env.RABBITMQ_PORT || 5672 -const jwksURI = process.env.KEYCLOAK_CERTS_URL || null -const amqpEnabled = process.env.RABBITMQ_ENABLED === 'True' || false - -const winston = require('winston') - -const consoleTransport = new winston.transports.Console() -const log = winston.createLogger({ - format: winston.format.combine( - winston.format.timestamp(), - winston.format.prettyPrint() - ), - transports: [consoleTransport] -}) - -const connect = (io) => { - const connection = amqp.createConnection({ - host: amqpHost, - port: amqpPort, - vhost, - login: user, - password - }) - - connection.on('error', (e) => { - log.error('AMQP error: ', e) - }) - - connection.on('ready', () => { - log.info('AMQP connection ready') - - connection.queue('', { exclusive: false, autoDelete: true }, (q) => { - connection.exchange( - 'notifications', - { - type: 'fanout', - autoDelete: false, - durable: true, - confirm: false - }, - (exchange) => { - q.bind(exchange, '#') - } - ) - - /* we got a notification -- tell the connected clients */ - q.subscribe((message) => { - // by default our audience is everyone - let room = 'global' - - if (message.audience && message.audience !== 'global') { - // this message is for someone in particular - room = `user_${message.audience}` - } - - if (!message.type) { - log.error('this message does not contain a type!') - return - } - - switch (message.type) { - case 'notification': - io.in(room).emit('action', { - type: 'SERVER_INITIATED_NOTIFICATION_RELOAD', - message: 'notification' - }) - break - default: - log.error(`unknown message type ${message.type}`) - } - }) - }) - }) -} - -const client = jwksClient({ jwksUri: jwksURI }) - -function getSigningKey (header, callback) { - client.getSigningKey(header.kid, (err, key) => { - if (err) { - callback({ error: 'certificate download failed' }, null) - return - } - const signingKey = key.publicKey || key.rsaPublicKey - callback(null, signingKey) - }) -} - -const setup = (io) => { - if (!jwksURI) { - log.error( - 'No KEYCLOAK_CERTS_URL in environment,' + - ' cannot validate tokens and will not serve socket.io clients' - ) - return - } - - io.on('connect', (socket) => { - socket.join('global') - let authenticated = false - let roomName - - socket.on('action', (action) => { - switch (action.type) { - case 'socketio/AUTHENTICATE': - jwt.verify(action.token, getSigningKey, {}, (err, decoded) => { - if (err) { - log.error(`error verifying token ${err}`) - authenticated = false - } else { - roomName = `user_${decoded.user_id}` - socket.join(roomName) - authenticated = true - io.in(roomName).emit('action', { - type: 'socketio/AUTHENTICATE_SUCCESS' - }) - io.in(roomName).emit('action', { - type: 'message', - data: `Hello from nodejs@${process.env.HOSTNAME}, ${decoded.user_id}` - }) - } - }) - break - case 'socketio/DEAUTHENTICATE': - authenticated = false - if (authenticated) { - socket.leave(roomName) - } - break - default: - log.error(`unknown action received ${action.type}`) - } - }) - }) - - if (amqpEnabled) { - connect(io) - } else { - log.info( - 'AMQP is disabled. Clients can connect and be authenticated but no messages will be relayed to them.' - ) - } -} - -module.exports = { - setup -} diff --git a/frontend/package.json b/frontend/package.json index c148eba2a..bdd0f7171 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,7 +13,6 @@ "@hot-loader/react-dom": "^16.13.0", "@storybook/preset-scss": "^1.0.2", "@types/react": "^16.9.36", - "amqp": "^0.2.7", "axios": "^0.21.1", "big.js": "6.2.1", "bootstrap": "^4.4.1", diff --git a/frontend/serve.js b/frontend/serve.js index e3b0e03cd..bc4365934 100644 --- a/frontend/serve.js +++ b/frontend/serve.js @@ -1,18 +1,7 @@ -const http = require('http') const express = require('express') const fallback = require('express-history-api-fallback') -const notifications = require('./notifications') const morgan = require('morgan') -const websocketServer = http.createServer((req, res) => { - res.end() -}) - -const io = require('socket.io')(websocketServer) - -notifications.setup(io) -websocketServer.listen(5002, '0.0.0.0') - const app = express() app.use(morgan('combined')) diff --git a/frontend/start.js b/frontend/start.js index 1c1a80446..fc8e48360 100644 --- a/frontend/start.js +++ b/frontend/start.js @@ -1,10 +1,8 @@ const Webpack = require('webpack') const DevServer = require('webpack-dev-server') const path = require('path') -const http = require('http') const webpackConfig = require('./webpack.config') -const notifications = require('./notifications') const devServerOptions = { static: { @@ -37,17 +35,7 @@ const devServerOptions = { } const compiler = Webpack(webpackConfig) -const devServer = new DevServer(devServerOptions, compiler) - -const websocketServer = http.createServer((req, res) => { - res.end() -}) - -const io = require('socket.io')(websocketServer) - -notifications.setup(io) - -websocketServer.listen(5002, '0.0.0.0'); +const devServer = new DevServer(devServerOptions, compiler); (async () => { await devServer.start() diff --git a/rabbitmq.env b/rabbitmq.env deleted file mode 100644 index cb87aa48c..000000000 --- a/rabbitmq.env +++ /dev/null @@ -1,5 +0,0 @@ -RABBITMQ_VHOST=/zeva -RABBITMQ_USER=rabbitmq -RABBITMQ_PASSWORD=rabbitmq -RABBITMQ_HOST=rabbitmq -RABBITMQ_PORT=5672