Emailer queue upgrade (#1911) (#1912)

*  queue replacement

* unit tests

* dockerfile

* fix logger

Author: bolyachevets

services/emailer/Dockerfile

@@ -1,6 +1,6 @@
-FROM python:3.12.2
 
-USER root
+# Stage 1: Build environment (with Poetry + dependencies installed)
+FROM python:3.12-slim AS development_build
 
 ARG VCS_REF="missing"
 ARG BUILD_DATE="missing"
@@ -10,32 +10,32 @@ ENV BUILD_DATE=${BUILD_DATE}
 ENV PORT=8080
 
 LABEL org.label-schema.vcs-ref=${VCS_REF} \
-    org.label-schema.build-date=${BUILD_DATE}
+      org.label-schema.build-date=${BUILD_DATE} \
+      vendor="BCROS"
 
-LABEL vendor="BCROS"
+USER root
 
 ARG APP_ENV \
-  # Needed for fixing permissions of files created by Docker:
-  UID=1000 \
-  GID=1000
+    UID=1000 \
+    GID=1000
 
 ENV APP_ENV=${APP_ENV} \
-  # python:
-  PYTHONFAULTHANDLER=1 \
-  PYTHONUNBUFFERED=1 \
-  PYTHONHASHSEED=random \
-  PYTHONDONTWRITEBYTECODE=1 \
-  # pip:
-  PIP_NO_CACHE_DIR=1 \
-  PIP_DISABLE_PIP_VERSION_CHECK=1 \
-  PIP_DEFAULT_TIMEOUT=100 \
-  PIP_ROOT_USER_ACTION=ignore \
-  # poetry:
-  POETRY_VERSION=1.3.2 \
-  POETRY_NO_INTERACTION=1 \
-  POETRY_VIRTUALENVS_CREATE=false \
-  POETRY_CACHE_DIR='/var/cache/pypoetry' \
-  POETRY_HOME='/usr/local'
+    # python:
+    PYTHONFAULTHANDLER=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONHASHSEED=random \
+    PYTHONDONTWRITEBYTECODE=1 \
+    # pip:
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_DEFAULT_TIMEOUT=100 \
+    PIP_ROOT_USER_ACTION=ignore \
+    # poetry:
+    POETRY_VERSION=2.1.3 \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_CACHE_DIR='/var/cache/pypoetry' \
+    POETRY_HOME='/usr/local'
 
 SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 
@@ -51,7 +51,7 @@ RUN apt-get update && apt-get upgrade -y \
     wait-for-it \
   && curl -sSL 'https://install.python-poetry.org' | python - \
   && poetry --version \
-  # Cleaning cache:
+  # Clean up apt cache
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
@@ -61,29 +61,30 @@ RUN groupadd -g "${GID}" -r web \
   && useradd -d '/code' -g web -l -r -u "${UID}" web \
   && chown web:web -R '/code'
 
-# Copy only requirements, to cache them in docker layer
+# Copy dependency files first (better caching)
 COPY --chown=web:web ./poetry.lock ./pyproject.toml /code/
 
+# Copy app source
 COPY --chown=web:web ./src /code/src
 COPY --chown=web:web ./README.md /code
 
-# Project initialization:
+# Install dependencies
 RUN --mount=type=cache,target="$POETRY_CACHE_DIR" \
   echo "$APP_ENV" \
+  && poetry config installer.max-workers 1 \
   && poetry version \
-  # Install deps:
   && poetry run pip install -U pip \
   && poetry install \
-    $(if [ -z ${APP_ENV+x} ] | [ "$APP_ENV" = 'production' ]; then echo '--only main'; fi) \
+    $(if [ -z "${APP_ENV}" ] || [ "$APP_ENV" = 'production' ]; then echo '--only main'; fi) \
     --no-interaction --no-ansi
 
-# Running as non-root user:
+# Run as non-root
 USER web
 
-# The following stage is only for production:
-# FROM development_build AS production_build
+# Stage 2: Production image (lighter)
+FROM development_build AS production_build
 COPY --chown=web:web . /code
 
-# ENV PYTHONPATH=/opt/app-root/src
+EXPOSE 8080
 
-CMD gunicorn --preload --bind 0.0.0.0:${PORT} --config /code/gunicorn_config.py wsgi:app
+CMD gunicorn --bind 0.0.0.0:${PORT} --config /code/gunicorn_config.py wsgi:app

services/emailer/Makefile

@@ -0,0 +1,81 @@
+.PHONY: license
+.PHONY: setup
+.PHONY: ci cd
+.PHONY: run
+
+MKFILE_PATH:=$(abspath $(lastword $(MAKEFILE_LIST)))
+CURRENT_ABS_DIR:=$(patsubst %/,%,$(dir $(MKFILE_PATH)))
+
+PROJECT_NAME:=solr_names_updater
+DOCKER_NAME:=solr-names-updater
+
+#################################################################################
+# COMMANDS -- Setup                                                             #
+#################################################################################
+setup: install ## Setup the project
+
+clean: clean-build clean-pyc clean-test ## Clean the project
+	rm -rf .venv/
+	rm -rf poetry.lock
+
+clean-build: ## Clean build files
+	rm -fr build/
+	rm -fr dist/
+	rm -fr .eggs/
+	find . -name '*.egg-info' -exec rm -fr {} +
+	find . -name '*.egg' -exec rm -fr {} +
+
+clean-pyc: ## Clean cache files
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	find . -name '__pycache__' -exec rm -fr {} +
+
+clean-test: ## clean test files
+	find . -name '.pytest_cache' -exec rm -fr {} +
+	rm -fr .tox/
+	rm -f .coverage
+	rm -fr htmlcov/
+
+update: ## Upgrade lock
+	poetry update
+
+install: clean ## Install python virtrual environment
+	test -f .venv/bin/activate || python3.12 -m venv  $(CURRENT_ABS_DIR)/.venv ;\
+	. .venv/bin/activate ;\
+	pipx install poetry
+	poetry install
+
+#################################################################################
+# COMMANDS - CI                                                                 #
+#################################################################################
+ci: docker-build-check ruff test ## CI flow
+
+docker-build-check: ## Check if Dockerfile builds successfully
+	docker build -f Dockerfile -t namex-emailer .
+
+ruff: ## ruff linter
+	poetry run ruff check
+
+ruff-fix: ## auto fix lint issues with ruff
+	poetry run ruff check --fix
+
+test: ## unit testing with local db
+	poetry run pytest
+
+#################################################################################
+# COMMANDS - Local                                                              #
+#################################################################################
+
+run: ## Run the project in local
+	. .venv/bin/activate && python3.12 run.py
+
+#################################################################################
+# Self Documenting Commands                                                     #
+#################################################################################
+.PHONY: help
+
+.DEFAULT_GOAL := help
+
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'