Namex pay db update (#1891) (#1892)

- sync namex-pay to namex-api cloudsql db connector / keyless pubsub

Author: bolyachevets

services/namex-pay/.env.sample

@@ -1,31 +1,36 @@
 SECRET_KEY=
 
 # Database
-NAMEX_DATABASE_HOST=
-NAMEX_DATABASE_NAME=
-NAMEX_DATABASE_PASSWORD=
-NAMEX_DATABASE_PORT=
-NAMEX_DATABASE_USERNAME=
+# only for local db
+# check dev-scripts/local-db/docker-compose.yml for exact values
+# similar values are also used in https://github.com/bcgov/bcregistry-sre/blob/main/.github/workflows/backend-ci.yaml
+DATABASE_USERNAME=postgres
+DATABASE_PASSWORD="postgres"
+DATABASE_NAME="unittesting"
+DATABASE_HOST="localhost"
+DATABASE_PORT="54345"
+DATABASE_SCHEMA="public"
+DATABASE_OWNER="postgres"
 
-DATABASE_TEST_HOST=
-DATABASE_TEST_NAME=
+# only when conenecting to clousql db
+DATABASE_INSTANCE_CONNECTION_NAME=
+DATABASE_NAME=namex
+DATABASE_USERNAME="...@gov.bc.ca" # your email, which needs to be added as IAM user to cloudsql instance and granted readwrite access
+DATABASE_IP_TYPE=public
+DATABASE_OWNER=userHQH
+
+#unit tests
+DATABASE_TEST_USERNAME=
 DATABASE_TEST_PASSWORD=
+DATABASE_TEST_NAME=
+DATABASE_TEST_HOST=
 DATABASE_TEST_PORT=
-DATABASE_TEST_USERNAME=
-
-# Oracle Database
-NRO_USER=
-NRO_PASSWORD=
-NRO_DB_NAME=
-ORACLE_HOST=
-ORACLE_PORT=1521
 
 # APIs
 PAY_API_URL=
 PAY_API_VERSION=
 
-BUSINESS_GCP_AUTH_KEY=
 NAMEX_MAILER_TOPIC=
-PAY_SUB_AUDIENCE=
-AUTHPAY_SERVICE_ACCOUNT=
+NAMEX_NR_STATE_TOPIC=
+
 DEBUG_REQUEST=

services/namex-pay/Dockerfile

@@ -1,40 +1,40 @@
-FROM python:3.12.2
-USER root
+# Stage 1: Build environment (with Poetry + dependencies installed)
+FROM python:3.12.5-slim AS development_build
 
 ARG VCS_REF="missing"
 ARG BUILD_DATE="missing"
 
 ENV VCS_REF=${VCS_REF}
 ENV BUILD_DATE=${BUILD_DATE}
-
+ENV PORT=8080
 
 LABEL org.label-schema.vcs-ref=${VCS_REF} \
-    org.label-schema.build-date=${BUILD_DATE}
+      org.label-schema.build-date=${BUILD_DATE} \
+      vendor="BCROS"
 
 USER root
 
 ARG APP_ENV \
-  # Needed for fixing permissions of files created by Docker:
-  UID=1000 \
-  GID=1000
+    UID=1000 \
+    GID=1000
 
 ENV APP_ENV=${APP_ENV} \
-  # python:
-  PYTHONFAULTHANDLER=1 \
-  PYTHONUNBUFFERED=1 \
-  PYTHONHASHSEED=random \
-  PYTHONDONTWRITEBYTECODE=1 \
-  # pip:
-  PIP_NO_CACHE_DIR=1 \
-  PIP_DISABLE_PIP_VERSION_CHECK=1 \
-  PIP_DEFAULT_TIMEOUT=100 \
-  PIP_ROOT_USER_ACTION=ignore \
-  # poetry:
-  POETRY_VERSION=1.3.2 \
-  POETRY_NO_INTERACTION=1 \
-  POETRY_VIRTUALENVS_CREATE=false \
-  POETRY_CACHE_DIR='/var/cache/pypoetry' \
-  POETRY_HOME='/usr/local'
+    # python:
+    PYTHONFAULTHANDLER=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONHASHSEED=random \
+    PYTHONDONTWRITEBYTECODE=1 \
+    # pip:
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_DEFAULT_TIMEOUT=100 \
+    PIP_ROOT_USER_ACTION=ignore \
+    # poetry:
+    POETRY_VERSION=2.1.3 \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_CACHE_DIR='/var/cache/pypoetry' \
+    POETRY_HOME='/usr/local'
 
 SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 
@@ -50,7 +50,7 @@ RUN apt-get update && apt-get upgrade -y \
     wait-for-it \
   && curl -sSL 'https://install.python-poetry.org' | python - \
   && poetry --version \
-  # Cleaning cache:
+  # Clean up apt cache
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
@@ -60,29 +60,29 @@ RUN groupadd -g "${GID}" -r web \
   && useradd -d '/code' -g web -l -r -u "${UID}" web \
   && chown web:web -R '/code'
 
-# Copy only requirements, to cache them in docker layer
+# Copy dependency files first (better caching)
 COPY --chown=web:web ./poetry.lock ./pyproject.toml /code/
 
+# Copy app source
 COPY --chown=web:web ./src /code/src
 COPY --chown=web:web ./README.md /code
 
-# Project initialization:
+# Install dependencies
 RUN --mount=type=cache,target="$POETRY_CACHE_DIR" \
   echo "$APP_ENV" \
   && poetry version \
-  # Install deps:
-  && poetry run pip install -U pip \
+  && poetry update \
   && poetry install \
-    $(if [ -z ${APP_ENV+x} ] | [ "$APP_ENV" = 'production' ]; then echo '--only main'; fi) \
+    $(if [ -z ${APP_ENV+x} ] || [ "$APP_ENV" = 'production' ]; then echo '--only main'; fi) \
     --no-interaction --no-ansi
 
-# Running as non-root user:
+# Run as non-root
 USER web
 
-# The following stage is only for production:
-# FROM development_build AS production_build
+# Stage 2: Production image (lighter)
+FROM development_build AS production_build
 COPY --chown=web:web . /code
 
 EXPOSE 8080
 
-CMD gunicorn --bind 0.0.0.0:8080 --config /code/gunicorn_config.py wsgi:app
+CMD gunicorn --bind 0.0.0.0:${PORT} --config /code/gunicorn_config.py wsgi:app

services/namex-pay/Makefile

@@ -50,19 +50,19 @@ install: clean ## Install python virtrual environment
 #################################################################################
 # COMMANDS - CI                                                                 #
 #################################################################################
-ci: pylint flake8 test ## CI flow
+ci: docker-build-check ruff test ## CI flow
 
-pylint: ## Linting with pylint
-	. .venv/bin/activate && pylint --rcfile=setup.cfg src/$(PROJECT_NAME)
+docker-build-check: ## Check if Dockerfile builds successfully
+	docker build -f Dockerfile -t namex-pay .
 
-flake8: ## Linting with flake8 ## tests
-	. .venv/bin/activate && flake8 src/$(PROJECT_NAME)
+ruff: ## ruff linter
+	poetry run ruff check
 
-test: ## Unit testing
-	. .venv/bin/activate && pytest
+ruff-fix: ## auto fix lint issues with ruff
+	poetry run ruff check --fix
 
-mac-cov: test ## Run the coverage report and display in a browser window (mac)
-	@open -a "Google Chrome" htmlcov/index.html
+test: ## unit testing with local db
+	poetry run pytest
 
 #################################################################################
 # COMMANDS - Local                                                              #

services/namex-pay/config.py

@@ -20,8 +20,8 @@
 or by accessing this configuration directly.
 """
 import os
-from dotenv import find_dotenv, load_dotenv
 
+from dotenv import find_dotenv, load_dotenv
 
 # this will load all the envars from a .env file located in the project root (api)
 load_dotenv(find_dotenv())
@@ -64,23 +64,25 @@ class Config():  # pylint: disable=too-few-public-methods
     PAYMENT_SVC_VERSION = os.getenv('PAY_API_VERSION', None)
 
     # POSTGRESQL
-    DB_USER = os.getenv('NAMEX_DATABASE_USERNAME', '')
-    DB_PASSWORD = os.getenv('NAMEX_DATABASE_PASSWORD', '')
-    DB_NAME = os.getenv('NAMEX_DATABASE_NAME', '')
-    DB_HOST = os.getenv('NAMEX_DATABASE_HOST', '')
-    DB_PORT = os.getenv('NAMEX_DATABASE_PORT', '5432')
-    if DB_UNIX_SOCKET := os.getenv('NAMEX_DATABASE_UNIX_SOCKET', None):
-        SQLALCHEMY_DATABASE_URI = f'postgresql+psycopg2://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?host={DB_UNIX_SOCKET}'
+    DB_USER = os.getenv('DATABASE_USERNAME', 'postgres')
+    DB_PASSWORD = os.getenv('DATABASE_PASSWORD', 'postgres')
+    DB_NAME = os.getenv('DATABASE_NAME', 'unittesting')
+    DB_HOST = os.getenv('DATABASE_HOST', 'localhost')
+    DB_PORT = int(os.getenv('DATABASE_PORT', '5432'))
+
+    DB_SCHEMA = os.getenv('DATABASE_SCHEMA', 'public')
+    DB_IP_TYPE = os.getenv('DATABASE_IP_TYPE', 'private')
+    DB_OWNER = os.getenv('DATABASE_OWNER', 'postgres')
+
+    if DB_INSTANCE_CONNECTION_NAME := os.getenv('DATABASE_INSTANCE_CONNECTION_NAME', None):
+        SQLALCHEMY_DATABASE_URI = 'postgresql+pg8000://'
     else:
-        SQLALCHEMY_DATABASE_URI = f'postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}'
+        SQLALCHEMY_DATABASE_URI = f'postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}'
 
-    GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None)
     EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '')
     NAMEX_NR_STATE_TOPIC = os.getenv('NAMEX_NR_STATE_TOPIC', '')
     AUDIENCE = os.getenv('AUDIENCE', 'https://pubsub.googleapis.com/google.pubsub.v1.Subscriber')
     PUBLISHER_AUDIENCE = os.getenv('PUBLISHER_AUDIENCE', 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher')
-    SUB_AUDIENCE = os.getenv('PAY_SUB_AUDIENCE', '')
-    SUB_SERVICE_ACCOUNT = os.getenv('AUTHPAY_SERVICE_ACCOUNT', '')
     DEBUG_REQUEST = os.getenv('DEBUG_REQUEST', False)
 
     ENVIRONMENT = os.getenv('ENVIRONMENT', 'prod')
@@ -104,14 +106,21 @@ class TestConfig(Config):  # pylint: disable=too-few-public-methods
     # POSTGRESQL
     DB_USER = os.getenv('DATABASE_TEST_USERNAME', 'postgres')
     DB_PASSWORD = os.getenv('DATABASE_TEST_PASSWORD', 'postgres')
-    DB_NAME = os.getenv('DATABASE_TEST_NAME', 'namex_pay')
-    DB_HOST = os.getenv('DATABASE_TEST_HOST', '127.0.0.1')
+    DB_NAME = os.getenv('DATABASE_TEST_NAME', 'unittesting')
+    DB_HOST = os.getenv('DATABASE_TEST_HOST', 'localhost')
     DB_PORT = os.getenv('DATABASE_TEST_PORT', '5432')
-    SQLALCHEMY_DATABASE_URI = f'postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}'
+    SQLALCHEMY_DATABASE_URI = f'postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}'
     EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '')
+    NAMEX_NR_STATE_TOPIC = os.getenv('NAMEX_NR_STATE_TOPIC', '')
 
 class ProdConfig(Config):  # pylint: disable=too-few-public-methods
     """Production environment configuration."""
 
     TESTING = False
     DEBUG = False
+
+class MigrationConfig(Config):
+    """Config for db migration."""
+
+    TESTING = False
+    DEBUG = True

services/namex-pay/devops/gcp/clouddeploy.yaml

@@ -32,6 +32,7 @@ serialPipeline:
       container-name: "namex-pay-dev"
       service-account: "sa-api@a083gt-dev.iam.gserviceaccount.com"
       cloudsql-instances: "a083gt-dev:northamerica-northeast1:namex-db-dev"
+      container-concurrency: "8"
  - targetId: a083gt-test
    profiles: [test]
    strategy:
@@ -45,6 +46,7 @@ serialPipeline:
       container-name: "namex-pay-test"
       service-account: "sa-api@a083gt-test.iam.gserviceaccount.com"
       cloudsql-instances: "a083gt-test:northamerica-northeast1:namex-db-test"
+      container-concurrency: "8"
  - targetId: a083gt-prod
    profiles: [prod]
    strategy:
@@ -57,4 +59,5 @@ serialPipeline:
       service-name: "namex-pay-prod"
       container-name: "namex-pay-prod"
       service-account: "sa-api@a083gt-prod.iam.gserviceaccount.com"
-      cloudsql-instances: "a083gt-prod:northamerica-northeast1:namex-db-prod"
+      cloudsql-instances: "a083gt-prod:northamerica-northeast1:namex-db-prod"
+      container-concurrency: "8"

services/namex-pay/devops/vaults.gcp.env

@@ -1,15 +1,11 @@
-NAMEX_DATABASE_UNIX_SOCKET="op://database/$APP_ENV/namex-db-gcp/DATABASE_UNIX_SOCKET"
-NAMEX_DATABASE_PORT="op://database/$APP_ENV/namex-db-gcp/DATABASE_PORT"
-NAMEX_DATABASE_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_NAME"
-NAMEX_DATABASE_USERNAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_USERNAME"
-NAMEX_DATABASE_PASSWORD="op://database/$APP_ENV/namex-db-gcp/DATABASE_PASSWORD"
+DATABASE_USERNAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_USERNAME"
+DATABASE_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_NAME"
+DATABASE_INSTANCE_CONNECTION_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_INSTANCE_CONNECTION_NAME"
+DATABASE_SCHEMA="op://database/$APP_ENV/namex-db-gcp/DATABASE_SCHEMA"
 PAY_API_URL="op://API/$APP_ENV/pay-api/PAY_API_URL"
 PAY_API_VERSION="op://API/$APP_ENV/pay-api/PAY_API_VERSION"
 AUDIENCE="op://gcp-queue/$APP_ENV/base/AUDIENCE"
 PUBLISHER_AUDIENCE="op://gcp-queue/$APP_ENV/base/PUBLISHER_AUDIENCE"
 NAMEX_MAILER_TOPIC="op://gcp-queue/$APP_ENV/topics/NAMEX_MAILER_TOPIC"
 NAMEX_NR_STATE_TOPIC="op://gcp-queue/$APP_ENV/topics/NAMEX_NR_STATE_TOPIC"
-BUSINESS_GCP_AUTH_KEY="op://gcp-queue/$APP_ENV/a083gt/BUSINESS_GCP_AUTH_KEY"
-PAY_SUB_AUDIENCE="op://gcp-queue/$APP_ENV/namex/PAY_SUB_AUDIENCE_GCP"
-AUTHPAY_SERVICE_ACCOUNT="op://gcp-queue/$APP_ENV/gtksf3/AUTHPAY_SERVICE_ACCOUNT"
 VPC_CONNECTOR="op://CD/$APP_ENV/namex-api/VPC_CONNECTOR"

services/namex-pay/gunicorn_config.py

@@ -1,45 +1,9 @@
-# Copyright © 2023 Province of British Columbia
-#
-# Licensed under the BSD 3 Clause License, (the "License");
-# you may not use this file except in compliance with the License.
-# The template for the license can be found here
-#    https://opensource.org/license/bsd-3-clause/
-#
-# Redistribution and use in source and binary forms,
-# with or without modification, are permitted provided that the
-# following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-#    this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-#    this list of conditions and the following disclaimer in the documentation
-#    and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-#    may be used to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS”
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
-# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-"""The configuration for gunicorn, which picks up the
-   runtime options from environment variables
-"""
-
 import os
 
-workers = int(os.environ.get("GUNICORN_PROCESSES", "1"))  # pylint: disable=invalid-name
-threads = int(os.environ.get("GUNICORN_THREADS", "1"))  # pylint: disable=invalid-name
-timeout = int(os.environ.get("GUNICORN_TIMEOUT", "600"))  # pylint: disable=invalid-name
+workers = int(os.environ.get('GUNICORN_PROCESSES', '1'))  # pylint: disable=invalid-name
+threads = int(os.environ.get('GUNICORN_THREADS', '8'))  # pylint: disable=invalid-name
+timeout = int(os.environ.get('GUNICORN_TIMEOUT', '0'))  # pylint: disable=invalid-name
+
 
-forwarded_allow_ips = "*"  # pylint: disable=invalid-name
-secure_scheme_headers = {"X-Forwarded-Proto": "https"}  # pylint: disable=invalid-name
+forwarded_allow_ips = '*'
+secure_scheme_headers = {'X-Forwarded-Proto': 'https'}