Solr updater upgrade (#1896)

api/devops/gcp/clouddeploy.yaml

@@ -34,6 +34,7 @@ serialPipeline:
       cloudsql-instances: "a083gt-dev:northamerica-northeast1:namex-db-dev"
       resources-cpu: 1000m
       resources-memory: 1Gi
+      container-concurrency: "8"
  - targetId: a083gt-test
    profiles: [test]
    strategy:
@@ -49,6 +50,7 @@ serialPipeline:
       cloudsql-instances: "a083gt-test:northamerica-northeast1:namex-db-test"
       resources-cpu: 2000m
       resources-memory: 2Gi
+      container-concurrency: "8"
  - targetId: a083gt-sandbox
    profiles: [sandbox]
    strategy:
@@ -64,6 +66,7 @@ serialPipeline:
       cloudsql-instances: "a083gt-integration:northamerica-northeast1:namex-db-integration"
       resources-cpu: 4000m
       resources-memory: 4Gi
+      container-concurrency: "8"
  - targetId: a083gt-prod
    profiles: [prod]
    strategy:
@@ -79,3 +82,4 @@ serialPipeline:
       cloudsql-instances: "a083gt-prod:northamerica-northeast1:namex-db-prod"
       resources-cpu: 4000m
       resources-memory: 8Gi
+      container-concurrency: "8"

services/namex-pay/Dockerfile

@@ -71,6 +71,7 @@ COPY --chown=web:web ./README.md /code
 RUN --mount=type=cache,target="$POETRY_CACHE_DIR" \
   echo "$APP_ENV" \
   && poetry version \
+  && poetry config installer.max-workers 1 \
   && poetry update \
   && poetry install \
     $(if [ -z ${APP_ENV+x} ] || [ "$APP_ENV" = 'production' ]; then echo '--only main'; fi) \

services/namex-pay/pyproject.toml

@@ -9,10 +9,8 @@ packages = [{include = "namex_pay", from = "src"}]
 
 [tool.poetry.dependencies]
 python = ">=3.12,<3.13"
-namex = { git = "https://github.com/bcgov/namex.git", subdirectory = "api" }
+namex = { git = "https://github.com/bcgov/namex.git", branch = "main", subdirectory = "api" }
 registry_schemas = { git = "https://github.com/bcgov/business-schemas.git"}
-swagger_client = { git = "https://github.com/bcgov/namex-synonyms-api-py-client.git" }
-sbc-common-components = { git = "https://github.com/bcgov/sbc-common-components.git", subdirectory = "python" }
 
 pyhumps = "3.8.0"
 

services/solr-names-updater/Dockerfile

@@ -1,40 +1,40 @@
-FROM python:3.12.2
-USER root
+# Stage 1: Build environment (with Poetry + dependencies installed)
+FROM python:3.12-slim AS development_build
 
 ARG VCS_REF="missing"
 ARG BUILD_DATE="missing"
 
 ENV VCS_REF=${VCS_REF}
 ENV BUILD_DATE=${BUILD_DATE}
-
+ENV PORT=8080
 
 LABEL org.label-schema.vcs-ref=${VCS_REF} \
-    org.label-schema.build-date=${BUILD_DATE}
+      org.label-schema.build-date=${BUILD_DATE} \
+      vendor="BCROS"
 
 USER root
 
 ARG APP_ENV \
-  # Needed for fixing permissions of files created by Docker:
-  UID=1000 \
-  GID=1000
+    UID=1000 \
+    GID=1000
 
 ENV APP_ENV=${APP_ENV} \
-  # python:
-  PYTHONFAULTHANDLER=1 \
-  PYTHONUNBUFFERED=1 \
-  PYTHONHASHSEED=random \
-  PYTHONDONTWRITEBYTECODE=1 \
-  # pip:
-  PIP_NO_CACHE_DIR=1 \
-  PIP_DISABLE_PIP_VERSION_CHECK=1 \
-  PIP_DEFAULT_TIMEOUT=100 \
-  PIP_ROOT_USER_ACTION=ignore \
-  # poetry:
-  POETRY_VERSION=1.3.2 \
-  POETRY_NO_INTERACTION=1 \
-  POETRY_VIRTUALENVS_CREATE=false \
-  POETRY_CACHE_DIR='/var/cache/pypoetry' \
-  POETRY_HOME='/usr/local'
+    # python:
+    PYTHONFAULTHANDLER=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONHASHSEED=random \
+    PYTHONDONTWRITEBYTECODE=1 \
+    # pip:
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_DEFAULT_TIMEOUT=100 \
+    PIP_ROOT_USER_ACTION=ignore \
+    # poetry:
+    POETRY_VERSION=2.1.3 \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_CACHE_DIR='/var/cache/pypoetry' \
+    POETRY_HOME='/usr/local'
 
 SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
 
@@ -50,7 +50,7 @@ RUN apt-get update && apt-get upgrade -y \
     wait-for-it \
   && curl -sSL 'https://install.python-poetry.org' | python - \
   && poetry --version \
-  # Cleaning cache:
+  # Clean up apt cache
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
@@ -60,29 +60,30 @@ RUN groupadd -g "${GID}" -r web \
   && useradd -d '/code' -g web -l -r -u "${UID}" web \
   && chown web:web -R '/code'
 
-# Copy only requirements, to cache them in docker layer
+# Copy dependency files first (better caching)
 COPY --chown=web:web ./poetry.lock ./pyproject.toml /code/
 
+# Copy app source
 COPY --chown=web:web ./src /code/src
 COPY --chown=web:web ./README.md /code
 
-# Project initialization:
+# Install dependencies
 RUN --mount=type=cache,target="$POETRY_CACHE_DIR" \
   echo "$APP_ENV" \
+  && poetry config installer.max-workers 1 \
   && poetry version \
-  # Install deps:
   && poetry run pip install -U pip \
   && poetry install \
-    $(if [ -z ${APP_ENV+x} ] | [ "$APP_ENV" = 'production' ]; then echo '--only main'; fi) \
+    $(if [ -z "${APP_ENV}" ] || [ "$APP_ENV" = 'production' ]; then echo '--only main'; fi) \
     --no-interaction --no-ansi
 
-# Running as non-root user:
+# Run as non-root
 USER web
 
-# The following stage is only for production:
-# FROM development_build AS production_build
+# Stage 2: Production image (lighter)
+FROM development_build AS production_build
 COPY --chown=web:web . /code
 
 EXPOSE 8080
 
-CMD gunicorn --bind 0.0.0.0:8080 --config /code/gunicorn_config.py wsgi:app
+CMD gunicorn --bind 0.0.0.0:${PORT} --config /code/gunicorn_config.py wsgi:app

services/solr-names-updater/Makefile

@@ -49,21 +49,19 @@ install: clean ## Install python virtrual environment
 #################################################################################
 # COMMANDS - CI                                                                 #
 #################################################################################
-ci: lint flake8 test ## CI flow
+ci: docker-build-check ruff test ## CI flow
 
-pylint: ## Linting with pylint
-	. .venv/bin/activate && pylint --rcfile=setup.cfg src/$(PROJECT_NAME)
+docker-build-check: ## Check if Dockerfile builds successfully
+	docker build -f Dockerfile -t solr-names-updater .
 
-flake8: ## Linting with flake8 ## tests
-	. .venv/bin/activate && flake8 src/$(PROJECT_NAME)
+ruff: ## ruff linter
+	poetry run ruff check
 
-lint: pylint flake8 ## run all lint type scripts
+ruff-fix: ## auto fix lint issues with ruff
+	poetry run ruff check --fix
 
-test: ## Unit testing
-	. .venv/bin/activate && pytest
-
-mac-cov: test ## Run the coverage report and display in a browser window (mac)
-	@open -a "Google Chrome" htmlcov/index.html
+test: ## unit testing with local db
+	poetry run pytest
 
 #################################################################################
 # COMMANDS - Local                                                              #

services/solr-names-updater/config.py

@@ -20,11 +20,9 @@
 or by accessing this configuration directly.
 """
 import os
-import random
 
 from dotenv import find_dotenv, load_dotenv
 
-
 # this will load all the envars from a .env file located in the project root (api)
 load_dotenv(find_dotenv())
 
@@ -60,28 +58,30 @@ class Config():  # pylint: disable=too-few-public-methods
 
     PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
 
-    SECRET_KEY = 'a secret'
+    SECRET_KEY = 'a secret' # noqa: S105
 
     SQLALCHEMY_TRACK_MODIFICATIONS = False
 
     # POSTGRESQL
-    DB_USER = os.getenv('DATABASE_USERNAME', '')
-    DB_PASSWORD = os.getenv('DATABASE_PASSWORD', '')
-    DB_NAME = os.getenv('DATABASE_NAME', '')
-    DB_HOST = os.getenv('DATABASE_HOST', '')
-    DB_PORT = os.getenv('DATABASE_PORT', '5432')
-    if DB_UNIX_SOCKET := os.getenv('DATABASE_UNIX_SOCKET', None):
-        SQLALCHEMY_DATABASE_URI = f'postgresql+psycopg2://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?host={DB_UNIX_SOCKET}'
+    DB_USER = os.getenv('DATABASE_USERNAME', 'postgres')
+    DB_PASSWORD = os.getenv('DATABASE_PASSWORD', 'postgres')
+    DB_NAME = os.getenv('DATABASE_NAME', 'unittesting')
+    DB_HOST = os.getenv('DATABASE_HOST', 'localhost')
+    DB_PORT = int(os.getenv('DATABASE_PORT', '5432'))
+
+    DB_SCHEMA = os.getenv('DATABASE_SCHEMA', 'public')
+    DB_IP_TYPE = os.getenv('DATABASE_IP_TYPE', 'private')
+    DB_OWNER = os.getenv('DATABASE_OWNER', 'postgres')
+
+    if DB_INSTANCE_CONNECTION_NAME := os.getenv('DATABASE_INSTANCE_CONNECTION_NAME', None):
+        SQLALCHEMY_DATABASE_URI = 'postgresql+pg8000://'
     else:
-        SQLALCHEMY_DATABASE_URI = f'postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}'
+        SQLALCHEMY_DATABASE_URI = f'postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}'
 
-    GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None)
     EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '')
     AUDIENCE = os.getenv('AUDIENCE', 'https://pubsub.googleapis.com/google.pubsub.v1.Subscriber')
     PUBLISHER_AUDIENCE = os.getenv('PUBLISHER_AUDIENCE', 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher')
     DEBUG_REQUEST = os.getenv('DEBUG_REQUEST', False)
-    SUB_AUDIENCE = os.getenv('SOLR_SUB_AUDIENCE', '')
-    SUB_SERVICE_ACCOUNT = os.getenv('BUSINESS_SERVICE_ACCOUNT', '')
 
     SOLR_FEEDER_API_URL = os.getenv('SOLR_FEEDER_API_URL', None)
 
@@ -118,22 +118,18 @@ class TestConfig(Config):  # pylint: disable=too-few-public-methods
 
     DEBUG = True
     TESTING = True
+
     # POSTGRESQL
     DB_USER = os.getenv('DATABASE_TEST_USERNAME', 'postgres')
     DB_PASSWORD = os.getenv('DATABASE_TEST_PASSWORD', 'postgres')
-    DB_NAME = os.getenv('DATABASE_TEST_NAME', 'namex_pay')
-    DB_HOST = os.getenv('DATABASE_TEST_HOST', '127.0.0.1')
+    DB_NAME = os.getenv('DATABASE_TEST_NAME', 'unittesting')
+    DB_HOST = os.getenv('DATABASE_TEST_HOST', 'localhost')
     DB_PORT = os.getenv('DATABASE_TEST_PORT', '5432')
     # Set this in your .env to debug SQL Alchemy queries (for local development)
     # pylint: disable=invalid-envvar-default
     SQLALCHEMY_ECHO = 'debug' if os.getenv('DEBUG_SQL_QUERIES', False) else False
-    SQLALCHEMY_DATABASE_URI = 'postgresql://{user}:{password}@{host}:{port}/{name}'.format(
-        user=DB_USER,
-        password=DB_PASSWORD,
-        host=DB_HOST,
-        port=int(DB_PORT),
-        name=DB_NAME,
-    )
+    SQLALCHEMY_DATABASE_URI = f'postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}'
+
     SOLR_FEEDER_API_URL = os.getenv('SOLR_FEEDER_API_URL', 'https://mock-solr-feeder/api/v1')
 
     # JWT OIDC settings
@@ -197,3 +193,9 @@ class ProdConfig(Config):  # pylint: disable=too-few-public-methods
 
     TESTING = False
     DEBUG = False
+
+class MigrationConfig(Config):
+    """Config for db migration."""
+
+    TESTING = False
+    DEBUG = True

services/solr-names-updater/devops/gcp/clouddeploy.yaml

@@ -32,6 +32,7 @@ serialPipeline:
       container-name: "namex-solr-names-updater-dev"
       service-account: "sa-api@a083gt-dev.iam.gserviceaccount.com"
       cloudsql-instances: "a083gt-dev:northamerica-northeast1:namex-db-dev"
+      container-concurrency: "8"
  - targetId: a083gt-test
    profiles: [test]
    strategy:
@@ -45,6 +46,7 @@ serialPipeline:
       container-name: "namex-solr-names-updater-test"
       service-account: "sa-api@a083gt-test.iam.gserviceaccount.com"
       cloudsql-instances: "a083gt-test:northamerica-northeast1:namex-db-test"
+      container-concurrency: "8"
  - targetId: a083gt-prod
    profiles: [prod]
    strategy:
@@ -57,4 +59,5 @@ serialPipeline:
       service-name: "namex-solr-names-updater-prod"
       container-name: "namex-solr-names-updater-prod"
       service-account: "sa-api@a083gt-prod.iam.gserviceaccount.com"
-      cloudsql-instances: "a083gt-prod:northamerica-northeast1:namex-db-prod"
+      cloudsql-instances: "a083gt-prod:northamerica-northeast1:namex-db-prod"
+      container-concurrency: "8"

services/solr-names-updater/devops/vaults.gcp.env

@@ -1,17 +1,13 @@
 DATABASE_USERNAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_USERNAME"
-DATABASE_PASSWORD="op://database/$APP_ENV/namex-db-gcp/DATABASE_PASSWORD"
-DATABASE_PORT="op://database/$APP_ENV/namex-db-gcp/DATABASE_PORT"
 DATABASE_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_NAME"
-DATABASE_UNIX_SOCKET="op://database/$APP_ENV/namex-db-gcp/DATABASE_UNIX_SOCKET"
+DATABASE_INSTANCE_CONNECTION_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_INSTANCE_CONNECTION_NAME"
+DATABASE_SCHEMA="op://database/$APP_ENV/namex-db-gcp/DATABASE_SCHEMA"
 SOLR_FEEDER_API_URL="op://namex/$APP_ENV/solr-updater/SOLR_FEEDER_API_URL_GCP"
 JWT_OIDC_AUDIENCE="op://namex/$APP_ENV/jwt/JWT_OIDC_AUDIENCE"
 JWT_OIDC_CLIENT_SECRET="op://namex/$APP_ENV/jwt/JWT_OIDC_CLIENT_SECRET"
 AUDIENCE="op://gcp-queue/$APP_ENV/base/AUDIENCE"
 PUBLISHER_AUDIENCE="op://gcp-queue/$APP_ENV/base/PUBLISHER_AUDIENCE"
-SOLR_SUB_AUDIENCE="op://gcp-queue/$APP_ENV/namex/SOLR_SUB_AUDIENCE_GCP"
 NAMEX_MAILER_TOPIC="op://gcp-queue/$APP_ENV/topics/NAMEX_MAILER_TOPIC"
-BUSINESS_GCP_AUTH_KEY="op://gcp-queue/$APP_ENV/a083gt/BUSINESS_GCP_AUTH_KEY"
-BUSINESS_SERVICE_ACCOUNT="op://gcp-queue/$APP_ENV/a083gt/BUSINESS_SERVICE_ACCOUNT"
 JWT_OIDC_ISSUER="op://keycloak/$APP_ENV/jwt-base/JWT_OIDC_ISSUER"
 JWT_OIDC_ALGORITHMS="op://keycloak/$APP_ENV/jwt-base/JWT_OIDC_ALGORITHMS"
 JWT_OIDC_JWKS_CACHE_TIMEOUT="op://keycloak/$APP_ENV/jwt-base/JWT_OIDC_JWKS_CACHE_TIMEOUT"