Feature db connector solr synonyms api

services/emailer/gunicorn_config.py

@@ -38,7 +38,7 @@
 import os
 
 workers = int(os.environ.get("GUNICORN_PROCESSES", "1"))
-threads = int(os.environ.get("GUNICORN_THREADS", "1"))
+threads = int(os.environ.get("GUNICORN_THREADS", "8"))
 timeout = int(os.environ.get("GUNICORN_TIMEOUT", "600"))
 
 forwarded_allow_ips = "*"

solr-synonyms-api/Makefile

@@ -48,21 +48,19 @@ install: clean ## Install python virtrual environment
 #################################################################################
 # COMMANDS - CI                                                                 #
 #################################################################################
-ci: pylint flake8 test ## CI flow
+ci: docker-build-check ruff test ## CI flow
 
-pylint: ## Linting with pylint
-	. .venv/bin/activate && pylint --rcfile=setup.cfg  $(PROJECT_NAME)
+docker-build-check: ## Check if Dockerfile builds successfully
+	docker build -f Dockerfile -t namex-emailer .
 
-flake8: ## Linting with flake8
-	. .venv/bin/activate && flake8 $(PROJECT_NAME) tests
+ruff: ## ruff linter
+	poetry run ruff check
 
-lint: pylint flake8 ## run all lint type scripts
+ruff-fix: ## auto fix lint issues with ruff
+	poetry run ruff check --fix
 
-test: ## Unit testing
-	. .venv/bin/activate && pytest
-
-mac-cov: test ## Run the coverage report and display in a browser window (mac)
-	@open -a "Google Chrome" htmlcov/index.html
+test: ## unit testing with local db
+	poetry run pytest
 
 #################################################################################
 # COMMANDS - Local                                                              #

solr-synonyms-api/config.py

@@ -3,42 +3,38 @@
 
 import dotenv
 
-
 # Load all the environment variables from a .env file located in some directory above.
 dotenv.load_dotenv(dotenv.find_dotenv())
 
-CONFIGURATION = {
-    'development': 'config.DevConfig',
-    'testing': 'config.TestConfig',
-    'production': 'config.Config',
-    'default': 'config.Config'
-}
-
-
 class Config(object):
     """Create base config object."""
 
     PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
     # The Flask secret key used to encrypt cookies. This must be kept secret and should be unique per environment. Do
     # not allow a missing value.
-    SECRET_KEY = os.getenv('SOLR_SYNONYMS_API_FLASK_SECRET_KEY')
+    SECRET_KEY = os.getenv("SOLR_SYNONYMS_API_FLASK_SECRET_KEY")
     if not SECRET_KEY:
-        raise RuntimeError('Environment variable SOLR_SYNONYMS_API_FLASK_SECRET_KEY in not defined')
+        raise RuntimeError("Environment variable SOLR_SYNONYMS_API_FLASK_SECRET_KEY in not defined")
 
     # Turn this off to get rid of warning messages. In future versions of SQLAlchemy, False will be the default and
     # this can be removed.
     SQLALCHEMY_TRACK_MODIFICATIONS = False
 
-    # PostgreSQL Connection information.
-    DATABASE_USER = os.getenv('SOLR_SYNONYMS_API_DATABASE_USERNAME', '')
-    DATABASE_PASSWORD = os.getenv('SOLR_SYNONYMS_API_DATABASE_PASSWORD', '')
-    DATABASE_HOST = os.getenv('SOLR_SYNONYMS_API_DATABASE_HOST', '')
-    DATABASE_PORT = os.getenv('SOLR_SYNONYMS_API_DATABASE_PORT', '5432')
-    DATABASE_NAME = os.getenv('SOLR_SYNONYMS_API_DATABASE_NAME', 'solr')
-    if DATABASE_UNIX_SOCKET := os.getenv('SOLR_SYNONYMS_API_DATABASE_UNIX_SOCKET', None):
-        SQLALCHEMY_DATABASE_URI = f'postgresql+psycopg2://{DATABASE_USER}:{DATABASE_PASSWORD}@/{DATABASE_NAME}?host={DATABASE_UNIX_SOCKET}'
+    # POSTGRESQL
+    DB_USER = os.getenv("DATABASE_USERNAME", "")
+    DB_PASSWORD = os.getenv("DATABASE_PASSWORD", "")
+    DB_NAME = os.getenv("DATABASE_NAME", "")
+    DB_HOST = os.getenv("DATABASE_HOST", "")
+    DB_PORT = int(os.getenv("DATABASE_PORT", "5432"))
+
+    DB_SCHEMA = os.getenv("DATABASE_SCHEMA", "public")
+    DB_IP_TYPE = os.getenv("DATABASE_IP_TYPE", "private")
+    DB_OWNER = os.getenv("DATABASE_OWNER", "postgres")
+
+    if DB_INSTANCE_CONNECTION_NAME := os.getenv("DATABASE_INSTANCE_CONNECTION_NAME", None):
+        SQLALCHEMY_DATABASE_URI = "postgresql+pg8000://"
     else:
-        SQLALCHEMY_DATABASE_URI = f'postgresql://{DATABASE_USER}:{DATABASE_PASSWORD}@{DATABASE_HOST}:{int(DATABASE_PORT)}/{DATABASE_NAME}'
+        SQLALCHEMY_DATABASE_URI = f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
 
     DEBUG = False
     TESTING = False
@@ -56,3 +52,24 @@ class TestConfig(Config):
 
     DEBUG = True
     TESTING = True
+
+    # POSTGRESQL
+    DB_USER = os.getenv("DATABASE_TEST_USERNAME", "postgres")
+    DB_PASSWORD = os.getenv("DATABASE_TEST_PASSWORD", "postgres")
+    DB_NAME = os.getenv("DATABASE_TEST_NAME", "unittesting")
+    DB_HOST = os.getenv("DATABASE_TEST_HOST", "localhost")
+    DB_PORT = os.getenv("DATABASE_TEST_PORT", "5432")
+    SQLALCHEMY_DATABASE_URI = f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}"
+
+class MigrationConfig(Config):
+    """Config for db migration."""
+
+    TESTING = False
+    DEBUG = True
+
+CONFIGURATION = {
+    "development": DevConfig,
+    "testing": TestConfig,
+    "production": Config,
+    "default": Config,
+}

solr-synonyms-api/devops/gcp/clouddeploy.yaml

@@ -32,6 +32,7 @@ serialPipeline:
       container-name: "namex-solr-synonyms-api-dev"
       service-account: "sa-api@a083gt-dev.iam.gserviceaccount.com"
       cloudsql-instances: "a083gt-dev:northamerica-northeast1:namex-db-dev"
+      container-concurrency: "8"
  - targetId: a083gt-test
    profiles: [test]
    strategy:
@@ -45,6 +46,7 @@ serialPipeline:
       container-name: "namex-solr-synonyms-api-test"
       service-account: "sa-api@a083gt-test.iam.gserviceaccount.com"
       cloudsql-instances: "a083gt-test:northamerica-northeast1:namex-db-test"
+      container-concurrency: "8"
  - targetId: a083gt-sandbox
    profiles: [sandbox]
    strategy:
@@ -58,6 +60,7 @@ serialPipeline:
       container-name: "namex-solr-synonyms-api-sandbox"
       service-account: "sa-api@a083gt-integration.iam.gserviceaccount.com"
       cloudsql-instances: "a083gt-integration:northamerica-northeast1:namex-db-integration"
+      container-concurrency: "8"
  - targetId: a083gt-prod
    profiles: [prod]
    strategy:
@@ -71,3 +74,4 @@ serialPipeline:
       container-name: "namex-solr-synonyms-api-prod"
       service-account: "sa-api@a083gt-prod.iam.gserviceaccount.com"
       cloudsql-instances: "a083gt-prod:northamerica-northeast1:namex-db-prod"
+      container-concurrency: "8"

solr-synonyms-api/devops/vaults.gcp.env

@@ -1,7 +1,7 @@
 SOLR_SYNONYMS_API_FLASK_SECRET_KEY="op://namex/$APP_ENV/solr-synonyms-api/SOLR_SYNONYMS_API_FLASK_SECRET_KEY"
-SOLR_SYNONYMS_API_DATABASE_USERNAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_USERNAME"
-SOLR_SYNONYMS_API_DATABASE_PASSWORD="op://database/$APP_ENV/namex-db-gcp/DATABASE_PASSWORD"
-SOLR_SYNONYMS_API_DATABASE_UNIX_SOCKET="op://database/$APP_ENV/namex-db-gcp/DATABASE_UNIX_SOCKET"
-SOLR_SYNONYMS_API_DATABASE_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_NAME"
-SOLR_SYNONYMS_API_DATABASE_PORT="op://database/$APP_ENV/namex-db-gcp/DATABASE_PORT"
+DATABASE_USERNAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_USERNAME"
+DATABASE_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_NAME"
+DATABASE_INSTANCE_CONNECTION_NAME="op://database/$APP_ENV/namex-db-gcp/DATABASE_INSTANCE_CONNECTION_NAME"
+DATABASE_SCHEMA="op://database/$APP_ENV/namex-db-gcp/DATABASE_SCHEMA"
+DATABASE_OWNER="op://database/$APP_ENV/namex-db-gcp/DATABASE_OWNER"
 VPC_CONNECTOR="op://CD/$APP_ENV/namex-solr-syn-api/VPC_CONNECTOR"

solr-synonyms-api/gunicorn_config.py

@@ -1,10 +1,10 @@
-"""Gunicorn config."""
 import os
 
-workers = int(os.environ.get('GUNICORN_PROCESSES', '1'))
-threads = int(os.environ.get('GUNICORN_THREADS', '3'))
-# 0 for unlimited -- 8190 is the highest set value
-limit_request_line = int(os.environ.get('GUNICORN_MAX_REQUEST_LINE', '0'))
+workers = int(os.environ.get("GUNICORN_PROCESSES", "1"))  # pylint: disable=invalid-name
+threads = int(os.environ.get("GUNICORN_THREADS", "8"))  # pylint: disable=invalid-name
+timeout = int(os.environ.get("GUNICORN_TIMEOUT", "0"))  # pylint: disable=invalid-name
+limit_request_line = int(os.environ.get("GUNICORN_MAX_REQUEST_LINE", "0"))
 
-forwarded_allow_ips = '*'
-secure_scheme_headers = {'X-Forwarded-Proto': 'https'}
+
+forwarded_allow_ips = "*"
+secure_scheme_headers = {"X-Forwarded-Proto": "https"}