diff --git a/.github/workflows/pay-api-ci.yml b/.github/workflows/pay-api-ci.yml index e920abe2f..5b9fda8e6 100644 --- a/.github/workflows/pay-api-ci.yml +++ b/.github/workflows/pay-api-ci.yml @@ -50,7 +50,7 @@ jobs: env: FLASK_ENV: "testing" # Needs different database than POSTGRES otherwise dropping database doesn't work - DATABASE_TEST_URL: "postgresql+pg8000://postgres:postgres@localhost:5432/pay-test" + DATABASE_TEST_URL: "postgresql+psycopg://postgres:postgres@localhost:5432/pay-test" USE_TEST_KEYCLOAK_DOCKER: "YES" USE_DOCKER_MOCK: "YES" diff --git a/bcol-api/poetry.lock b/bcol-api/poetry.lock index fb10cc4ae..017f629f2 100644 --- a/bcol-api/poetry.lock +++ b/bcol-api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "aniso8601" @@ -1697,36 +1697,36 @@ files = [ [[package]] name = "urllib3" -version = "2.5.0" +version = "2.6.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, - {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, + {file = "urllib3-2.6.0-py3-none-any.whl", hash = "sha256:c90f7a39f716c572c4e3e58509581ebd83f9b59cced005b7db7ad2d22b0db99f"}, + {file = "urllib3-2.6.0.tar.gz", hash = "sha256:cb9bcef5a4b345d5da5d145dc3e30834f58e8018828cbc724d30b4cb7d4d49f1"}, ] [package.extras] -brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "werkzeug" -version = "3.0.6" +version = "3.1.4" description = "The comprehensive WSGI web application library." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "werkzeug-3.0.6-py3-none-any.whl", hash = "sha256:1bc0c2310d2fbb07b1dd1105eba2f7af72f322e1e455f2f93c993bee8c8a5f17"}, - {file = "werkzeug-3.0.6.tar.gz", hash = "sha256:a8dd59d4de28ca70471a34cba79bed5f7ef2e036a76b3ab0835474246eb41f8d"}, + {file = "werkzeug-3.1.4-py3-none-any.whl", hash = "sha256:2ad50fb9ed09cc3af22c54698351027ace879a0b60a3b5edf5730b2f7d876905"}, + {file = "werkzeug-3.1.4.tar.gz", hash = "sha256:cd3cd98b1b92dc3b7b3995038826c68097dcb16f9baa63abe35f20eafeb9fe5e"}, ] [package.dependencies] -MarkupSafe = ">=2.1.1" +markupsafe = ">=2.1.1" [package.extras] watchdog = ["watchdog (>=2.3)"] @@ -1762,4 +1762,4 @@ xmlsec = ["xmlsec (>=0.6.1)"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "d24b48a77c98d15c19658640719ff8ea46c95684d9f4bee38023bf2de31b7919" +content-hash = "d4d60c2271785e8c1213444aded3c7d6f42a233ebcf87155feacff1542cd2db9" diff --git a/bcol-api/pyproject.toml b/bcol-api/pyproject.toml index 94ed56827..30e979ca2 100644 --- a/bcol-api/pyproject.toml +++ b/bcol-api/pyproject.toml @@ -20,7 +20,7 @@ requests = "^2.32.4" zeep = "^4.2.1" python-ldap = "^3.4.5" attrs = "^23.2.0" -werkzeug = "^3.0.1" +werkzeug = "^3.1.4" jaeger-client = "^4.8.0" pycountry = "^23.12.11" itsdangerous = "^2.1.2" diff --git a/jobs/ftp-poller/poetry.lock b/jobs/ftp-poller/poetry.lock index 0a3547ffa..afcd8a00a 100644 --- a/jobs/ftp-poller/poetry.lock +++ b/jobs/ftp-poller/poetry.lock @@ -171,18 +171,6 @@ files = [ [package.extras] dev = ["black", "coverage", "isort", "pre-commit", "pyenchant", "pylint"] -[[package]] -name = "asn1crypto" -version = "1.5.1" -description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, - {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, -] - [[package]] name = "astroid" version = "3.2.2" @@ -2143,9 +2131,9 @@ marshmallow = "3.21.1" marshmallow-sqlalchemy = "1.0.0" opentracing = "2.4.0" packaging = "24.0" -pg8000 = "^1.31.5" proto-plus = "1.23.0" protobuf = "4.25.8" +psycopg = "^3.3.1" psycopg2-binary = "2.9.9" pyasn1 = "0.5.1" pyasn1-modules = "0.3.0" @@ -2169,15 +2157,15 @@ structured-logging = {git = "https://github.com/bcgov/sbc-connect-common.git", r threadloop = "1.0.2" thrift = "0.16.0" tornado = "^6.5.1" -typing-extensions = "4.10.0" -urllib3 = "2.5.0" -werkzeug = "^3.0.3" +typing-extensions = "4.12.0" +urllib3 = "2.6.0" +werkzeug = "^3.1.4" [package.source] type = "git" url = "https://github.com/seeker25/sbc-pay.git" -reference = "update_deps_2" -resolved_reference = "d8bb4b6384ff2847fe458f99d863e0d642d917cf" +reference = "fix_deadlock" +resolved_reference = "1775ca2cc4391d7cfea73b1a5d28649e7ad72cc5" subdirectory = "pay-api" [[package]] @@ -2195,22 +2183,6 @@ files = [ [package.dependencies] flake8 = ">=5.0.0" -[[package]] -name = "pg8000" -version = "1.31.5" -description = "PostgreSQL interface library" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "pg8000-1.31.5-py3-none-any.whl", hash = "sha256:0af2c1926b153307639868d2ee5cef6cd3a7d07448e12736989b10e1d491e201"}, - {file = "pg8000-1.31.5.tar.gz", hash = "sha256:46ebb03be52b7a77c03c725c79da2ca281d6e8f59577ca66b17c9009618cae78"}, -] - -[package.dependencies] -python-dateutil = ">=2.8.2" -scramp = ">=1.4.5" - [[package]] name = "platformdirs" version = "4.2.2" @@ -2391,6 +2363,30 @@ files = [ {file = "protobuf-4.25.8.tar.gz", hash = "sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd"}, ] +[[package]] +name = "psycopg" +version = "3.3.1" +description = "PostgreSQL database adapter for Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "psycopg-3.3.1-py3-none-any.whl", hash = "sha256:e44d8eae209752efe46318f36dd0fdf5863e928009338d736843bb1084f6435c"}, + {file = "psycopg-3.3.1.tar.gz", hash = "sha256:ccfa30b75874eef809c0fbbb176554a2640cc1735a612accc2e2396a92442fc6"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.6", markers = "python_version < \"3.13\""} +tzdata = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +binary = ["psycopg-binary (==3.3.1) ; implementation_name != \"pypy\""] +c = ["psycopg-c (==3.3.1) ; implementation_name != \"pypy\""] +dev = ["ast-comments (>=1.1.2)", "black (>=24.1.0)", "codespell (>=2.2)", "cython-lint (>=0.16)", "dnspython (>=2.1)", "flake8 (>=4.0)", "isort-psycopg", "isort[colors] (>=6.0)", "mypy (>=1.19.0)", "pre-commit (>=4.0.1)", "types-setuptools (>=57.4)", "types-shapely (>=2.0)", "wheel (>=0.37)"] +docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] +pool = ["psycopg-pool"] +test = ["anyio (>=4.0)", "mypy (>=1.19.0) ; implementation_name != \"pypy\"", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] + [[package]] name = "psycopg2-binary" version = "2.9.9" @@ -2936,21 +2932,6 @@ reference = "HEAD" resolved_reference = "d640dc75ea51add2d611a30259d15d93d8654381" subdirectory = "python" -[[package]] -name = "scramp" -version = "1.4.5" -description = "An implementation of the SCRAM protocol." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "scramp-1.4.5-py3-none-any.whl", hash = "sha256:50e37c464fc67f37994e35bee4151e3d8f9320e9c204fca83a5d313c121bbbe7"}, - {file = "scramp-1.4.5.tar.gz", hash = "sha256:be3fbe774ca577a7a658117dca014e5d254d158cecae3dd60332dfe33ce6d78e"}, -] - -[package.dependencies] -asn1crypto = ">=1.5.1" - [[package]] name = "semver" version = "3.0.2" @@ -3281,48 +3262,61 @@ files = [ [[package]] name = "typing-extensions" -version = "4.10.0" +version = "4.12.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, - {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, + {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, + {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, +] + +[[package]] +name = "tzdata" +version = "2025.2" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +groups = ["main"] +markers = "sys_platform == \"win32\"" +files = [ + {file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"}, + {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, ] [[package]] name = "urllib3" -version = "2.5.0" +version = "2.6.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, - {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, + {file = "urllib3-2.6.0-py3-none-any.whl", hash = "sha256:c90f7a39f716c572c4e3e58509581ebd83f9b59cced005b7db7ad2d22b0db99f"}, + {file = "urllib3-2.6.0.tar.gz", hash = "sha256:cb9bcef5a4b345d5da5d145dc3e30834f58e8018828cbc724d30b4cb7d4d49f1"}, ] [package.extras] -brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "werkzeug" -version = "3.1.3" +version = "3.1.4" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e"}, - {file = "werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746"}, + {file = "werkzeug-3.1.4-py3-none-any.whl", hash = "sha256:2ad50fb9ed09cc3af22c54698351027ace879a0b60a3b5edf5730b2f7d876905"}, + {file = "werkzeug-3.1.4.tar.gz", hash = "sha256:cd3cd98b1b92dc3b7b3995038826c68097dcb16f9baa63abe35f20eafeb9fe5e"}, ] [package.dependencies] -MarkupSafe = ">=2.1.1" +markupsafe = ">=2.1.1" [package.extras] watchdog = ["watchdog (>=2.3)"] @@ -3442,4 +3436,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "021e30530143a0da2f87b38795530735cddf579d2d6eeada867761c69b58f75f" +content-hash = "2ff160ecc4e7086d71d8b2edd18cbb029731e927a3ed5392fa4bc4e57ece5174" diff --git a/jobs/ftp-poller/pyproject.toml b/jobs/ftp-poller/pyproject.toml index 1ac8d1460..b0a9a394b 100644 --- a/jobs/ftp-poller/pyproject.toml +++ b/jobs/ftp-poller/pyproject.toml @@ -21,7 +21,7 @@ itsdangerous = "^2.1.2" jinja2 = "^3.1.3" launchdarkly-server-sdk = "^8.2.1" sbc-common-components = {git = "https://github.com/bcgov/sbc-common-components.git", subdirectory = "python"} -pay-api = { git = "https://github.com/seeker25/sbc-pay.git", branch = "update_deps_2", subdirectory = "pay-api" } +pay-api = { git = "https://github.com/seeker25/sbc-pay.git", branch = "fix_deadlock", subdirectory = "pay-api" } wheel = "^0.43.0" diff --git a/jobs/notebook-report/poetry.lock b/jobs/notebook-report/poetry.lock index edc33fc6f..dd7e7a1b3 100644 --- a/jobs/notebook-report/poetry.lock +++ b/jobs/notebook-report/poetry.lock @@ -4288,21 +4288,21 @@ files = [ [[package]] name = "urllib3" -version = "2.5.0" +version = "2.6.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, - {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, + {file = "urllib3-2.6.0-py3-none-any.whl", hash = "sha256:c90f7a39f716c572c4e3e58509581ebd83f9b59cced005b7db7ad2d22b0db99f"}, + {file = "urllib3-2.6.0.tar.gz", hash = "sha256:cb9bcef5a4b345d5da5d145dc3e30834f58e8018828cbc724d30b4cb7d4d49f1"}, ] [package.extras] -brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "wcwidth" @@ -4410,4 +4410,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.13" -content-hash = "c92c9d44d8fbaeb83fbb3ceb555e26c86e7d7088cfea4d9c00b7fb255411d0eb" +content-hash = "21692d667bde6a636b5a3ba73e17adcda326c756355e1758717fa4aadd23f2c7" diff --git a/jobs/notebook-report/pyproject.toml b/jobs/notebook-report/pyproject.toml index b73393982..d0b5bd7cc 100644 --- a/jobs/notebook-report/pyproject.toml +++ b/jobs/notebook-report/pyproject.toml @@ -36,7 +36,7 @@ requests = "^2.32.4" marshmallow = "^3.23.1" werkzeug = "^3.1.4" certifi = "^2024.8.30" -urllib3 = "^2.5.0" +urllib3 = "^2.6.0" idna = "^3.10" pg8000 = "^1.31.2" diff --git a/jobs/payment-jobs/config.py b/jobs/payment-jobs/config.py index 5f93e806b..42da2e439 100644 --- a/jobs/payment-jobs/config.py +++ b/jobs/payment-jobs/config.py @@ -66,10 +66,10 @@ class _Config: # pylint: disable=too-few-public-methods DB_PORT = os.getenv("DATABASE_PORT", "5432") if DB_UNIX_SOCKET := os.getenv("DATABASE_UNIX_SOCKET", None): SQLALCHEMY_DATABASE_URI = ( - f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?unix_sock={DB_UNIX_SOCKET}/.s.PGSQL.5432" + f"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?host={DB_UNIX_SOCKET}&port={DB_PORT}" ) else: - SQLALCHEMY_DATABASE_URI = f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}" + SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}" SQLALCHEMY_ECHO = False # Data Warehouse Settings @@ -225,7 +225,7 @@ class TestConfig(_Config): # pylint: disable=too-few-public-methods DB_PORT = os.getenv("DATABASE_TEST_PORT", "5432") SQLALCHEMY_DATABASE_URI = os.getenv( "DATABASE_TEST_URL", - f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}", + f"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}", ) SERVER_NAME = "localhost:5001" diff --git a/jobs/payment-jobs/poetry.lock b/jobs/payment-jobs/poetry.lock index f6a5ec795..5e0b12e77 100644 --- a/jobs/payment-jobs/poetry.lock +++ b/jobs/payment-jobs/poetry.lock @@ -1970,9 +1970,9 @@ marshmallow = "3.21.1" marshmallow-sqlalchemy = "1.0.0" opentracing = "2.4.0" packaging = "24.0" -pg8000 = "^1.31.5" proto-plus = "1.23.0" protobuf = "4.25.8" +psycopg = "^3.3.1" psycopg2-binary = "2.9.9" pyasn1 = "0.5.1" pyasn1-modules = "0.3.0" @@ -1996,15 +1996,15 @@ structured-logging = {git = "https://github.com/bcgov/sbc-connect-common.git", r threadloop = "1.0.2" thrift = "0.16.0" tornado = "^6.5.1" -typing-extensions = "4.10.0" -urllib3 = "2.5.0" -werkzeug = "^3.0.3" +typing-extensions = "4.12.0" +urllib3 = "2.6.0" +werkzeug = "^3.1.4" [package.source] type = "git" -url = "https://github.com/Jxio/sbc-pay.git" -reference = "31164" -resolved_reference = "2429bda09297896585f264430cfdb42b68d14039" +url = "https://github.com/seeker25/sbc-pay.git" +reference = "fix_deadlock" +resolved_reference = "1775ca2cc4391d7cfea73b1a5d28649e7ad72cc5" subdirectory = "pay-api" [[package]] @@ -2186,6 +2186,30 @@ files = [ {file = "protobuf-4.25.8.tar.gz", hash = "sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd"}, ] +[[package]] +name = "psycopg" +version = "3.3.1" +description = "PostgreSQL database adapter for Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "psycopg-3.3.1-py3-none-any.whl", hash = "sha256:e44d8eae209752efe46318f36dd0fdf5863e928009338d736843bb1084f6435c"}, + {file = "psycopg-3.3.1.tar.gz", hash = "sha256:ccfa30b75874eef809c0fbbb176554a2640cc1735a612accc2e2396a92442fc6"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.6", markers = "python_version < \"3.13\""} +tzdata = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +binary = ["psycopg-binary (==3.3.1) ; implementation_name != \"pypy\""] +c = ["psycopg-c (==3.3.1) ; implementation_name != \"pypy\""] +dev = ["ast-comments (>=1.1.2)", "black (>=24.1.0)", "codespell (>=2.2)", "cython-lint (>=0.16)", "dnspython (>=2.1)", "flake8 (>=4.0)", "isort-psycopg", "isort[colors] (>=6.0)", "mypy (>=1.19.0)", "pre-commit (>=4.0.1)", "types-setuptools (>=57.4)", "types-shapely (>=2.0)", "wheel (>=0.37)"] +docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] +pool = ["psycopg-pool"] +test = ["anyio (>=4.0)", "mypy (>=1.19.0) ; implementation_name != \"pypy\"", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] + [[package]] name = "psycopg2-binary" version = "2.9.9" @@ -2664,14 +2688,14 @@ subdirectory = "python" [[package]] name = "scramp" -version = "1.4.5" +version = "1.4.6" description = "An implementation of the SCRAM protocol." optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "scramp-1.4.5-py3-none-any.whl", hash = "sha256:50e37c464fc67f37994e35bee4151e3d8f9320e9c204fca83a5d313c121bbbe7"}, - {file = "scramp-1.4.5.tar.gz", hash = "sha256:be3fbe774ca577a7a658117dca014e5d254d158cecae3dd60332dfe33ce6d78e"}, + {file = "scramp-1.4.6-py3-none-any.whl", hash = "sha256:a0cf9d2b4624b69bac5432dd69fecfc55a542384fe73c3a23ed9b138cda484e1"}, + {file = "scramp-1.4.6.tar.gz", hash = "sha256:fe055ebbebf4397b9cb323fcc4b299f219cd1b03fd673ca40c97db04ac7d107e"}, ] [package.dependencies] @@ -2983,48 +3007,61 @@ files = [ [[package]] name = "typing-extensions" -version = "4.10.0" +version = "4.12.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, - {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, + {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, + {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, +] + +[[package]] +name = "tzdata" +version = "2025.2" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +groups = ["main"] +markers = "sys_platform == \"win32\"" +files = [ + {file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"}, + {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, ] [[package]] name = "urllib3" -version = "2.5.0" +version = "2.6.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, - {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, + {file = "urllib3-2.6.0-py3-none-any.whl", hash = "sha256:c90f7a39f716c572c4e3e58509581ebd83f9b59cced005b7db7ad2d22b0db99f"}, + {file = "urllib3-2.6.0.tar.gz", hash = "sha256:cb9bcef5a4b345d5da5d145dc3e30834f58e8018828cbc724d30b4cb7d4d49f1"}, ] [package.extras] -brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "werkzeug" -version = "3.1.3" +version = "3.1.4" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e"}, - {file = "werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746"}, + {file = "werkzeug-3.1.4-py3-none-any.whl", hash = "sha256:2ad50fb9ed09cc3af22c54698351027ace879a0b60a3b5edf5730b2f7d876905"}, + {file = "werkzeug-3.1.4.tar.gz", hash = "sha256:cd3cd98b1b92dc3b7b3995038826c68097dcb16f9baa63abe35f20eafeb9fe5e"}, ] [package.dependencies] -MarkupSafe = ">=2.1.1" +markupsafe = ">=2.1.1" [package.extras] watchdog = ["watchdog (>=2.3)"] @@ -3129,4 +3166,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "cc5407c1eb615cd07fb2efd5f50ee6875903e4ba0db695cb5cb0a9fa051627c6" +content-hash = "7ed339ef841acdebc884bab279de202d5d021eef3871ef2bf6f4bf273eb84b6a" diff --git a/jobs/payment-jobs/pyproject.toml b/jobs/payment-jobs/pyproject.toml index 0294a2a61..6d77b5266 100644 --- a/jobs/payment-jobs/pyproject.toml +++ b/jobs/payment-jobs/pyproject.toml @@ -8,7 +8,7 @@ package-mode = false [tool.poetry.dependencies] python = "^3.12" -pay-api = { git = "https://github.com/Jxio/sbc-pay.git", branch = "31164", subdirectory = "pay-api" } +pay-api = { git = "https://github.com/seeker25/sbc-pay.git", branch = "fix_deadlock", subdirectory = "pay-api" } flask = "^3.0.2" flask-sqlalchemy = "^3.1.1" sqlalchemy = "^2.0.28" @@ -26,9 +26,9 @@ itsdangerous = "^2.1.2" dataclass-wizard = "^0.22.3" launchdarkly-server-sdk = "^8.2.1" more-itertools = "^10.2.0" -pg8000 = "^1.30.5" cloud-sql-python-connector = "^1.13.0" ruff = "^0.14.1" +pg8000 = "^1.31.5" [tool.poetry.group.dev.dependencies] diff --git a/jobs/payment-jobs/tasks/common/cgi_ap.py b/jobs/payment-jobs/tasks/common/cgi_ap.py index 9c24d8722..48314912b 100644 --- a/jobs/payment-jobs/tasks/common/cgi_ap.py +++ b/jobs/payment-jobs/tasks/common/cgi_ap.py @@ -20,7 +20,7 @@ from pay_api.models.eft_refund import EFTRefund from pay_api.utils.enums import DisbursementMethod -from pay_api.utils.util import get_fiscal_year +from pay_api.utils.util import get_fiscal_year, get_nearest_business_day from tasks.common.dataclasses import APFlow, APHeader, APLine, APSupplier from .cgi_ejv import CgiEjv @@ -53,7 +53,7 @@ def get_ap_header(cls, ap_header: APHeader): invoice_type = "ST" remit_code = f"{current_app.config.get('CGI_AP_REMITTANCE_CODE'):<4}" currency = "CAD" - effective_date = cls._get_date(datetime.now(tz=UTC)) + effective_date = cls._get_date(get_nearest_business_day(datetime.now(tz=UTC))) invoice_date = cls._get_date(ap_header.invoice_date) oracle_invoice_batch_name = cls._get_oracle_invoice_batch_name(ap_header.ap_flow, ap_header.invoice_number) ap_flow_to_disbursement_method = { @@ -82,7 +82,7 @@ def get_ap_invoice_line(cls, ap_line: APLine): commit_line_number = f"{cls.EMPTY:<4}" # Pad Zeros to four digits. EG. 0001 line_number = f"{ap_line.line_number:04}" - effective_date = cls._get_date(datetime.now(tz=UTC)) + effective_date = cls._get_date(get_nearest_business_day(datetime.now(tz=UTC))) line_code = cls._get_line_code(ap_line) supplier_number = cls._supplier_number(ap_line.ap_flow, ap_line.ap_supplier.supplier_number) dist_vendor = cls._dist_vendor(ap_line.ap_flow, ap_line.ap_supplier.supplier_number) diff --git a/jobs/payment-jobs/tasks/routing_slip_task.py b/jobs/payment-jobs/tasks/routing_slip_task.py index 1d3235647..500109ccd 100644 --- a/jobs/payment-jobs/tasks/routing_slip_task.py +++ b/jobs/payment-jobs/tasks/routing_slip_task.py @@ -328,9 +328,7 @@ def adjust_routing_slips(cls): except ValueError as e: routing_slip.cas_mismatch = True routing_slip.save() - current_app.logger.error( - f"Skipping adjustment for routing slip {routing_slip.number}: {str(e)}" - ) + current_app.logger.error(f"Skipping adjustment for routing slip {routing_slip.number}: {str(e)}") continue except Exception as e: # NOQA # pylint: disable=broad-except @@ -343,23 +341,21 @@ def adjust_routing_slips(cls): @classmethod def _has_pending_invoices( - cls, - routing_slip: RoutingSlipModel, - child_routing_slips: list[RoutingSlipModel] + cls, routing_slip: RoutingSlipModel, child_routing_slips: list[RoutingSlipModel] ) -> tuple[bool, int]: """Check if routing slip or its children have pending invoices.""" all_routing_slips = [routing_slip] + child_routing_slips all_routing_slip_numbers = [rs.number for rs in all_routing_slips] - + pending_invoice_count = ( db.session.query(InvoiceModel) .filter( InvoiceModel.routing_slip.in_(all_routing_slip_numbers), - InvoiceModel.invoice_status_code.in_([InvoiceStatus.APPROVED.value, InvoiceStatus.CREATED.value]) + InvoiceModel.invoice_status_code.in_([InvoiceStatus.APPROVED.value, InvoiceStatus.CREATED.value]), ) .count() ) - + return (pending_invoice_count > 0, pending_invoice_count) @classmethod @@ -386,7 +382,7 @@ def _get_applied_invoices_amount(cls, routing_slips: list[RoutingSlipModel]) -> db.session.query(func.sum(InvoiceModel.paid - func.coalesce(InvoiceModel.refund, 0))) .filter( InvoiceModel.routing_slip.in_(routing_slip_numbers), - InvoiceModel.invoice_status_code == InvoiceStatus.PAID.value + InvoiceModel.invoice_status_code == InvoiceStatus.PAID.value, ) .scalar() ) @@ -394,17 +390,11 @@ def _get_applied_invoices_amount(cls, routing_slips: list[RoutingSlipModel]) -> @classmethod def _check_data_consistency( - cls, - routing_slip: RoutingSlipModel, - sbc_pay_applied_amount: Decimal, - cfs_receipt_details: list[dict] + cls, routing_slip: RoutingSlipModel, sbc_pay_applied_amount: Decimal, cfs_receipt_details: list[dict] ) -> None: """Check data consistency between SBC-PAY and CFS.""" sbc_pay_has_invoices = sbc_pay_applied_amount > 0 - cfs_has_invoices = any( - receipt['has_applied_invoices'] - for receipt in cfs_receipt_details - ) + cfs_has_invoices = any(receipt["has_applied_invoices"] for receipt in cfs_receipt_details) if sbc_pay_has_invoices != cfs_has_invoices: error_msg = ( @@ -426,10 +416,7 @@ def _check_data_consistency( @classmethod def _validate_and_calculate_adjustment_amount( - cls, - routing_slip: RoutingSlipModel, - cfs_account: CfsAccountModel, - child_routing_slips: list[RoutingSlipModel] + cls, routing_slip: RoutingSlipModel, cfs_account: CfsAccountModel, child_routing_slips: list[RoutingSlipModel] ) -> None: """Validate adjustment amount for routing slip.""" all_routing_slips = [routing_slip] + child_routing_slips @@ -442,17 +429,13 @@ def _validate_and_calculate_adjustment_amount( for rs in all_routing_slips: receipt_number = rs.generate_cas_receipt_number() receipt_data = CFSService.get_receipt(cfs_account, receipt_number) - unapplied_amount = Decimal(str(receipt_data.get('unapplied_amount', 0))) - has_applied_invoices = len(receipt_data.get('invoices', [])) > 0 - receipt_details.append({'has_applied_invoices': has_applied_invoices}) + unapplied_amount = Decimal(str(receipt_data.get("unapplied_amount", 0))) + has_applied_invoices = len(receipt_data.get("invoices", [])) > 0 + receipt_details.append({"has_applied_invoices": has_applied_invoices}) cfs_unapplied_total += unapplied_amount # may raise ValueError - cls._check_data_consistency( - routing_slip, - sbc_pay_applied, - receipt_details - ) + cls._check_data_consistency(routing_slip, sbc_pay_applied, receipt_details) all_rs_total = sum(rs.total for rs in all_routing_slips) expected_adjustment = all_rs_total - sbc_pay_applied diff --git a/jobs/payment-jobs/tests/docker/docker-compose.yml b/jobs/payment-jobs/tests/docker/docker-compose.yml index 650ea646d..b61384cd6 100644 --- a/jobs/payment-jobs/tests/docker/docker-compose.yml +++ b/jobs/payment-jobs/tests/docker/docker-compose.yml @@ -54,7 +54,7 @@ services: image: stoplight/prism:3.3.0 command: > mock -p 4010 --host 0.0.0.0 - https://raw.githubusercontent.com/bcgov/sbc-pay/main/docs/docs/PayBC%20Mocking/paybc-1.0.0.yaml + https://raw.githubusercontent.com/bcgov/sbc-pay/d8091408d5cd591dd3e0c83ae8c5738de90699de/docs/docs/PayBC%20Mocking/paybc-1.0.0.yaml auth: image: stoplight/prism:3.3.0 command: > diff --git a/jobs/payment-jobs/tests/jobs/conftest.py b/jobs/payment-jobs/tests/jobs/conftest.py index 02b5a12c9..43f22ed94 100644 --- a/jobs/payment-jobs/tests/jobs/conftest.py +++ b/jobs/payment-jobs/tests/jobs/conftest.py @@ -15,7 +15,6 @@ """Common setup and fixtures for the py-test suite used by this service.""" import os -import time from unittest.mock import Mock import pytest @@ -137,14 +136,13 @@ def auto(docker_services, app): """Spin up docker instances.""" if app.config["USE_DOCKER_MOCK"]: docker_services.start("keycloak") - docker_services.wait_for_service("keycloak", 8081) docker_services.start("bcol") docker_services.start("auth") docker_services.start("paybc") docker_services.start("reports") - docker_services.start("proxy") docker_services.start("sftp") - time.sleep(2) + docker_services.start("proxy") + docker_services.wait_for_service("keycloak", 8081) @pytest.fixture(scope="session") diff --git a/jobs/payment-jobs/tests/jobs/test_ap_task.py b/jobs/payment-jobs/tests/jobs/test_ap_task.py index 3d7e709e3..db57a3cee 100644 --- a/jobs/payment-jobs/tests/jobs/test_ap_task.py +++ b/jobs/payment-jobs/tests/jobs/test_ap_task.py @@ -17,6 +17,8 @@ Test-Suite to ensure that the AP Refund Job is working as expected. """ +import re +from datetime import UTC, datetime from unittest.mock import MagicMock, patch from pay_api.models import FeeSchedule as FeeScheduleModel @@ -31,6 +33,8 @@ RoutingSlipStatus, ) from tasks.ap_task import ApTask +from tasks.common.cgi_ap import CgiAP +from tasks.common.dataclasses import APFlow, APHeader, APLine, APSupplier from .factory import ( factory_create_eft_account, @@ -223,3 +227,86 @@ def mock_upload_error(*args, **kwargs): routing_slip = RoutingSlip.find_by_number(rs_1) assert routing_slip.status == RoutingSlipStatus.REFUND_AUTHORIZED.value + + +def test_get_ap_header_and_line_weekend_and_holiday_date_adjustment(session, monkeypatch): + """Test that get_ap_header and get_ap_invoice_line use next business day. + + Tests scenario where Saturday and Sunday are weekends, and Monday is a + holiday, so the next business day is Tuesday. + + Steps: + 1) Mock datetime.now() to return Saturday (June 29, 2024) + 2) Call get_ap_header and get_ap_invoice_line + 3) Assert that the effective_date is Tuesday (July 2, 2024), + skipping Sunday and Monday holiday (Canada Day) + """ + # June 29, 2024 is a Saturday, July 1, 2024 (Monday) is Canada Day holiday + # July 2, 2024 (Tuesday) is the next business day + saturday_date = datetime(2024, 6, 29, 12, 0, 0, tzinfo=UTC) + expected_date = datetime(2024, 7, 2).date() + + with patch("tasks.common.cgi_ap.datetime") as mock_dt, patch("pay_api.utils.util.datetime") as mock_util_dt: + + def datetime_side_effect(*args, **kw): + if args or kw: + return datetime(*args, **kw) + return datetime + + mock_dt.side_effect = datetime_side_effect + mock_dt.now = lambda *_args, **_kwargs: saturday_date + mock_dt.UTC = UTC + + mock_util_dt.side_effect = datetime_side_effect + mock_util_dt.now = lambda *_args, **_kwargs: saturday_date + mock_util_dt.UTC = UTC + + ap_header = APHeader( + ap_flow=APFlow.NON_GOV_TO_EFT, + total=100.00, + invoice_number="TEST123", + invoice_date=datetime.now(tz=UTC).date(), + ap_supplier=APSupplier(), + ) + + ap_line = APLine( + ap_flow=APFlow.NON_GOV_TO_EFT, + total=100.00, + invoice_number="TEST123", + line_number=1, + is_reversal=False, + distribution="12345678901234567890", + ap_supplier=APSupplier(), + ) + + header_result = CgiAP.get_ap_header(ap_header) + line_result = CgiAP.get_ap_invoice_line(ap_line) + + header_date_match = re.search(r"CAD(\d{8})", header_result) + assert header_date_match, "Could not find effective_date in header" + header_effective_date_str = header_date_match.group(1) + + # Extract date from line: search for 8-digit date pattern (YYYYMMDD) + # The date appears after the distribution code and padding + # Look for date pattern starting with expected year (2024) + expected_date_str = expected_date.strftime("%Y%m%d") + line_date_match = re.search(re.escape(expected_date_str), line_result) + assert line_date_match, ( + f"Could not find expected date {expected_date_str} in line. " + f"Line preview: {line_result[:500]}" + ) + line_effective_date_str = expected_date_str + + header_effective_date = datetime.strptime(header_effective_date_str, "%Y%m%d").date() + line_effective_date = datetime.strptime(line_effective_date_str, "%Y%m%d").date() + + assert header_effective_date == expected_date, ( + f"Header effective_date {header_effective_date} " + f"should be {expected_date} " + f"(Tuesday, skipping weekend and holiday)" + ) + assert line_effective_date == expected_date, ( + f"Line effective_date {line_effective_date} " + f"should be {expected_date} " + f"(Tuesday, skipping weekend and holiday)" + ) diff --git a/jobs/payment-jobs/tests/jobs/test_routing_slip_task.py b/jobs/payment-jobs/tests/jobs/test_routing_slip_task.py index 8828a73b8..2c89bcbf9 100644 --- a/jobs/payment-jobs/tests/jobs/test_routing_slip_task.py +++ b/jobs/payment-jobs/tests/jobs/test_routing_slip_task.py @@ -340,20 +340,22 @@ def test_receipt_adjustments(session, rs_status): parent_rs.save() # Test exception path first. - with patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, \ - patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust: + with ( + patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, + patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust, + ): # Return different values for parent and child receipts mock_get_receipt.side_effect = [ { - 'unapplied_amount': 0.0, # parent - 'receipt_amount': 20.0, - 'invoices': [] + "unapplied_amount": 0.0, # parent + "receipt_amount": 20.0, + "invoices": [], }, { - 'unapplied_amount': 0.0, # child - 'receipt_amount': 10.0, - 'invoices': [] - } + "unapplied_amount": 0.0, # child + "receipt_amount": 10.0, + "invoices": [], + }, ] mock_adjust.side_effect = Exception("ERROR!") RoutingSlipTask.adjust_routing_slips() @@ -365,20 +367,22 @@ def test_receipt_adjustments(session, rs_status): parent_rs.cas_mismatch = False parent_rs.save() - with patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, \ - patch("pay_api.services.CFSService.adjust_receipt_to_zero"): + with ( + patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, + patch("pay_api.services.CFSService.adjust_receipt_to_zero"), + ): # Return different values for parent and child receipts mock_get_receipt.side_effect = [ { - 'unapplied_amount': 20.0, # parent - 'receipt_amount': 20.0, - 'invoices': [] + "unapplied_amount": 20.0, # parent + "receipt_amount": 20.0, + "invoices": [], }, { - 'unapplied_amount': 10.0, # child - 'receipt_amount': 10.0, - 'invoices': [] - } + "unapplied_amount": 10.0, # child + "receipt_amount": 10.0, + "invoices": [], + }, ] RoutingSlipTask.adjust_routing_slips() @@ -396,22 +400,19 @@ def test_receipt_adjustments_amount_mismatch(session): total=100, remaining_amount=50, ) - + rs = RoutingSlipModel.find_by_number(rs_number) rs.status = RoutingSlipStatus.REFUND_AUTHORIZED.value rs.cas_mismatch = False rs.save() # doesn't match remaining_amount - with patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, \ - patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust: - - mock_get_receipt.return_value = { - 'unapplied_amount': 30.0, - 'receipt_amount': 100.0, - 'invoices': [] - } - + with ( + patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, + patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust, + ): + mock_get_receipt.return_value = {"unapplied_amount": 30.0, "receipt_amount": 100.0, "invoices": []} + RoutingSlipTask.adjust_routing_slips() rs = RoutingSlipModel.find_by_number(rs_number) @@ -436,15 +437,12 @@ def test_receipt_adjustments_data_mismatch(session): rs.save() # data mismatch - with patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, \ - patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust: - - mock_get_receipt.return_value = { - 'unapplied_amount': 85.0, - 'receipt_amount': 100.0, - 'invoices': [] - } - + with ( + patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, + patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust, + ): + mock_get_receipt.return_value = {"unapplied_amount": 85.0, "receipt_amount": 100.0, "invoices": []} + RoutingSlipTask.adjust_routing_slips() rs = RoutingSlipModel.find_by_number(rs_number) @@ -462,20 +460,21 @@ def test_receipt_adjustments_skip_cas_mismatch(session): total=100, remaining_amount=50, ) - + rs = RoutingSlipModel.find_by_number(rs_number) rs.status = RoutingSlipStatus.REFUND_AUTHORIZED.value rs.cas_mismatch = True rs.save() - with patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, \ - patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust: - + with ( + patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, + patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust, + ): RoutingSlipTask.adjust_routing_slips() assert not mock_get_receipt.called assert not mock_adjust.called - + rs = RoutingSlipModel.find_by_number(rs_number) assert rs.remaining_amount == 50 assert rs.cas_mismatch is True @@ -496,15 +495,16 @@ def test_receipt_adjustments_cfs_has_invoices_sbc_pay_doesnt(session): rs.cas_mismatch = False rs.save() - with patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, \ - patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust: - + with ( + patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, + patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust, + ): mock_get_receipt.return_value = { - 'unapplied_amount': 100.0, - 'receipt_amount': 100.0, - 'invoices': [{'invoice_number': 'INV123', 'amount': 50.0}] + "unapplied_amount": 100.0, + "receipt_amount": 100.0, + "invoices": [{"invoice_number": "INV123", "amount": 50.0}], } - + RoutingSlipTask.adjust_routing_slips() rs = RoutingSlipModel.find_by_number(rs_number) @@ -546,27 +546,28 @@ def test_receipt_adjustments_with_multiple_invoices_consistent(session): rs.cas_mismatch = False rs.save() - with patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, \ - patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust: - + with ( + patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, + patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust, + ): # total: 100.33, applied: 33.22 (22.11 + 11.11), unapplied: 67.11 mock_get_receipt.return_value = { - 'unapplied_amount': 67.11, - 'receipt_amount': 100.33, - 'invoices': [ + "unapplied_amount": 67.11, + "receipt_amount": 100.33, + "invoices": [ { - 'invoice_number': 'REG01036828', - 'total': 22.11, - 'amount_applied': 22.11, - 'links': [{'rel': 'self', 'href': 'https://xxx/invs/REG01036828/'}] + "invoice_number": "REG01036828", + "total": 22.11, + "amount_applied": 22.11, + "links": [{"rel": "self", "href": "https://xxx/invs/REG01036828/"}], }, { - 'invoice_number': 'REG01036829', - 'total': 11.11, - 'amount_applied': 11.11, - 'links': [{'rel': 'self', 'href': 'https://xxx/invs/REG01036829/'}] - } - ] + "invoice_number": "REG01036829", + "total": 11.11, + "amount_applied": 11.11, + "links": [{"rel": "self", "href": "https://xxx/invs/REG01036829/"}], + }, + ], } RoutingSlipTask.adjust_routing_slips() @@ -585,9 +586,7 @@ def test_receipt_adjustments_skip_child_pending_invoices(session): parent_rs_number = "12353" child_pay_account = factory_routing_slip_account( - number=child_rs_number, - status=CfsAccountStatus.ACTIVE.value, - total=10 + number=child_rs_number, status=CfsAccountStatus.ACTIVE.value, total=10 ) factory_routing_slip_account( number=parent_rs_number, @@ -615,9 +614,10 @@ def test_receipt_adjustments_skip_child_pending_invoices(session): routing_slip=child_rs_number, ) - with patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, \ - patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust: - + with ( + patch("pay_api.services.CFSService.get_receipt") as mock_get_receipt, + patch("pay_api.services.CFSService.adjust_receipt_to_zero") as mock_adjust, + ): RoutingSlipTask.adjust_routing_slips() assert not mock_get_receipt.called diff --git a/pay-admin/admin/config.py b/pay-admin/admin/config.py index 99fee9746..0407a8908 100755 --- a/pay-admin/admin/config.py +++ b/pay-admin/admin/config.py @@ -79,10 +79,10 @@ class _Config: # pylint: disable=too-few-public-methods DB_PORT = _get_config("DATABASE_PORT", default="5432") if DB_UNIX_SOCKET := os.getenv("DATABASE_UNIX_SOCKET", None): SQLALCHEMY_DATABASE_URI = ( - f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?unix_sock={DB_UNIX_SOCKET}/.s.PGSQL.5432" + f"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?host={DB_UNIX_SOCKET}&port={DB_PORT}" ) else: - SQLALCHEMY_DATABASE_URI = f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}" + SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}" SQLALCHEMY_ECHO = _get_config("SQLALCHEMY_ECHO", default="False").lower() == "true" # Normal Keycloak parameters. @@ -124,7 +124,7 @@ class TestConfig(_Config): # pylint: disable=too-few-public-methods DB_NAME = _get_config("DATABASE_TEST_NAME", default="paytestdb") DB_HOST = _get_config("DATABASE_TEST_HOST", default="localhost") DB_PORT = _get_config("DATABASE_TEST_PORT", default="5432") - SQLALCHEMY_DATABASE_URI = f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}" + SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}" class ProdConfig(_Config): # pylint: disable=too-few-public-methods diff --git a/pay-admin/poetry.lock b/pay-admin/poetry.lock index 2e76457cf..f0af418fc 100644 --- a/pay-admin/poetry.lock +++ b/pay-admin/poetry.lock @@ -156,18 +156,6 @@ typing-extensions = ">=4" [package.extras] tz = ["backports.zoneinfo ; python_version < \"3.9\""] -[[package]] -name = "asn1crypto" -version = "1.5.1" -description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, - {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, -] - [[package]] name = "attrs" version = "23.2.0" @@ -1937,9 +1925,9 @@ marshmallow = "3.21.1" marshmallow-sqlalchemy = "1.0.0" opentracing = "2.4.0" packaging = "24.0" -pg8000 = "^1.31.5" proto-plus = "1.23.0" protobuf = "4.25.8" +psycopg = "^3.3.1" psycopg2-binary = "2.9.9" pyasn1 = "0.5.1" pyasn1-modules = "0.3.0" @@ -1963,33 +1951,17 @@ structured-logging = {git = "https://github.com/bcgov/sbc-connect-common.git", r threadloop = "1.0.2" thrift = "0.16.0" tornado = "^6.5.1" -typing-extensions = "4.10.0" -urllib3 = "2.5.0" -werkzeug = "^3.0.3" +typing-extensions = "4.12.0" +urllib3 = "2.6.0" +werkzeug = "^3.1.4" [package.source] type = "git" -url = "https://github.com/bcgov/sbc-pay.git" -reference = "main" -resolved_reference = "21af5226c9a1ed687f39eb4435af4f7b0236484e" +url = "https://github.com/seeker25/sbc-pay.git" +reference = "fix_deadlock" +resolved_reference = "1775ca2cc4391d7cfea73b1a5d28649e7ad72cc5" subdirectory = "pay-api" -[[package]] -name = "pg8000" -version = "1.31.5" -description = "PostgreSQL interface library" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "pg8000-1.31.5-py3-none-any.whl", hash = "sha256:0af2c1926b153307639868d2ee5cef6cd3a7d07448e12736989b10e1d491e201"}, - {file = "pg8000-1.31.5.tar.gz", hash = "sha256:46ebb03be52b7a77c03c725c79da2ca281d6e8f59577ca66b17c9009618cae78"}, -] - -[package.dependencies] -python-dateutil = ">=2.8.2" -scramp = ">=1.4.5" - [[package]] name = "pluggy" version = "1.5.0" @@ -2153,6 +2125,30 @@ files = [ {file = "protobuf-4.25.8.tar.gz", hash = "sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd"}, ] +[[package]] +name = "psycopg" +version = "3.3.1" +description = "PostgreSQL database adapter for Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "psycopg-3.3.1-py3-none-any.whl", hash = "sha256:e44d8eae209752efe46318f36dd0fdf5863e928009338d736843bb1084f6435c"}, + {file = "psycopg-3.3.1.tar.gz", hash = "sha256:ccfa30b75874eef809c0fbbb176554a2640cc1735a612accc2e2396a92442fc6"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.6", markers = "python_version < \"3.13\""} +tzdata = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +binary = ["psycopg-binary (==3.3.1) ; implementation_name != \"pypy\""] +c = ["psycopg-c (==3.3.1) ; implementation_name != \"pypy\""] +dev = ["ast-comments (>=1.1.2)", "black (>=24.1.0)", "codespell (>=2.2)", "cython-lint (>=0.16)", "dnspython (>=2.1)", "flake8 (>=4.0)", "isort-psycopg", "isort[colors] (>=6.0)", "mypy (>=1.19.0)", "pre-commit (>=4.0.1)", "types-setuptools (>=57.4)", "types-shapely (>=2.0)", "wheel (>=0.37)"] +docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] +pool = ["psycopg-pool"] +test = ["anyio (>=4.0)", "mypy (>=1.19.0) ; implementation_name != \"pypy\"", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] + [[package]] name = "psycopg2-binary" version = "2.9.9" @@ -2620,21 +2616,6 @@ reference = "HEAD" resolved_reference = "22978d810dc4e85c51c3129936686b0a17124e64" subdirectory = "python" -[[package]] -name = "scramp" -version = "1.4.5" -description = "An implementation of the SCRAM protocol." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "scramp-1.4.5-py3-none-any.whl", hash = "sha256:50e37c464fc67f37994e35bee4151e3d8f9320e9c204fca83a5d313c121bbbe7"}, - {file = "scramp-1.4.5.tar.gz", hash = "sha256:be3fbe774ca577a7a658117dca014e5d254d158cecae3dd60332dfe33ce6d78e"}, -] - -[package.dependencies] -asn1crypto = ">=1.5.1" - [[package]] name = "semver" version = "3.0.2" @@ -2953,48 +2934,61 @@ files = [ [[package]] name = "typing-extensions" -version = "4.10.0" +version = "4.12.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, - {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, + {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, + {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, +] + +[[package]] +name = "tzdata" +version = "2025.2" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +groups = ["main"] +markers = "sys_platform == \"win32\"" +files = [ + {file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"}, + {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, ] [[package]] name = "urllib3" -version = "2.5.0" +version = "2.6.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, - {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, + {file = "urllib3-2.6.0-py3-none-any.whl", hash = "sha256:c90f7a39f716c572c4e3e58509581ebd83f9b59cced005b7db7ad2d22b0db99f"}, + {file = "urllib3-2.6.0.tar.gz", hash = "sha256:cb9bcef5a4b345d5da5d145dc3e30834f58e8018828cbc724d30b4cb7d4d49f1"}, ] [package.extras] -brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "werkzeug" -version = "3.1.3" +version = "3.1.4" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e"}, - {file = "werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746"}, + {file = "werkzeug-3.1.4-py3-none-any.whl", hash = "sha256:2ad50fb9ed09cc3af22c54698351027ace879a0b60a3b5edf5730b2f7d876905"}, + {file = "werkzeug-3.1.4.tar.gz", hash = "sha256:cd3cd98b1b92dc3b7b3995038826c68097dcb16f9baa63abe35f20eafeb9fe5e"}, ] [package.dependencies] -MarkupSafe = ">=2.1.1" +markupsafe = ">=2.1.1" [package.extras] watchdog = ["watchdog (>=2.3)"] @@ -3117,4 +3111,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "65a6c03b69e81baa671c5b89b9f15784e07023b2b2686da3334de2282ffef4a4" +content-hash = "d26c5f69f86364a55d042c4860fc4edba25ef64d9e231d04e5cddc7df9e7ff5d" diff --git a/pay-admin/pyproject.toml b/pay-admin/pyproject.toml index db3a0cca4..b248e37e6 100644 --- a/pay-admin/pyproject.toml +++ b/pay-admin/pyproject.toml @@ -19,7 +19,7 @@ wtforms = "^3.1.2" werkzeug = "^3.0.1" itsdangerous = "^2.1.2" jinja2 = "^3.1.3" -pay-api = { git = "https://github.com/bcgov/sbc-pay.git", branch = "main", subdirectory = "pay-api" } +pay-api = { git = "https://github.com/seeker25/sbc-pay.git", branch = "fix_deadlock", subdirectory = "pay-api" } flask-session = {extras = ["filesystem"], version = "^0.8.0"} ruff = "^0.14.1" diff --git a/pay-api/poetry.lock b/pay-api/poetry.lock index 3edc2f04d..dc0b6c47f 100644 --- a/pay-api/poetry.lock +++ b/pay-api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -156,18 +156,6 @@ typing-extensions = ">=4" [package.extras] tz = ["backports.zoneinfo ; python_version < \"3.9\""] -[[package]] -name = "asn1crypto" -version = "1.5.1" -description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, - {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, -] - [[package]] name = "attrs" version = "23.2.0" @@ -1802,22 +1790,6 @@ files = [ {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, ] -[[package]] -name = "pg8000" -version = "1.31.5" -description = "PostgreSQL interface library" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "pg8000-1.31.5-py3-none-any.whl", hash = "sha256:0af2c1926b153307639868d2ee5cef6cd3a7d07448e12736989b10e1d491e201"}, - {file = "pg8000-1.31.5.tar.gz", hash = "sha256:46ebb03be52b7a77c03c725c79da2ca281d6e8f59577ca66b17c9009618cae78"}, -] - -[package.dependencies] -python-dateutil = ">=2.8.2" -scramp = ">=1.4.5" - [[package]] name = "pluggy" version = "1.5.0" @@ -1981,6 +1953,30 @@ files = [ {file = "protobuf-4.25.8.tar.gz", hash = "sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd"}, ] +[[package]] +name = "psycopg" +version = "3.3.1" +description = "PostgreSQL database adapter for Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "psycopg-3.3.1-py3-none-any.whl", hash = "sha256:e44d8eae209752efe46318f36dd0fdf5863e928009338d736843bb1084f6435c"}, + {file = "psycopg-3.3.1.tar.gz", hash = "sha256:ccfa30b75874eef809c0fbbb176554a2640cc1735a612accc2e2396a92442fc6"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.6", markers = "python_version < \"3.13\""} +tzdata = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +binary = ["psycopg-binary (==3.3.1) ; implementation_name != \"pypy\""] +c = ["psycopg-c (==3.3.1) ; implementation_name != \"pypy\""] +dev = ["ast-comments (>=1.1.2)", "black (>=24.1.0)", "codespell (>=2.2)", "cython-lint (>=0.16)", "dnspython (>=2.1)", "flake8 (>=4.0)", "isort-psycopg", "isort[colors] (>=6.0)", "mypy (>=1.19.0)", "pre-commit (>=4.0.1)", "types-setuptools (>=57.4)", "types-shapely (>=2.0)", "wheel (>=0.37)"] +docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] +pool = ["psycopg-pool"] +test = ["anyio (>=4.0)", "mypy (>=1.19.0) ; implementation_name != \"pypy\"", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] + [[package]] name = "psycopg2-binary" version = "2.9.9" @@ -2417,21 +2413,6 @@ reference = "HEAD" resolved_reference = "d640dc75ea51add2d611a30259d15d93d8654381" subdirectory = "python" -[[package]] -name = "scramp" -version = "1.4.5" -description = "An implementation of the SCRAM protocol." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "scramp-1.4.5-py3-none-any.whl", hash = "sha256:50e37c464fc67f37994e35bee4151e3d8f9320e9c204fca83a5d313c121bbbe7"}, - {file = "scramp-1.4.5.tar.gz", hash = "sha256:be3fbe774ca577a7a658117dca014e5d254d158cecae3dd60332dfe33ce6d78e"}, -] - -[package.dependencies] -asn1crypto = ">=1.5.1" - [[package]] name = "semver" version = "3.0.2" @@ -2738,33 +2719,46 @@ files = [ [[package]] name = "typing-extensions" -version = "4.10.0" +version = "4.12.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, - {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, + {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, + {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, +] + +[[package]] +name = "tzdata" +version = "2025.2" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +groups = ["main"] +markers = "sys_platform == \"win32\"" +files = [ + {file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"}, + {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, ] [[package]] name = "urllib3" -version = "2.5.0" +version = "2.6.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, - {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, + {file = "urllib3-2.6.0-py3-none-any.whl", hash = "sha256:c90f7a39f716c572c4e3e58509581ebd83f9b59cced005b7db7ad2d22b0db99f"}, + {file = "urllib3-2.6.0.tar.gz", hash = "sha256:cb9bcef5a4b345d5da5d145dc3e30834f58e8018828cbc724d30b4cb7d4d49f1"}, ] [package.extras] -brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "werkzeug" @@ -2884,4 +2878,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "8c4869954a27ff183a4ec995fd4e0f5fddc8742dee98fbd3c50e6f0fbdf99b3a" +content-hash = "bc635bd9ffa4da3462011bae79f8f8b41f28325c6cf9b645293f670a48cc11a0" diff --git a/pay-api/pyproject.toml b/pay-api/pyproject.toml index 18eb77f53..0a92cf379 100644 --- a/pay-api/pyproject.toml +++ b/pay-api/pyproject.toml @@ -67,10 +67,9 @@ six = "1.16.0" threadloop = "1.0.2" thrift = "0.16.0" tornado = "^6.5.1" -typing-extensions = "4.10.0" -urllib3 = "2.5.0" +typing-extensions = "4.12.0" +urllib3 = "2.6.0" sbc-common-components = {git = "https://github.com/bcgov/sbc-common-components.git", subdirectory = "python"} -pg8000 = "^1.31.5" sql-versioning = { git = "https://github.com/bcgov/sbc-connect-common.git", subdirectory = "python/sql-versioning", branch = "main" } aiohttp = "^3.12.14" setuptools = "^78.1.1" @@ -79,6 +78,7 @@ cachelib = "^0.13.0" flask-jwt-oidc = {git = "https://github.com/seeker25/flask-jwt-oidc.git", branch="main"} structured-logging = {git = "https://github.com/bcgov/sbc-connect-common.git", rev = "main", subdirectory = "python/structured-logging"} google-cloud-storage = "^2.19.0" +psycopg = "^3.3.1" [tool.poetry.group.dev.dependencies] pytest = "^8.3.4" diff --git a/pay-api/src/pay_api/config.py b/pay-api/src/pay_api/config.py index 1067936a8..caa024f5d 100755 --- a/pay-api/src/pay_api/config.py +++ b/pay-api/src/pay_api/config.py @@ -104,10 +104,10 @@ class _Config: # pylint: disable=too-few-public-methods # POSTGRESQL if DB_UNIX_SOCKET := os.getenv("DATABASE_UNIX_SOCKET", None): SQLALCHEMY_DATABASE_URI = ( - f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?unix_sock={DB_UNIX_SOCKET}/.s.PGSQL.5432" + f"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?host={DB_UNIX_SOCKET}&port={DB_PORT}" ) else: - SQLALCHEMY_DATABASE_URI = f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}" + SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}" # JWT_OIDC Settings JWT_OIDC_WELL_KNOWN_CONFIG = _get_config("JWT_OIDC_WELL_KNOWN_CONFIG") @@ -251,7 +251,7 @@ class TestConfig(_Config): # pylint: disable=too-few-public-methods DB_NAME = f"pay-test-{worker_id}" SQLALCHEMY_DATABASE_URI = _get_config( - "DATABASE_TEST_URL", default=f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}" + "DATABASE_TEST_URL", default=f"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}" ) SQLALCHEMY_DATABASE_URI = SQLALCHEMY_DATABASE_URI.rsplit("/", 1)[0] + f"/{DB_NAME}" diff --git a/pay-api/src/pay_api/dtos/refund.py b/pay-api/src/pay_api/dtos/refund.py index 232f16d32..6245cfe58 100644 --- a/pay-api/src/pay_api/dtos/refund.py +++ b/pay-api/src/pay_api/dtos/refund.py @@ -26,6 +26,7 @@ class RefundPatchRequest(Serializable): class RefundRequestGetRequest(Serializable): """Refund search.""" + invoice_id: int = None refund_status: str = None requested_by: str = None requested_start_date: str = None diff --git a/pay-api/src/pay_api/models/__init__.py b/pay-api/src/pay_api/models/__init__.py index f3778f2f2..35def32c5 100755 --- a/pay-api/src/pay_api/models/__init__.py +++ b/pay-api/src/pay_api/models/__init__.py @@ -74,4 +74,5 @@ from .comment import Comment, CommentSchema # isort: skip - This has to be at the bottom otherwise FeeSchedule errors + event.listen(Engine, "before_cursor_execute", DBTracing.query_tracing) diff --git a/pay-api/src/pay_api/models/invoice.py b/pay-api/src/pay_api/models/invoice.py index 8412ac804..5b4d3004d 100644 --- a/pay-api/src/pay_api/models/invoice.py +++ b/pay-api/src/pay_api/models/invoice.py @@ -28,7 +28,6 @@ from pay_api.models.applied_credits import AppliedCreditsSearchModel from pay_api.models.payment_line_item import PaymentLineItemSearchModel -from pay_api.utils.converter import Converter from pay_api.utils.enums import InvoiceReferenceStatus, InvoiceStatus, LineItemStatus, PaymentMethod, PaymentStatus from .audit import Audit, AuditSchema @@ -343,6 +342,7 @@ class InvoiceSearchModel: # pylint: disable=too-few-public-methods, too-many-in full_refundable: bool | None latest_refund_id: int | None latest_refund_status: str | None + routing_slip: str | None @classmethod def from_row( @@ -387,6 +387,7 @@ def from_row( disbursement_date=row.disbursement_date, disbursement_reversal_date=row.disbursement_reversal_date, invoice_number=row.references[0].invoice_number if len(row.references) > 0 else None, + routing_slip=getattr(row, "routing_slip", None), # refund fields that are optional, this might not be returned if not using the invoice composite model latest_refund_id=getattr(row, "latest_refund_id", None), latest_refund_status=getattr(row, "latest_refund_status", None), @@ -400,9 +401,3 @@ def from_row( [AppliedCreditsSearchModel.from_row(x) for x in row.applied_credits] if row.applied_credits else None ), ) - - @classmethod - def dao_to_dict(cls, invoice_dao: Invoice) -> dict: - """Convert from DAO to Schema dict.""" - invoice_dict = Converter().unstructure(InvoiceSearchModel.from_row(invoice_dao)) - return invoice_dict diff --git a/pay-api/src/pay_api/models/refund.py b/pay-api/src/pay_api/models/refund.py index 996aa00fb..695363db3 100644 --- a/pay-api/src/pay_api/models/refund.py +++ b/pay-api/src/pay_api/models/refund.py @@ -131,6 +131,7 @@ class PartialRefundLineDTO: # pylint: disable=too-few-public-methods """Schema used to serialize refund partial lines.""" payment_line_item_id: int + description: str statutory_fee_amount: Decimal future_effective_fee_amount: Decimal priority_fee_amount: Decimal @@ -144,6 +145,7 @@ def from_row(cls, row: dict): """ return cls( payment_line_item_id=row.get("payment_line_item_id"), + description=row.get("description"), statutory_fee_amount=row.get("statutory_fee_amount"), future_effective_fee_amount=row.get("future_effective_fee_amount"), priority_fee_amount=row.get("priority_fee_amount"), diff --git a/pay-api/src/pay_api/models/search/invoice_composite_model.py b/pay-api/src/pay_api/models/search/invoice_composite_model.py index a2156732a..b149ebcab 100644 --- a/pay-api/src/pay_api/models/search/invoice_composite_model.py +++ b/pay-api/src/pay_api/models/search/invoice_composite_model.py @@ -13,19 +13,22 @@ # limitations under the License. """Composite Model to handle invoice search queries.""" +from typing import Self + from sqlalchemy import and_, exists, func, select from sqlalchemy.orm import column_property +from sqlalchemy.orm.decl_api import declared_attr from pay_api.models import Invoice as InvoiceModel -from pay_api.models import PaymentMethod +from pay_api.models import InvoiceSearchModel, PaymentLineItemSchema, PaymentMethod from pay_api.models import Refund as RefundModel +from pay_api.utils.converter import Converter from pay_api.utils.enums import InvoiceStatus -class InvoiceCompositeModel(InvoiceModel): - """This class is a composite of the Invoice and other additional information required for search results.""" - - latest_refund_subq = ( +def get_latest_refund_subq(): + """Get the latest refund subquery.""" + return ( select(RefundModel.id.label("refund_id"), RefundModel.status.label("refund_status")) .where(RefundModel.invoice_id == InvoiceModel.id) .order_by(RefundModel.id.desc()) @@ -34,11 +37,22 @@ class InvoiceCompositeModel(InvoiceModel): .subquery() ) - latest_refund_id_expr = select(latest_refund_subq.c.refund_id).scalar_subquery() - latest_refund_status_expr = select(latest_refund_subq.c.refund_status).scalar_subquery() +def get_latest_refund_id_expr(): + """Get the latest refund ID expression.""" + latest_refund_subq = get_latest_refund_subq() + return select(latest_refund_subq.c.refund_id).scalar_subquery() + - full_refundable_expr = ( +def get_latest_refund_status_expr(): + """Get the latest refund status expression.""" + latest_refund_subq = get_latest_refund_subq() + return select(latest_refund_subq.c.refund_status).scalar_subquery() + + +def get_full_refundable_expr(): + """Get the full refundable expression.""" + return ( exists() .where( and_( @@ -49,7 +63,10 @@ class InvoiceCompositeModel(InvoiceModel): .label("full_refundable") ) - partial_refundable_expr = ( + +def get_partial_refundable_expr(): + """Get the partial refundable expression.""" + return ( exists() .where( and_( @@ -62,7 +79,39 @@ class InvoiceCompositeModel(InvoiceModel): .label("partial_refundable") ) - latest_refund_id = column_property(latest_refund_id_expr) - latest_refund_status = column_property(latest_refund_status_expr) - full_refundable = column_property(full_refundable_expr) - partial_refundable = column_property(partial_refundable_expr) + +class InvoiceCompositeModel(InvoiceModel): + """This class is a composite of the Invoice and other additional information required for search results.""" + + @declared_attr + def latest_refund_id(self): + """Latest refund ID as a column property.""" + return column_property(get_latest_refund_id_expr()) + + @declared_attr + def latest_refund_status(self): + """Latest refund status as a column property.""" + return column_property(get_latest_refund_status_expr()) + + @declared_attr + def full_refundable(self): + """Full refundable indicator as a column property.""" + return column_property(get_full_refundable_expr()) + + @declared_attr + def partial_refundable(self): + """Partial refundable indicator as a column property.""" + return column_property(get_partial_refundable_expr()) + + @classmethod + def dao_to_dict(cls, invoice_dao: Self) -> dict: + """Convert from DAO to Schema dict.""" + invoice_dict = Converter().unstructure(InvoiceSearchModel.from_row(invoice_dao)) + # This is done for backwards compatibility and due to the mixture of two schema frameworks and only used for + # the invoice composite route. + # This will be refactored in an upcoming ticket to remove marshmallow and consolidate schema definitions + if invoice_dao.payment_line_items: + line_items_schema = PaymentLineItemSchema(many=True) + invoice_dict["line_items"] = line_items_schema.dump(invoice_dao.payment_line_items) + + return invoice_dict diff --git a/pay-api/src/pay_api/resources/v1/refund_requests.py b/pay-api/src/pay_api/resources/v1/refund_requests.py index ccdb06e2c..16b64a43d 100644 --- a/pay-api/src/pay_api/resources/v1/refund_requests.py +++ b/pay-api/src/pay_api/resources/v1/refund_requests.py @@ -48,6 +48,7 @@ def get_refund_requests(): response, status = ( RefundRequestService.search( RefundRequestsSearch( + invoice_id=request_data.invoice_id, payment_method=request_data.payment_method, refund_reason=request_data.refund_reason, refund_amount=request_data.refund_amount, diff --git a/pay-api/src/pay_api/services/auth.py b/pay-api/src/pay_api/services/auth.py index 453f11a64..094e2af98 100644 --- a/pay-api/src/pay_api/services/auth.py +++ b/pay-api/src/pay_api/services/auth.py @@ -199,3 +199,21 @@ def get_service_account_token(): ) bearer_token = token_response.json()["access_token"] return bearer_token + + +def get_account_info_with_contact(**kwargs) -> dict: + """Get account info with contact details from kwargs.""" + account_info = {} + if kwargs.get("auth", None): + account_id = kwargs.get("auth")["account"]["id"] + contact_url = current_app.config.get("AUTH_API_ENDPOINT") + f"orgs/{account_id}/contacts" + contact = RestService.get( + endpoint=contact_url, + token=kwargs["user"].bearer_token, + auth_header_type=AuthHeaderType.BEARER, + content_type=ContentType.JSON, + ).json() + + account_info = kwargs.get("auth").get("account") + account_info["contact"] = contact["contacts"][0] + return account_info diff --git a/pay-api/src/pay_api/services/base_payment_system.py b/pay-api/src/pay_api/services/base_payment_system.py index b86824a8e..60793ff4b 100644 --- a/pay-api/src/pay_api/services/base_payment_system.py +++ b/pay-api/src/pay_api/services/base_payment_system.py @@ -17,10 +17,11 @@ import functools import traceback from abc import ABC, abstractmethod +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime from typing import Any -from flask import current_app +from flask import copy_current_request_context, current_app from sbc_common_components.utils.enums import QueueMessageTypes from pay_api.exceptions import BusinessException @@ -57,6 +58,8 @@ from .payment_line_item import PaymentLineItem +_executor = ThreadPoolExecutor(max_workers=5) + class PaymentSystemService(ABC): # pylint: disable=too-many-instance-attributes, too-many-public-methods """Abstract base class for payment system. @@ -252,7 +255,7 @@ def _refund_and_create_credit_memo( # Don't do anything is the status is APPROVED. is_partial = bool(refund_partial) - current_app.logger.info(f"Creating credit memo for invoice : {invoice.id}, {invoice.invoice_status_code}") + current_app.logger.info(f"Processing refund for invoice : {invoice.id}, {invoice.invoice_status_code}") if is_partial and invoice.invoice_status_code != InvoiceStatus.PAID.value: raise BusinessException(Error.PARTIAL_REFUND_INVOICE_NOT_PAID) @@ -289,6 +292,7 @@ def _refund_and_create_credit_memo( line_items = [PaymentLineItemModel.find_by_id(li.id) for li in invoice.payment_line_items] refund_amount = invoice.total + current_app.logger.info(f"Creating credit memo for invoice : {invoice.id}, {invoice.invoice_status_code}") cms_response = CFSService.create_cms(line_items=line_items, cfs_account=cfs_account) # TODO Create a payment record for this to show up on transactions, when the ticket comes. # Create a credit with CM identifier as CMs are not reported in payment interface file @@ -319,15 +323,22 @@ def _refund_and_create_credit_memo( current_app.logger.info( f"Updating {cfs_account.payment_method} credit amount for account {payment_account.auth_account_id}" ) - payment_account.flush() - try: - if send_credit_notification: - PaymentSystemService._send_credit_notification(payment_account, refund_amount) - except Exception as e: - current_app.logger.error( - f"{{Error sending credit notification: {str(e)} stack_trace: {traceback.format_exc()}}}" - ) + if send_credit_notification: + + @copy_current_request_context + def _send_notification(): + """Send credit notification in background thread.""" + try: + PaymentSystemService._send_credit_notification(payment_account, refund_amount) + except Exception as e: + current_app.logger.error( + f"{{Error sending credit notification: {str(e)} stack_trace: {traceback.format_exc()}}}" + ) + + _executor.submit(_send_notification) + + payment_account.flush() if is_partial and refund_amount != invoice.total: return InvoiceStatus.PAID.value diff --git a/pay-api/src/pay_api/services/invoice.py b/pay-api/src/pay_api/services/invoice.py index 7a47cdc8f..a8c66a5c6 100644 --- a/pay-api/src/pay_api/services/invoice.py +++ b/pay-api/src/pay_api/services/invoice.py @@ -26,7 +26,7 @@ from pay_api.exceptions import BusinessException from pay_api.models import CfsAccount as CfsAccountModel from pay_api.models import Invoice as InvoiceModel -from pay_api.models import InvoiceCompositeModel, InvoiceSchema, InvoiceSearchModel, db +from pay_api.models import InvoiceCompositeModel, InvoiceSchema, db from pay_api.models import InvoiceReference as InvoiceReferenceModel from pay_api.models import PaymentAccount as PaymentAccountModel from pay_api.services.auth import check_auth @@ -443,7 +443,7 @@ def find_composite_by_id(identifier: int, **kwargs): abort(403) current_app.logger.debug(">find_composite_by_id") - return InvoiceSearchModel.dao_to_dict(invoice_composite) + return InvoiceCompositeModel.dao_to_dict(invoice_composite) @staticmethod def find_invoices_for_payment( diff --git a/pay-api/src/pay_api/services/invoice_search.py b/pay-api/src/pay_api/services/invoice_search.py index 63d807d73..de236170c 100644 --- a/pay-api/src/pay_api/services/invoice_search.py +++ b/pay-api/src/pay_api/services/invoice_search.py @@ -13,6 +13,9 @@ # limitations under the License. """Service to support invoice searches.""" +from collections import defaultdict +from datetime import datetime + from dateutil import parser from flask import current_app from sqlalchemy import String, and_, cast, exists, func, or_, select @@ -33,23 +36,31 @@ db, ) from pay_api.models.payment import TransactionSearchParams -from pay_api.models.search.invoice_composite_model import InvoiceCompositeModel -from pay_api.services.code import Code as CodeService +from pay_api.models.search.invoice_composite_model import ( + InvoiceCompositeModel, + get_full_refundable_expr, + get_latest_refund_id_expr, + get_latest_refund_status_expr, + get_partial_refundable_expr, +) +from pay_api.models.statement import Statement +from pay_api.services.auth import get_account_info_with_contact from pay_api.services.invoice import Invoice as InvoiceService -from pay_api.services.oauth_service import OAuthService from pay_api.services.payment import PaymentReportInput -from pay_api.services.payment_calculations import ( - build_grouped_invoice_context, - build_statement_context, - build_statement_summary_context, -) from pay_api.utils.converter import Converter from pay_api.utils.dataclasses import PurchaseHistorySearch -from pay_api.utils.enums import AuthHeaderType, Code, ContentType, InvoiceStatus, PaymentMethod, RefundStatus +from pay_api.utils.enums import ContentType, InvoiceStatus, PaymentMethod, RefundStatus, StatementTemplate from pay_api.utils.errors import Error from pay_api.utils.sqlalchemy import JSONPath +from pay_api.utils.statement_dtos import ( + GroupedInvoicesDTO, + StatementContextDTO, + StatementPDFContextDTO, + StatementSummaryDTO, + SummariesGroupedByPaymentMethodDTO, +) from pay_api.utils.user_context import user_context -from pay_api.utils.util import get_local_formatted_date, get_statement_currency_string +from pay_api.utils.util import get_local_formatted_date from .csv_service import CsvService from .report_service import ReportRequest, ReportService @@ -110,12 +121,10 @@ def generate_base_transaction_query(cls, include_credits_and_partial_refunds: bo InvoiceReference.reference_number, InvoiceReference.status_code, ), - with_expression(InvoiceCompositeModel.latest_refund_id, InvoiceCompositeModel.latest_refund_id_expr), - with_expression( - InvoiceCompositeModel.latest_refund_status, InvoiceCompositeModel.latest_refund_status_expr - ), - with_expression(InvoiceCompositeModel.full_refundable, InvoiceCompositeModel.full_refundable_expr), - with_expression(InvoiceCompositeModel.partial_refundable, InvoiceCompositeModel.partial_refundable_expr), + with_expression(InvoiceCompositeModel.latest_refund_id, get_latest_refund_id_expr()), + with_expression(InvoiceCompositeModel.latest_refund_status, get_latest_refund_status_expr()), + with_expression(InvoiceCompositeModel.full_refundable, get_full_refundable_expr()), + with_expression(InvoiceCompositeModel.partial_refundable, get_partial_refundable_expr()), ] if include_credits_and_partial_refunds: @@ -432,7 +441,9 @@ def create_payment_report_details(cls, purchases: tuple, data: dict) -> dict: # return data @staticmethod - def search_all_purchase_history(auth_account_id: str, search_filter: dict, content_type: str): + def search_all_purchase_history( + auth_account_id: str, search_filter: dict, query_only: bool = False + ): """Return all results for the purchase history.""" return InvoiceSearch.search_purchase_history( PurchaseHistorySearch( @@ -441,7 +452,7 @@ def search_all_purchase_history(auth_account_id: str, search_filter: dict, conte page=0, limit=0, return_all=True, - query_only=content_type == ContentType.CSV.value, + query_only=query_only, ) ) @@ -450,7 +461,9 @@ def create_payment_report(auth_account_id: str, search_filter: dict, content_typ """Create payment report.""" current_app.logger.debug(f" dict: @staticmethod @user_context - def generate_payment_report(report_inputs: PaymentReportInput, **kwargs): # pylint: disable=too-many-locals - """Prepare data and generate payment report by calling report api.""" + def generate_payment_report(report_inputs: PaymentReportInput, **kwargs): # noqa: ARG004 pylint: disable=too-many-locals,unused-argument + """Prepare data and generate payment report by calling report api. + + This method is used for: + 1. Statement CSV export - uses statement_report template + 2. Payment transactions report (CSV/PDF) - uses payment_transactions template + + Both CSV and PDF use the same simple format: columns + values. + Note: PDF statement generation uses generate_statement_pdf_report instead. + """ content_type = report_inputs.content_type results = report_inputs.results report_name = report_inputs.report_name template_name = report_inputs.template_name + csv_data = CsvService.prepare_csv_data(results) if content_type == ContentType.CSV.value: - csv_data = CsvService.prepare_csv_data(results) return CsvService.create_report(csv_data) else: - # Use the status_code_description instead of status_code. - # Future: move this into something more specialized. - invoice_status_codes = CodeService.find_code_values_by_type(Code.INVOICE_STATUS.value) - for invoice in results.get("items", None): - filtered_codes = [cd for cd in invoice_status_codes["codes"] if cd["code"] == invoice["status_code"]] - if filtered_codes: - invoice["status_code"] = filtered_codes[0]["description"] - invoices = results.get("items", None) - statement = kwargs.get("statement", {}) - totals = InvoiceSearch.get_invoices_totals(invoices, statement) - - account_info = None - if kwargs.get("auth", None): - account_id = kwargs.get("auth")["account"]["id"] - contact_url = current_app.config.get("AUTH_API_ENDPOINT") + f"orgs/{account_id}/contacts" - contact = OAuthService.get( - endpoint=contact_url, - token=kwargs["user"].bearer_token, - auth_header_type=AuthHeaderType.BEARER, - content_type=ContentType.JSON, - ).json() - - account_info = kwargs.get("auth").get("account") - account_info["contact"] = contact["contacts"][0] # Get the first one from the list - - invoices = results.get("items", []) - statement_summary = report_inputs.statement_summary - grouped_invoices: list[dict] = build_grouped_invoice_context(invoices, statement, statement_summary) - - has_payment_instructions = any(item.get("payment_method") == "EFT" for item in grouped_invoices) - - formatted_totals = {} - for key, value in totals.items(): - formatted_totals[key] = get_statement_currency_string(value) - - template_vars = { - "statementSummary": build_statement_summary_context(statement_summary), - "groupedInvoices": grouped_invoices, - "total": formatted_totals, - "account": account_info, - "statement": build_statement_context(kwargs.get("statement")), - } - - if has_payment_instructions: - template_vars["hasPaymentInstructions"] = True + report_response = ReportService.get_report_response( + ReportRequest( + report_name=report_name, + template_name=template_name, + template_vars=csv_data, + populate_page_number=True, + content_type=content_type, + ) + ) + return report_response + + @staticmethod + @user_context + def generate_statement_pdf_report( + invoices_orm: list[Invoice], + db_summaries: SummariesGroupedByPaymentMethodDTO, + statement: Statement, + statement_summary: dict, + report_name: str, + content_type: str, + **kwargs, + ): + """Generate PDF statement report using ORM objects and database summaries.""" + db_summaries = db_summaries.summaries + + statement_to_date = statement.to_date + account_info = get_account_info_with_contact(**kwargs) + + grouped_invoices = InvoiceSearch._group_invoices_by_payment_method( + invoices_orm=invoices_orm, + db_summaries=db_summaries, + statement=statement, + statement_summary=statement_summary, + statement_to_date=statement_to_date, + ) + + statement_summary_dto = StatementSummaryDTO.from_dict(statement_summary) + statement_dto = StatementContextDTO.from_statement(statement) + + context_dto = StatementPDFContextDTO( + statement_summary=statement_summary_dto, + grouped_invoices=grouped_invoices, + account=account_info, + statement=statement_dto, + has_payment_instructions=any(g.payment_method == PaymentMethod.EFT.value for g in grouped_invoices), + ) + + template_vars = context_dto.to_dict() report_response = ReportService.get_report_response( ReportRequest( report_name=report_name, - template_name=template_name, + template_name=StatementTemplate.STATEMENT_REPORT.value, template_vars=template_vars, populate_page_number=True, content_type=content_type, @@ -579,3 +603,38 @@ def generate_payment_report(report_inputs: PaymentReportInput, **kwargs): # pyl ) ) return report_response + + @staticmethod + def _group_invoices_by_payment_method( + invoices_orm: list, + db_summaries: dict, + statement: Statement, + statement_summary: dict, + statement_to_date: datetime, + ) -> list[GroupedInvoicesDTO]: + """Group invoices by payment method and create DTOs.""" + grouped_by_method = defaultdict(list) + for invoice in invoices_orm: + grouped_by_method[invoice.payment_method_code].append(invoice) + + grouped_invoices = [] + for method in [m.value for m in PaymentMethod.Order]: + if method not in grouped_by_method: + continue + + items = grouped_by_method[method] + summary = db_summaries.get(method, {}) + + group_dto = GroupedInvoicesDTO.from_invoices_and_summary( + payment_method=method, + invoices_orm=items, + db_summary=summary, + statement=statement, + statement_summary=statement_summary, + statement_to_date=statement_to_date, + is_first=(len(grouped_invoices) == 0), + ) + + grouped_invoices.append(group_dto) + + return grouped_invoices diff --git a/pay-api/src/pay_api/services/payment_account.py b/pay-api/src/pay_api/services/payment_account.py index 929eeb334..a048daa12 100644 --- a/pay-api/src/pay_api/services/payment_account.py +++ b/pay-api/src/pay_api/services/payment_account.py @@ -460,6 +460,7 @@ def update(cls, auth_account_id: str, account_request: dict[str, Any]) -> Paymen """Create or update payment account record.""" current_app.logger.debug(" bool: - """Determine if service was provided based on invoice status code and payment method.""" - status_code = status_code.upper().replace(" ", "_") - if status_code in InvoiceStatus.__members__: - status_enum = InvoiceStatus[status_code] - else: - status_enum = next((s for s in InvoiceStatus if s.value == status_code), status_code) - - if status_enum is None: - return False - - default_statuses = { - InvoiceStatus.PAID, - InvoiceStatus.CANCELLED, - InvoiceStatus.CREDITED, - InvoiceStatus.REFUND_REQUESTED, - InvoiceStatus.REFUNDED, - InvoiceStatus.COMPLETED, - } - - if status_enum in default_statuses: - return True - - match payment_method: - case PaymentMethod.PAD.value: - return status_enum in { - InvoiceStatus.APPROVED, - InvoiceStatus.SETTLEMENT_SCHEDULED, - } - - case PaymentMethod.EFT.value: - return status_enum in { - InvoiceStatus.APPROVED, - InvoiceStatus.OVERDUE, - } - - case PaymentMethod.EJV.value: - return status_enum in { - InvoiceStatus.APPROVED, - } - - case PaymentMethod.INTERNAL.value: - return status_enum in { - InvoiceStatus.APPROVED, - } - - case _: - return False - - -def build_grouped_invoice_context(invoices: list[dict], statement: dict, statement_summary: dict) -> list[dict]: - """Build grouped invoice context, with fixed payment method order.""" - grouped = defaultdict(list) - for inv in invoices: - method = inv.get("payment_method") - grouped[method].append(inv) - - grouped_invoices = [] - first_group = True - - for method in [m.value for m in PaymentMethod.Order]: - if method not in grouped: - continue - - items = grouped[method] - transactions = build_transaction_rows(items, method, statement) - summary = calculate_invoice_summaries(items, method, statement) - has_staff_payment = False - if method == PaymentMethod.INTERNAL.value: - has_staff_payment = any("routing_slip" not in inv or inv["routing_slip"] is None for inv in items) - statement_header_text = ( - StatementTitles["INTERNAL_STAFF"].value if has_staff_payment else StatementTitles[method].value - ) - else: - statement_header_text = StatementTitles[method].value - - method_context = { - **summary, - "payment_method": method, - "total_paid": get_statement_currency_string(sum(Decimal(inv.get("paid", 0)) for inv in items)), - "transactions": transactions, - "is_index_0": first_group, - "statement_header_text": statement_header_text, - "include_service_provided": any(t.get("service_provided", False) for t in transactions), - } - - if method == PaymentMethod.EFT.value: - method_context["amount_owing"] = get_statement_currency_string(statement.get("amount_owing", 0.00)) - if statement.get("is_interim_statement") and statement_summary: - method_context["latest_payment_date"] = statement_summary.get("latestStatementPaymentDate") - elif not statement.get("is_interim_statement") and statement_summary: - method_context["due_date"] = get_statement_date_string(statement_summary.get("dueDate")) - - if method == PaymentMethod.INTERNAL.value: - method_context["is_staff_payment"] = has_staff_payment - - grouped_invoices.append(method_context) - first_group = False - - return grouped_invoices - - -def calculate_invoice_summaries(invoices: list[dict], payment_method: str, statement: dict) -> dict: - """Calculate invoice summaries for a payment method using database aggregation.""" - invoice_ids = [inv.get("id") for inv in invoices if inv.get("payment_method") == payment_method and inv.get("id")] - statement_to_date = statement.get("to_date") - - if not invoice_ids: - return { - "paid_summary": 0.0, - "due_summary": 0.0, - "totals_summary": 0.0, - "fees_total": 0.0, - "service_fees_total": 0.0, - "gst_total": 0.0, - "refunds_total": 0.0, - "credits_total": 0.0, - } - - if payment_method not in [PaymentMethod.EFT.value, PaymentMethod.PAD.value]: - # For non-EFT: refund applies if paid == 0 and refund > 0 - refund_condition = case((and_(InvoiceModel.paid == 0, InvoiceModel.refund > 0), InvoiceModel.refund), else_=0) - else: - # For EFT: refund applies if paid == 0 and refund > 0 and refund_date <= statement.to_date - if statement_to_date: - refund_condition = case( - ( - and_( - InvoiceModel.paid == 0, - InvoiceModel.refund > 0, - InvoiceModel.refund_date.isnot(None), - InvoiceModel.refund_date <= func.cast(statement_to_date, db.Date), - ), - InvoiceModel.refund, - ), - else_=0, - ) - else: - # Fallback if no statement_to_date provided - refund_condition = case( - (and_(InvoiceModel.paid == 0, InvoiceModel.refund > 0), InvoiceModel.refund), else_=0 - ) - - if payment_method in [PaymentMethod.EFT.value, PaymentMethod.PAD.value] and statement_to_date: - paid_condition = case( - ( - and_( - InvoiceModel.payment_date.isnot(None), - InvoiceModel.payment_date <= func.cast(statement_to_date, db.Date), - ), - InvoiceModel.paid, - ), - else_=0, - ) - else: - paid_condition = InvoiceModel.paid - - result = ( - db.session.query( - func.coalesce(func.sum(paid_condition), 0).label("paid_summary"), - func.coalesce(func.sum(InvoiceModel.total - refund_condition), 0).label("totals_summary"), - func.coalesce(func.sum(InvoiceModel.total - paid_condition - refund_condition), 0).label("due_summary"), - func.coalesce(func.sum(InvoiceModel.total - InvoiceModel.service_fees - InvoiceModel.gst), 0).label( - "fees_total" - ), - func.coalesce(func.sum(InvoiceModel.service_fees), 0).label("service_fees_total"), - func.coalesce(func.sum(InvoiceModel.gst), 0).label("gst_total"), - func.coalesce( - func.sum( - case( - (InvoiceModel.invoice_status_code == InvoiceStatus.REFUNDED.value, InvoiceModel.refund), - else_=0, - ) - ), - 0, - ).label("refunds_total"), - func.coalesce( - func.sum( - case( - (InvoiceModel.invoice_status_code == InvoiceStatus.CREDITED.value, InvoiceModel.refund), - else_=0, - ) - ), - 0, - ).label("credits_total"), - ).filter(and_(InvoiceModel.id.in_(invoice_ids), InvoiceModel.payment_method_code == payment_method)) - ).first() - - summary = {k: float(v or 0) for k, v in result._asdict().items()} - return summary - - -def get_statement_status_for_invoice(inv: dict, payment_method: str, statement: dict) -> str: - """For PAD: if payment_date is after statement.to_date, mark as 'Pending'.""" - default_status = inv.get("status_code", "") - - if payment_method == PaymentMethod.PAD.value and inv: - payment_date = inv.get("payment_date") - to_date = (statement or {}).get("to_date") - if payment_date and to_date and parser.parse(payment_date) > parser.parse(to_date): - return InvoiceStatus.APPROVED.value - return default_status - - -@dataclass -class TransactionRow: - """transactions details.""" - - products: list[str] - details: list[str] - folio: str - created_on: str - fee: str - service_fee: str - gst: str - total: str - extra: dict = field(default_factory=dict) - status_code: str = "" - - -def build_transaction_rows( - invoices: list[dict], payment_method: PaymentMethod = None, statement: dict = None -) -> list[dict]: - """Build transactions for grouped_invoices.""" - rows = [] - for inv in invoices: - product_lines = [] - for item in inv.get("line_items", []): - label = "(Cancelled) " if inv.get("status_code") == InvoiceStatus.CANCELLED.value else "" - product_lines.append(f"{label}{item.get('description', '')}") - - detail_lines = [] - for detail in inv.get("details", []): - detail_lines.append(f"{detail.get('label', '')} {detail.get('value', '')}") - fee = max(inv.get("total", 0) - inv.get("service_fees", 0) - inv.get("gst", 0), 0) - - row = TransactionRow( - products=product_lines, - details=detail_lines, - folio=inv.get("folio_number") or "-", - created_on=get_statement_date_string( - datetime.fromisoformat(inv["created_on"]).strftime("%b %d,%Y") if inv.get("created_on") else "-" - ), - fee=get_statement_currency_string(fee), - service_fee=get_statement_currency_string(inv.get("service_fees", 0)), - gst=get_statement_currency_string(inv.get("gst", 0)), - total=get_statement_currency_string(inv.get("total", 0)), - extra={ - k: v - for k, v in inv.items() - if k - not in { - "details", - "folio_number", - "created_on", - "fee", - "gst", - "total", - "service_fees", - "status_code", - } - }, - ) - service_provided = False - if payment_method: - service_provided = determine_service_provision_status(inv.get("status_code", ""), payment_method) - row.status_code = get_statement_status_for_invoice(inv, payment_method, statement) - row.extra["service_provided"] = service_provided - - row_dict = cattrs.unstructure(row) - row_dict.update(row_dict.pop("extra")) - rows.append(row_dict) - - return rows - - -@dataclass -class StatementContext: - """A class representing the context of a statement.""" - - duration: str | None = None - amount_owing: str | None = None - from_date: str | None = None - to_date: str | None = None - created_on: str | None = None - frequency: str | None = None - extra: dict = field(default_factory=dict) - - -def build_statement_context(statement: dict) -> dict: - """Build and enhance statement context with formatted fields.""" - if not statement: - return statement - - statement_ = statement.copy() - - from_date = get_statement_date_string(statement.get("from_date")) - to_date = get_statement_date_string(statement.get("to_date")) - created_on = get_statement_date_string(statement.get("created_on")) - frequency = statement.get("frequency", "") - - if frequency == StatementFrequency.DAILY.value and from_date: - duration = from_date - elif from_date and to_date: - duration = f"{from_date} - {to_date}" - elif from_date: - duration = from_date - else: - duration = None - - amount_owing = statement.get("amount_owing") - amount_owing_str = get_statement_currency_string(amount_owing) if amount_owing else get_statement_currency_string(0) - - enhanced_statement = StatementContext( - duration=duration, - amount_owing=amount_owing_str, - from_date=from_date, - to_date=to_date, - created_on=created_on, - frequency=frequency, - extra={ - k: v - for k, v in statement_.items() - if k not in {"from_date", "to_date", "amount_owing", "created_on", "frequency"} - }, - ) - enhanced_statement_dict = cattrs.unstructure(enhanced_statement) - enhanced_statement_dict.update(enhanced_statement_dict.pop("extra")) - return enhanced_statement_dict - - -@dataclass -class StatementSummary: - """A class representing the summary of a statement.""" - - last_statement_total: str | None = None - last_statement_paid_amount: str | None = None - cancelled_transactions: str | None = None - latest_statement_payment_date: str | None = None - due_date: str | None = None - extra: dict = field(default_factory=dict) - - -def build_statement_summary_context(statement_summary: dict) -> list[dict]: - """Build and enhance statement_summary context with formatted fields.""" - if not statement_summary: - return None - - def currency(v): - return get_statement_currency_string(v) - - def date(v): - return get_statement_date_string(v, "%B %d, %Y") if v else None - - handled_keys = {humps.camelize(f.name) for f in fields(StatementSummary)} - - summary_row = StatementSummary( - last_statement_total=currency(statement_summary.get("lastStatementTotal")), - last_statement_paid_amount=currency(statement_summary.get("lastStatementPaidAmount")), - cancelled_transactions=( - currency(statement_summary["cancelledTransactions"]) - if statement_summary.get("cancelledTransactions") not in [None, 0, "0", "0.00"] - else None - ), - latest_statement_payment_date=date(statement_summary.get("latestStatementPaymentDate")), - due_date=date(statement_summary.get("dueDate")), - extra={k: v for k, v in statement_summary.items() if k not in handled_keys}, - ) - - summary_row_dict = {humps.camelize(k): v for k, v in cattrs.unstructure(summary_row).items()} - summary_row_dict.update(summary_row_dict.pop("extra")) - if summary_row_dict.get("cancelledTransactions") is None: - summary_row_dict.pop("cancelledTransactions") - return summary_row_dict diff --git a/pay-api/src/pay_api/services/refund.py b/pay-api/src/pay_api/services/refund.py index 3adfe9719..33d65ff98 100644 --- a/pay-api/src/pay_api/services/refund.py +++ b/pay-api/src/pay_api/services/refund.py @@ -362,6 +362,7 @@ def create_refund(cls, invoice_id: int, request: dict[str, str], products: list[ return cls._complete_refund(invoice, refund, refund_partial_lines) refund.save() + payment_account = PaymentAccount.find_by_id(invoice.payment_account_id) product_recipients = get_product_refund_recipients(product_code=invoice.corp_type.product, refund=refund) if product_recipients: @@ -376,7 +377,7 @@ def create_refund(cls, invoice_id: int, request: dict[str, str], products: list[ refund_amount=refund_amount, invoice_id=invoice.id, invoice_reference_number=invoice.references[0].invoice_number if len(invoice.references) > 0 else None, - url=f"{current_app.config.get('PAY_WEB_URL')}/refund-request/{invoice.id}", + url=f"{current_app.config.get('PAY_WEB_URL')}/transaction-view/{refund.invoice_id}/refund-request/{refund.id}", ).render_body(status=refund.status, is_for_client=False) send_email(product_recipients, subject, html_body) @@ -503,7 +504,7 @@ def approve_or_decline_refund(refund: RefundModel, data: RefundPatchRequest, pro refund_amount=refund_total, invoice_id=invoice.id, invoice_reference_number=invoice.references[0].invoice_number if len(invoice.references) > 0 else None, - url=f"{current_app.config.get('PAY_WEB_URL')}/refund-request/{refund.id}", + url=f"{current_app.config.get('PAY_WEB_URL')}/transaction-view/{refund.invoice_id}/refund-request/{refund.id}", ) staff_html_body = email_config.render_body(status=refund.status, is_for_client=False) send_email( @@ -538,7 +539,9 @@ def normalize_partial_refund_lines(partial_refund_lines: list[RefundPartialModel payment_line_items[partial_refund_line.payment_line_item_id].append(partial_refund_line) for line_item_id, line_items in payment_line_items.items(): + line_item_model = PaymentLineItemModel.find_by_id(line_item_id) line_item_dto = PartialRefundLineDTO( + description=line_item_model.description, payment_line_item_id=line_item_id, statutory_fee_amount=Decimal(0), future_effective_fee_amount=Decimal(0), diff --git a/pay-api/src/pay_api/services/refund_request.py b/pay-api/src/pay_api/services/refund_request.py index 7b369410a..27f31ea4e 100644 --- a/pay-api/src/pay_api/services/refund_request.py +++ b/pay-api/src/pay_api/services/refund_request.py @@ -38,6 +38,7 @@ class RefundRequestsSearch: """Used for searching refund requests records.""" refund_type: str | None = RefundType.INVOICE.value + invoice_id: int | None = None payment_method: str | None = None refund_reason: str | None = None refund_amount: Decimal | None = None @@ -111,15 +112,18 @@ def get_search_query(cls, search_criteria: RefundRequestsSearch): current_app.logger.debug(" 0, prt_subquery.refund_partial_total), + else_=InvoiceModel.total, + ) + query = ( db.session.query( RefundModel, InvoiceModel.total.label("transaction_amount"), InvoiceModel.payment_method_code.label("payment_method"), - case( - (prt_subquery.refund_partial_total > 0, prt_subquery.refund_partial_total), - else_=InvoiceModel.total, - ).label("refund_amount"), + refund_amount_expr.label("refund_amount"), ) .join(InvoiceModel, InvoiceModel.id == RefundModel.invoice_id) .join(CorpTypeModel, CorpTypeModel.code == InvoiceModel.corp_type_code) @@ -128,6 +132,8 @@ def get_search_query(cls, search_criteria: RefundRequestsSearch): partial_refund_total_subquery.c.refund_id == RefundModel.id, ) ) + + query = query.filter_conditionally(search_criteria.invoice_id, InvoiceModel.id) query = query.filter_conditionally(search_criteria.refund_type, RefundModel.type) query = query.filter_conditionally(search_criteria.status, RefundModel.status) query = query.filter_conditionally(search_criteria.requested_by, RefundModel.requested_by, is_like=True) @@ -135,7 +141,7 @@ def get_search_query(cls, search_criteria: RefundRequestsSearch): query = query.filter_conditionally(search_criteria.refund_method, RefundModel.refund_method) query = query.filter_conditionally(search_criteria.payment_method, InvoiceModel.payment_method_code) query = query.filter_conditionally(search_criteria.transaction_amount, InvoiceModel.total) - query = query.filter_conditionally(search_criteria.refund_amount, prt_subquery.refund_partial_total) + query = query.filter_conditionally(search_criteria.refund_amount, refund_amount_expr) query = query.filter_conditional_date_range( start_date=search_criteria.requested_start_date, diff --git a/pay-api/src/pay_api/services/statement.py b/pay-api/src/pay_api/services/statement.py index 88d63bb6a..edbead458 100644 --- a/pay-api/src/pay_api/services/statement.py +++ b/pay-api/src/pay_api/services/statement.py @@ -13,13 +13,16 @@ # limitations under the License. """Service class to control all the operations related to statements.""" +from __future__ import annotations + from datetime import UTC, date, datetime, timedelta -from decimal import Decimal +from decimal import Decimal # noqa: TC003 from dateutil.relativedelta import relativedelta from flask import current_app from sqlalchemy import Integer, and_, case, cast, distinct, exists, func, literal, literal_column, select from sqlalchemy.dialects.postgresql import ARRAY, INTEGER +from sqlalchemy.orm import Query, subqueryload, with_expression from sqlalchemy.sql.functions import coalesce from pay_api.models import EFTCredit as EFTCreditModel @@ -28,11 +31,13 @@ from pay_api.models import EFTTransaction as EFTTransactionModel from pay_api.models import Invoice as InvoiceModel from pay_api.models import PaymentAccount as PaymentAccountModel +from pay_api.models import Refund as RefundModel +from pay_api.models import RefundsPartial, db from pay_api.models import Statement as StatementModel from pay_api.models import StatementInvoices as StatementInvoicesModel from pay_api.models import StatementSchema as StatementModelSchema from pay_api.models import StatementSettings as StatementSettingsModel -from pay_api.models import db +from pay_api.models.applied_credits import AppliedCredits from pay_api.services.activity_log_publisher import ActivityLogPublisher from pay_api.utils.constants import DT_SHORT_FORMAT from pay_api.utils.dataclasses import StatementIntervalChangeEvent @@ -45,9 +50,11 @@ NotificationStatus, PaymentMethod, QueueSources, + RefundsPartialStatus, StatementFrequency, StatementTemplate, ) +from pay_api.utils.statement_dtos import PaymentMethodSummaryRawDTO, SummariesGroupedByPaymentMethodDTO from pay_api.utils.util import get_first_and_last_of_frequency, get_local_time from .invoice import Invoice @@ -394,13 +401,15 @@ def _populate_statement_summary( if latest_payment_date is None or invoice.payment_date > latest_payment_date: latest_payment_date = invoice.payment_date + last_total = previous_totals["fees"] if previous_totals else 0 + last_paid = previous_totals["paid"] if previous_totals else 0 + return { - "lastStatementTotal": previous_totals["fees"] if previous_totals else 0, - "lastStatementPaidAmount": (previous_totals["paid"] if previous_totals else 0), - "latestStatementPaymentDate": ( - latest_payment_date.strftime(DT_SHORT_FORMAT) if latest_payment_date else None - ), + "lastStatementTotal": last_total, + "lastStatementPaidAmount": last_paid, + "latestStatementPaymentDate": latest_payment_date, "dueDate": cls.calculate_due_date(statement.to_date) if statement else None, + "balanceForward": last_total - last_paid, } @staticmethod @@ -409,13 +418,9 @@ def _build_statement_summary_for_methods( ) -> dict: """Build statement_summary for EFT and PAD without inflating locals in caller.""" summary: dict = {} - if Statement.is_payment_method_statement(statement_dao, statement_purchases, PaymentMethod.EFT.value): - summary.update(Statement._populate_statement_summary(statement_dao, statement_purchases, PaymentMethod.EFT)) - if Statement.is_payment_method_statement(statement_dao, statement_purchases, PaymentMethod.PAD.value): - pad_summary = Statement._populate_statement_summary(statement_dao, statement_purchases, PaymentMethod.PAD) - pad_amount = pad_summary.get("lastStatementPaidAmount") - if pad_amount: - summary["lastPADStatementPaidAmount"] = pad_amount + for method in [PaymentMethod.EFT, PaymentMethod.PAD]: + if Statement.is_payment_method_statement(statement_dao, statement_purchases, method.value): + summary.update(Statement._populate_statement_summary(statement_dao, statement_purchases, method)) return summary @staticmethod @@ -427,43 +432,56 @@ def get_statement_report(statement_id: str, content_type: str, **kwargs): statement_dao: StatementModel = Statement.find_by_id(statement_id) Statement.populate_overdue_from_invoices([statement_dao]) - statement_svc = Statement() - statement_svc._dao = statement_dao # pylint: disable=protected-access + from_date_string: str = statement_dao.from_date.strftime(DT_SHORT_FORMAT) + to_date_string: str = statement_dao.to_date.strftime(DT_SHORT_FORMAT) + statement_to_date = statement_dao.to_date - from_date_string: str = statement_svc.from_date.strftime(DT_SHORT_FORMAT) - to_date_string: str = statement_svc.to_date.strftime(DT_SHORT_FORMAT) + is_pdf = content_type == ContentType.PDF.value + extension = "pdf" if is_pdf else "csv" - extension = "pdf" if content_type == ContentType.PDF.value else "csv" - - if statement_svc.frequency == StatementFrequency.DAILY.value: + if statement_dao.frequency == StatementFrequency.DAILY.value: report_name = f"{report_name}-{from_date_string}.{extension}" else: report_name = f"{report_name}-{from_date_string}-to-{to_date_string}.{extension}" - statement_purchases = Statement.find_all_payments_and_invoices_for_statement(statement_id) + statement_purchases = Statement.find_all_payments_and_invoices_for_statement( + statement_id, is_pdf_statement=is_pdf, statement_to_date=statement_to_date + ) + if extension == "pdf": + db_summaries = Statement.get_totals_by_payment_method_from_db(statement_purchases, statement_to_date) statement_purchases = statement_purchases.all() - result_items = InvoiceSearch.create_payment_report_details(purchases=statement_purchases, data=None) + statement_purchases = [(invoice := row[0], setattr(invoice, "refund_id", row[1]))[0] for row in statement_purchases] + # Build statement summary for EFT/PAD + summary = Statement._build_statement_summary_for_methods(statement_dao, statement_purchases) + + report_response = InvoiceSearch.generate_statement_pdf_report( + invoices_orm=statement_purchases, + db_summaries=db_summaries, + statement=statement_dao, + statement_summary=summary, + report_name=report_name, + content_type=content_type, + auth=kwargs.get("auth", None), + ) else: - result_items = statement_purchases - statement = statement_svc.asdict() - statement["from_date"] = from_date_string - statement["to_date"] = to_date_string - - report_inputs = PaymentReportInput( - content_type=content_type, - report_name=report_name, - template_name=StatementTemplate.STATEMENT_REPORT.value, - results=result_items, - ) + statement_svc = Statement() + statement_svc._dao = statement_dao # pylint: disable=protected-access + statement_dict = statement_svc.asdict() + statement_dict["from_date"] = from_date_string + statement_dict["to_date"] = to_date_string - summary = Statement._build_statement_summary_for_methods(statement_dao, statement_purchases) - if summary: - report_inputs.statement_summary = summary + result_items = statement_purchases + report_inputs = PaymentReportInput( + content_type=content_type, + report_name=report_name, + template_name=StatementTemplate.STATEMENT_REPORT.value, + results=result_items, + ) + report_response = InvoiceSearch.generate_payment_report( + report_inputs, auth=kwargs.get("auth", None), statement=statement_dict + ) - report_response = InvoiceSearch.generate_payment_report( - report_inputs, auth=kwargs.get("auth", None), statement=statement - ) current_app.logger.debug(">get_statement_report") return report_response, report_name @@ -700,16 +718,211 @@ def generate_interim_statement(auth_account_id: str, new_frequency: str): @staticmethod def find_all_payments_and_invoices_for_statement( - statement_id: str, payment_method: PaymentMethod = None - ) -> list[InvoiceModel]: + statement_id: str, + payment_method: PaymentMethod = None, + is_pdf_statement: bool = False, + statement_to_date: datetime = None, + ) -> Query | list[InvoiceModel]: """Find all payment and invoices specific to a statement.""" query = ( db.session.query(InvoiceModel) .join(StatementInvoicesModel, StatementInvoicesModel.invoice_id == InvoiceModel.id) .filter(StatementInvoicesModel.statement_id == cast(statement_id, Integer)) - .order_by(InvoiceModel.id.asc()) ) if payment_method: query = query.filter(InvoiceModel.payment_method_code == payment_method.value) + if is_pdf_statement: + query = Statement._apply_partial_refunds_and_credits(query, statement_to_date) + return query + + @staticmethod + def get_totals_by_payment_method_from_db( + invoices_query, statement_to_date: datetime + ) -> SummariesGroupedByPaymentMethodDTO: + """Calculate payment method totals using database aggregation (no Python IDs, clean CTE).""" + paid_statuses = InvoiceStatus.paid_statuses() + invoice_ids_subq = invoices_query.with_entities(InvoiceModel.id).subquery() + inv = ( + db.session.query( + InvoiceModel.id, + InvoiceModel.payment_method_code, + InvoiceModel.total, + InvoiceModel.paid, + InvoiceModel.refund, + InvoiceModel.service_fees, + InvoiceModel.gst, + InvoiceModel.payment_date, + InvoiceModel.refund_date, + InvoiceModel.invoice_status_code, + ) + .join(invoice_ids_subq, invoice_ids_subq.c.id == InvoiceModel.id) + .filter(InvoiceModel.invoice_status_code != InvoiceStatus.CANCELLED.value) + .cte("inv") + ) + agg = ( + db.session.query( + inv.c.payment_method_code.label("payment_method_code"), + func.sum(inv.c.total).label("total"), + func.sum(inv.c.service_fees).label("service_fees"), + func.sum(inv.c.gst).label("gst"), + func.sum(inv.c.total - inv.c.service_fees - inv.c.gst).label("statutory_fee"), + func.sum( + case( + ( + and_( + inv.c.payment_date.isnot(None), + func.date(inv.c.payment_date) <= statement_to_date, + inv.c.invoice_status_code.in_(paid_statuses), + ), + inv.c.paid, + ), + else_=0, + ) + ).label("counted_paid"), + # Calculate whether refund should be counted + func.sum( + case( + ( + and_( + inv.c.refund > 0, + inv.c.paid != 0, + inv.c.refund_date.isnot(None), + func.date(inv.c.refund_date) <= statement_to_date, + ), + inv.c.refund, + ), + else_=0, + ) + ).label("counted_refund"), + func.sum( + case( + (func.date(AppliedCredits.created_on) <= statement_to_date, AppliedCredits.amount_applied), + else_=0, + ) + ).label("credits_applied"), + func.count(inv.c.id).label("invoice_id"), + ) + .outerjoin(AppliedCredits, AppliedCredits.invoice_id == inv.c.id) + .group_by(inv.c.payment_method_code) + .cte("agg") + ) + + result = ( + db.session.query( + agg.c.payment_method_code, + func.sum(agg.c.total).label(PaymentMethodSummaryRawDTO.TOTALS), + func.sum(agg.c.statutory_fee).label(PaymentMethodSummaryRawDTO.FEES), + func.sum(agg.c.service_fees).label(PaymentMethodSummaryRawDTO.SERVICE_FEES), + func.sum(agg.c.gst).label(PaymentMethodSummaryRawDTO.GST), + func.sum(agg.c.counted_paid).label(PaymentMethodSummaryRawDTO.PAID), + func.sum(agg.c.counted_refund).label(PaymentMethodSummaryRawDTO.COUNTED_REFUND), + func.sum(agg.c.credits_applied).label(PaymentMethodSummaryRawDTO.CREDITS_APPLIED), + func.count(agg.c.invoice_id).label(PaymentMethodSummaryRawDTO.INVOICE_COUNT), + ) + .group_by(agg.c.payment_method_code) + .all() + ) + + return SummariesGroupedByPaymentMethodDTO.from_db_result( + {row.payment_method_code: PaymentMethodSummaryRawDTO.from_db_row(row) for row in result} + ) + + @staticmethod + def _apply_partial_refunds_and_credits(query, statement_to_date: datetime = None): + """Apply partial refunds and credits subquery and computed status to the query.""" + partial_refund_subquery = ( + db.session.query(RefundsPartial.invoice_id, func.bool_or(RefundsPartial.is_credit).label("is_credit")) + .filter(RefundsPartial.status == RefundsPartialStatus.REFUND_PROCESSED.value) + .group_by(RefundsPartial.invoice_id) + .subquery() + ) + + refund_id, latest_refund_cte = Statement.build_refund_id_expr() + partial_refund_condition = and_( + InvoiceModel.invoice_status_code == InvoiceStatus.PAID.value, + InvoiceModel.refund != 0, + partial_refund_subquery.c.is_credit.isnot(None), + ) + + partial_refund_case = case( + (partial_refund_subquery.c.is_credit.is_(True), InvoiceStatus.PARTIALLY_CREDITED.value), + else_=InvoiceStatus.PARTIALLY_REFUNDED.value, + ) + + base_computed_status = case( + (partial_refund_condition, partial_refund_case), + else_=InvoiceModel.invoice_status_code, + ) + + if statement_to_date: + refund_statuses = InvoiceStatus.refund_statuses() + paid_statuses = InvoiceStatus.paid_statuses() + + computed_status = case( + ( + and_( + InvoiceModel.invoice_status_code.in_(refund_statuses), + InvoiceModel.refund_date.isnot(None), + func.date(InvoiceModel.refund_date) > statement_to_date, + ), + InvoiceStatus.PAID.value, + ), + ( + and_( + InvoiceModel.invoice_status_code.in_(paid_statuses), + InvoiceModel.payment_date.isnot(None), + func.date(InvoiceModel.payment_date) > statement_to_date, + ), + InvoiceStatus.APPROVED.value, + ), + (partial_refund_condition, partial_refund_case), + else_=InvoiceModel.invoice_status_code, + ) + else: + computed_status = base_computed_status + + return ( + query.outerjoin(partial_refund_subquery, InvoiceModel.id == partial_refund_subquery.c.invoice_id) + .outerjoin(latest_refund_cte, InvoiceModel.id == latest_refund_cte.c.invoice_id) + .options( + with_expression(InvoiceModel.invoice_status_code, computed_status), + subqueryload(InvoiceModel.applied_credits), + ) + .add_columns(refund_id.label("refund_id")) + ) + + @staticmethod + def _build__refund_cte(): + """Build CTE for refund per invoice.""" + refund_ranked = db.session.query( + RefundModel.invoice_id, + RefundModel.id.label("refund_id"), + func.row_number() + .over( + partition_by=RefundModel.invoice_id, + order_by=RefundModel.requested_date.desc(), + ) + .label("rn"), + ).cte("refund_ranked") + + return ( + db.session.query(refund_ranked.c.invoice_id, refund_ranked.c.refund_id) + .filter(refund_ranked.c.rn == 1) + .cte("latest_refund") + ) + + @staticmethod + def build_refund_id_expr(): + """Return (refund_id_case_expression, latest_refund_cte).""" + latest_refund_cte = Statement._build__refund_cte() + + refund_id = case( + ( + InvoiceModel.invoice_status_code.in_(InvoiceStatus.full_refund_statuses()), + latest_refund_cte.c.refund_id, + ) + ) + + return refund_id, latest_refund_cte diff --git a/pay-api/src/pay_api/templates/product_refund_client_notification.html b/pay-api/src/pay_api/templates/product_refund_client_notification.html index ee52aea66..28cb14f04 100644 --- a/pay-api/src/pay_api/templates/product_refund_client_notification.html +++ b/pay-api/src/pay_api/templates/product_refund_client_notification.html @@ -8,8 +8,7 @@ {% endif %} -A refund amount of ${{ '%.2f'|format(refundAmount) }} has been issued to you by {{ refundMethod }}. -The processing time will be 3-5 business days. +A refund amount of ${{ '%.2f'|format(refundAmount) }} has been issued. The processing time will be 3-5 business days. Reason for Refund: {{ reason }} diff --git a/pay-api/src/pay_api/utils/converter.py b/pay-api/src/pay_api/utils/converter.py index ab7bf4175..64550057f 100644 --- a/pay-api/src/pay_api/utils/converter.py +++ b/pay-api/src/pay_api/utils/converter.py @@ -1,7 +1,7 @@ """Converter module to support decimal and datetime serialization.""" import re -from datetime import datetime +from datetime import date, datetime from decimal import Decimal from enum import Enum from typing import Any @@ -11,6 +11,38 @@ from cattrs.gen import make_dict_structure_fn, make_dict_unstructure_fn, override +class CurrencyStr(str): + """Formatted currency string type.""" + + def __new__(cls, value): + """Create a new CurrencyStr instance.""" + if value is None: + value = "0.00" + elif isinstance(value, Decimal | int | float): + value = f"{value:.2f}" + else: + value = str(value) + return str.__new__(cls, value) + + +class FullMonthDateStr(str): + """Formatted date string type in '%B %d, %Y' format.""" + + def __new__(cls, value): + """Create a new FullMonthDateStr instance.""" + if value is None: + return None + if isinstance(value, datetime | date): + value = value.strftime("%B %d, %Y") + elif isinstance(value, str): + try: + dt = datetime.fromisoformat(value) + value = dt.strftime("%B %d, %Y") + except ValueError: + pass + return str.__new__(cls, value) + + class Converter(cattrs.Converter): """Addon to cattr converter.""" @@ -26,6 +58,10 @@ def __init__( self.register_structure_hook(Decimal, self._structure_decimal) self.register_unstructure_hook(Decimal, self._unstructure_decimal) self.register_unstructure_hook(datetime, self._unstructure_datetime) + self.register_structure_hook(CurrencyStr, self.structure_formatted_currency) + self.register_unstructure_hook(CurrencyStr, self._unstructure_formatted_currency) + self.register_structure_hook(FullMonthDateStr, self.structure_month_date_year_str) + self.register_unstructure_hook(FullMonthDateStr, self._unstructure_statement_date_str) # Note we may need a hook to handle str = None, sometimes a str set to None would become 'None' if enum_to_value: @@ -87,6 +123,10 @@ def _unstructure_decimal(obj: Decimal) -> float: def _unstructure_datetime(obj: datetime) -> str: return obj.isoformat() if obj else None + @staticmethod + def _structure_datetime(): + return lambda value, _: (datetime.fromisoformat(value) if isinstance(value, str) else value) + @staticmethod def remove_nones(data: dict[Any, Any]) -> dict[str, Any]: """Remove nones from payload.""" @@ -99,3 +139,28 @@ def remove_nones(data: dict[Any, Any]) -> dict[str, Any]: elif val is not None: new_data[key] = val return new_data + + @staticmethod + def structure_formatted_currency(obj: Any) -> CurrencyStr: + """Structure a value into a CurrencyStr.""" + return CurrencyStr(obj) + + @staticmethod + def _unstructure_formatted_currency(obj: CurrencyStr) -> str: + """Convert CurrencyStr to formatted string.""" + try: + return f"{float(obj):,.2f}" + except (TypeError, ValueError): + return "0.00" + + @staticmethod + def structure_month_date_year_str(obj: Any) -> FullMonthDateStr | None: + """Structure a value into a FullMonthDateStr.""" + if obj is None: + return None + return FullMonthDateStr(obj) + + @staticmethod + def _unstructure_statement_date_str(obj: FullMonthDateStr) -> str | None: + """FullMonthDateStr is already a formatted string.""" + return obj if obj else None diff --git a/pay-api/src/pay_api/utils/enums.py b/pay-api/src/pay_api/utils/enums.py index 34c8963c0..390a07c2a 100644 --- a/pay-api/src/pay_api/utils/enums.py +++ b/pay-api/src/pay_api/utils/enums.py @@ -66,6 +66,38 @@ class InvoiceStatus(Enum): # This status is not stored in the database but used in models/invoice.py COMPLETED = "COMPLETED" + @classmethod + def partial_fund_statuses(cls): + """Return list of partially refunded and credited statuses.""" + return [ + cls.PARTIALLY_CREDITED.value, + cls.PARTIALLY_REFUNDED.value, + ] + + @classmethod + def full_refund_statuses(cls): + """Return list of full refund related statuses.""" + return [ + cls.REFUNDED.value, + cls.CREDITED.value, + ] + + @classmethod + def refund_statuses(cls): + """Return list of refund-related statuses.""" + return [ + *cls.full_refund_statuses(), + *cls.partial_fund_statuses(), + ] + + @classmethod + def paid_statuses(cls): + """Return list of paid-related statuses (including refunded).""" + return [ + cls.PAID.value, + *cls.refund_statuses(), + ] + class TransactionStatus(Enum): """Transaction status codes.""" diff --git a/pay-api/src/pay_api/utils/statement_dtos.py b/pay-api/src/pay_api/utils/statement_dtos.py new file mode 100644 index 000000000..4f5fc2ff0 --- /dev/null +++ b/pay-api/src/pay_api/utils/statement_dtos.py @@ -0,0 +1,524 @@ +# Copyright © 2025 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Data Transfer Objects (DTOs) for PDF Statement generation. + +This module contains all DTOs used in the statement PDF generation process. +""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC001 TC003 +from decimal import Decimal # noqa: TC001 TC003 + +from attrs import define + +from pay_api.models.applied_credits import AppliedCreditsSearchModel +from pay_api.models.invoice import Invoice as InvoiceModel # noqa: TC001 +from pay_api.models.payment_line_item import PaymentLineItemSearchModel +from pay_api.models.statement import Statement # noqa: TC001 TC003 +from pay_api.utils.converter import CurrencyStr, FullMonthDateStr +from pay_api.utils.enums import InvoiceStatus, PaymentMethod, RefundsPartialType, StatementFrequency, StatementTitles +from pay_api.utils.serializable import Serializable + + +@define +class StatementTransactionDTO(Serializable): + """DTO for a single invoice transaction in PDF statement. + + Represents a formatted transaction row with all display-ready values. + """ + + invoice_id: int + products: list[str] + details: list[str] + folio: str + status_code: str + service_provided: bool + line_items: list[PaymentLineItemSearchModel] + created_on: FullMonthDateStr + fee: CurrencyStr + service_fee: CurrencyStr + gst: CurrencyStr + total: CurrencyStr + is_full_applied_credits: bool + applied_credits_amount: CurrencyStr + refund_total: CurrencyStr + refund_fee: CurrencyStr + refund_gst: CurrencyStr + refund_service_fee: CurrencyStr + payment_date: FullMonthDateStr | None + refund_date: FullMonthDateStr | None + refund_id: str | None = None + applied_credits: list[AppliedCreditsSearchModel] | None = None + + @staticmethod + def determine_service_provision_status(status_code: str, payment_method: str) -> bool: + """Determine if service was provided based on invoice status code and payment method.""" + default_statuses = { + InvoiceStatus.PAID.value, + InvoiceStatus.CANCELLED.value, + InvoiceStatus.CREDITED.value, + InvoiceStatus.REFUND_REQUESTED.value, + InvoiceStatus.REFUNDED.value, + InvoiceStatus.COMPLETED.value, + InvoiceStatus.PARTIALLY_CREDITED.value, + InvoiceStatus.PARTIALLY_REFUNDED.value, + } + + if status_code in default_statuses: + return True + + match payment_method: + case PaymentMethod.PAD.value: + return status_code in { + InvoiceStatus.APPROVED.value, + InvoiceStatus.SETTLEMENT_SCHEDULED.value, + } + + case PaymentMethod.EFT.value: + return status_code in { + InvoiceStatus.APPROVED.value, + InvoiceStatus.OVERDUE.value, + } + + case PaymentMethod.EJV.value: + return status_code in { + InvoiceStatus.APPROVED.value, + } + + case PaymentMethod.INTERNAL.value: + return status_code in { + InvoiceStatus.APPROVED.value, + } + + case _: + return False + + @staticmethod + def _compute_refund_lines_for_display(invoice: InvoiceModel, fee: Decimal): + """Compute refund lines for display in statement.""" + if invoice.invoice_status_code not in InvoiceStatus.refund_statuses(): + return 0, 0, 0, None + + if invoice.invoice_status_code in InvoiceStatus.partial_fund_statuses(): + base_partial = sum( + r.refund_amount for r in invoice.partial_refunds if r.refund_type == RefundsPartialType.BASE_FEES.value + ) + service_partial = sum( + r.refund_amount + for r in invoice.partial_refunds + if r.refund_type == RefundsPartialType.SERVICE_FEES.value + ) + gst_partial = invoice.refund - base_partial - service_partial + refund_id = ",".join(str(r.id) for r in invoice.partial_refunds) + return base_partial, gst_partial, service_partial, refund_id + + return fee, invoice.gst, invoice.service_fees, invoice.refund_id + + @classmethod + def from_orm( + cls, + invoice: InvoiceModel, + payment_method: str, + statement_to_date: datetime, + ) -> StatementTransactionDTO: + """Create DTO from ORM invoice object for PDF statement.""" + products = [item.description for item in invoice.payment_line_items] + details = [f"{d.get('label', '')} {d.get('value', '')}".strip() for d in (invoice.details or [])] + fee = invoice.total - invoice.service_fees - (invoice.gst or 0) + status_code = invoice.invoice_status_code + service_provided = cls.determine_service_provision_status(status_code, payment_method) + line_items = [PaymentLineItemSearchModel.from_row(item) for item in invoice.payment_line_items] + + applied_credits = None + applied_credits_amount = 0 + if invoice.applied_credits: + # Convert date to datetime for comparison + statement_to_datetime = datetime.combine(statement_to_date, datetime.max.time()) + filtered_credits = [c for c in invoice.applied_credits if c.created_on <= statement_to_datetime] + if filtered_credits: + applied_credits = [AppliedCreditsSearchModel.from_row(c) for c in filtered_credits] + applied_credits_amount = sum((c.amount_applied for c in filtered_credits), Decimal("0")) + + is_full_applied_credits = applied_credits_amount == invoice.total + + refund_fee, refund_gst, refund_service_fee, refund_id = cls._compute_refund_lines_for_display(invoice, fee) + + return cls( + invoice_id=invoice.id, + products=products, + details=details, + folio=invoice.folio_number or "-", + created_on=FullMonthDateStr(invoice.created_on), + fee=fee, + service_fee=invoice.service_fees, + gst=invoice.gst, + total=invoice.total, + status_code=status_code, + service_provided=service_provided, + line_items=line_items, + payment_date=FullMonthDateStr(invoice.payment_date), + refund_date=FullMonthDateStr(invoice.refund_date), + applied_credits=applied_credits, + applied_credits_amount=CurrencyStr(applied_credits_amount), + is_full_applied_credits=is_full_applied_credits, + refund_id=refund_id, + refund_fee=CurrencyStr(refund_fee), + refund_gst=CurrencyStr(refund_gst), + refund_service_fee=CurrencyStr(refund_service_fee), + refund_total=CurrencyStr(invoice.refund), + ) + + +@define +class PaymentMethodSummaryDTO(Serializable): + """DTO for payment method summary totals in PDF statement.""" + + totals: CurrencyStr + fees: CurrencyStr + service_fees: CurrencyStr + gst: CurrencyStr + paid: CurrencyStr + due: CurrencyStr + credits_applied: CurrencyStr | None = None + counted_refund: CurrencyStr | None = None + + @classmethod + def from_db_summary(cls, db_summary: PaymentMethodSummaryRawDTO) -> PaymentMethodSummaryDTO: + """Create from database aggregation summary for PDF statement.""" + if db_summary is None: + return cls( + totals=0.00, + fees=0.00, + service_fees=0.00, + gst=0.00, + paid=0.00, + due=0.00, + counted_refund=0.00, + credits_applied=0.00, + ) + return cls( + totals=db_summary.totals, + fees=db_summary.fees, + service_fees=db_summary.service_fees, + gst=db_summary.gst, + paid=db_summary.paid, + due=db_summary.due, + credits_applied=db_summary.credits_applied, + counted_refund=db_summary.counted_refund, + ) + + +@define +class PaymentMethodSummaryRawDTO(Serializable): + """DTO for raw payment method summary from database aggregation. + + This represents the raw numeric values from database queries, + before formatting for display. + """ + + # Field name constants - used by SQL queries to ensure consistency + TOTALS = "totals" + FEES = "fees" + SERVICE_FEES = "service_fees" + GST = "gst" + PAID = "paid" + COUNTED_REFUND = "counted_refund" + CREDITS_APPLIED = "credits_applied" + INVOICE_COUNT = "invoice_count" + + totals: Decimal + fees: Decimal + service_fees: Decimal + gst: Decimal + paid: Decimal + due: Decimal + credits_applied: Decimal + counted_refund: Decimal + invoice_count: int + + @classmethod + def from_db_row(cls, row) -> PaymentMethodSummaryRawDTO: + """Create from database query row.""" + totals = getattr(row, cls.TOTALS) + counted_refund = getattr(row, cls.COUNTED_REFUND) + credits_applied = getattr(row, cls.CREDITS_APPLIED) + paid = getattr(row, cls.PAID) + + totals_remaining_after_credits = max(totals - credits_applied, 0) + + refund_applied_to_total = min(counted_refund, totals_remaining_after_credits) + + net_total = totals - credits_applied - refund_applied_to_total + net_paid = paid - credits_applied - refund_applied_to_total + + return cls( + totals=net_total, + fees=getattr(row, cls.FEES), + service_fees=getattr(row, cls.SERVICE_FEES), + gst=getattr(row, cls.GST), + credits_applied=credits_applied, + counted_refund=counted_refund, + paid=net_paid, + due=totals - paid, + invoice_count=getattr(row, cls.INVOICE_COUNT), + ) + + +@define +class GroupedInvoicesDTO(Serializable): + """DTO for invoices grouped by payment method in PDF statement.""" + + payment_method: str + totals: CurrencyStr + fees: CurrencyStr + service_fees: CurrencyStr + gst: CurrencyStr + paid: CurrencyStr + due: CurrencyStr + transactions: list[StatementTransactionDTO] + is_index_0: bool + statement_header_text: str = None + include_service_provided: bool = False + credits_applied: CurrencyStr | None = None + counted_refund: CurrencyStr | None = None + # EFT-specific fields + amount_owing: CurrencyStr | None = None + latest_payment_date: str | None = None + due_date: FullMonthDateStr | None = None + # INTERNAL-specific fields + is_staff_payment: bool | None = None + + @classmethod + def from_invoices_and_summary( + cls, + payment_method: str, + invoices_orm: list[InvoiceModel], + db_summary: PaymentMethodSummaryRawDTO, + statement: Statement, + statement_summary: dict, + statement_to_date: datetime, + is_first: bool = False, + ) -> GroupedInvoicesDTO: + """Create DTO from ORM invoices and database summary for PDF statement.""" + transactions = [ + t + for t in (StatementTransactionDTO.from_orm(inv, payment_method, statement_to_date) for inv in invoices_orm) + if t.service_provided + ] + + summary = PaymentMethodSummaryDTO.from_db_summary(db_summary) + include_service_provided = any(t.service_provided for t in transactions) + + # Compute payment method specific fields before instantiation + # INTERNAL-specific: header text and staff payment flag + is_staff_payment = None + if payment_method == PaymentMethod.INTERNAL.value: + is_staff_payment = any(not hasattr(inv, "routing_slip") or inv.routing_slip is None for inv in invoices_orm) + statement_header_text = ( + StatementTitles.INTERNAL_STAFF.value if is_staff_payment else StatementTitles[payment_method].value + ) + else: + statement_header_text = StatementTitles[payment_method].value + + # EFT-specific: amount owing and dates + amount_owing = None + latest_payment_date = None + due_date = None + if payment_method == PaymentMethod.EFT.value: + amount_owing = statement.amount_owing + if statement.is_interim_statement and statement_summary: + latest_payment_date = statement_summary.get("latestStatementPaymentDate") + elif not statement.is_interim_statement and statement_summary: + due_date = FullMonthDateStr(statement_summary.get("dueDate")) + + return cls( + payment_method=payment_method, + totals=summary.totals, + fees=summary.fees, + service_fees=summary.service_fees, + gst=summary.gst, + paid=summary.paid, + due=summary.due + Decimal(statement_summary.get("balanceForward") or 0), + credits_applied=summary.credits_applied, + counted_refund=summary.counted_refund, + transactions=transactions, + is_index_0=is_first, + statement_header_text=statement_header_text, + include_service_provided=include_service_provided, + amount_owing=amount_owing, + latest_payment_date=latest_payment_date, + due_date=due_date, + is_staff_payment=is_staff_payment, + ) + + +@define +class StatementContextDTO(Serializable): + """DTO for statement metadata in PDF rendering.""" + + duration: str | None = None + amount_owing: CurrencyStr | None = None + from_date: FullMonthDateStr | None = None + to_date: FullMonthDateStr | None = None + created_on: FullMonthDateStr | None = None + frequency: str | None = None + # Store extra fields that don't have explicit attributes + id: int | None = None + is_interim_statement: bool | None = None + overdue_notification_date: str | None = None + notification_date: FullMonthDateStr | None = None + payment_methods: list[str] | None = None + statement_total: CurrencyStr | None = None + is_overdue: bool | None = None + + @staticmethod + def _compute_duration(from_date: str | None, to_date: str | None, frequency: str) -> str | None: + """Compute duration string based on dates and frequency.""" + if not from_date: + return None + + if frequency == StatementFrequency.DAILY.value: + return FullMonthDateStr(from_date) + + if to_date: + return f"{FullMonthDateStr(from_date)} - {FullMonthDateStr(to_date)}" + + return FullMonthDateStr(from_date) + + @classmethod + def from_statement(cls, statement) -> StatementContextDTO: + """Create DTO from Statement object.""" + if not statement: + return None + + duration = cls._compute_duration(statement.from_date, statement.to_date, statement.frequency or "") + + # Convert the comma-separated payment_methods string (e.g., "EFT,PAD") into a list. + # The DTO expects a list[str], and without this conversion the Converter in converter.py + # would treat the string as an iterable and split it into individual characters + payment_methods = None + if statement.payment_methods: + payment_methods = [m.strip() for m in statement.payment_methods.split(",") if m.strip()] + + # Directly assign formatted string values + return cls( + duration=duration, + amount_owing=statement.amount_owing, + from_date=FullMonthDateStr(statement.from_date), + to_date=FullMonthDateStr(statement.to_date), + created_on=FullMonthDateStr(statement.created_on), + frequency=statement.frequency or "", + id=statement.id, + is_interim_statement=statement.is_interim_statement, + overdue_notification_date=FullMonthDateStr(statement.overdue_notification_date), + notification_date=FullMonthDateStr(statement.notification_date), + payment_methods=payment_methods, + statement_total=statement.statement_total, + is_overdue=statement.is_overdue, + ) + + @classmethod + def from_dict(cls, statement: dict) -> StatementContextDTO: + """Create DTO from statement dictionary with formatting.""" + if not statement: + return None + + from_date = statement.get("from_date") + to_date = statement.get("to_date") + frequency = statement.get("frequency", "") + + duration = cls._compute_duration(from_date, to_date, frequency) + + return cls( + duration=duration, + amount_owing=statement.get("amount_owing"), + from_date=from_date, + to_date=to_date, + created_on=statement.get("created_on"), + frequency=frequency, + id=statement.get("id"), + is_interim_statement=statement.get("is_interim_statement"), + overdue_notification_date=statement.get("overdue_notification_date"), + notification_date=statement.get("notification_date"), + payment_methods=statement.get("payment_methods"), + statement_total=statement.get("statement_total"), + is_overdue=statement.get("is_overdue"), + ) + + +@define +class StatementSummaryDTO(Serializable): + """DTO for statement summary in PDF rendering.""" + + last_statement_total: CurrencyStr + last_statement_paid_amount: CurrencyStr + balance_forward: CurrencyStr + cancelled_transactions: CurrencyStr | None = None + latest_statement_payment_date: FullMonthDateStr | None = None + due_date: FullMonthDateStr | None = None + + @classmethod + def from_dict(cls, statement_summary: dict) -> StatementSummaryDTO: + """Create DTO from statement_summary dictionary with formatting.""" + if not statement_summary: + return None + + # Handle zero cancelled transactions - convert to None + cancelled_transactions = statement_summary.get("cancelledTransactions") + if cancelled_transactions == 0: + cancelled_transactions = None + + return cls( + last_statement_total=statement_summary.get("lastStatementTotal"), + last_statement_paid_amount=statement_summary.get("lastStatementPaidAmount"), + cancelled_transactions=cancelled_transactions, + latest_statement_payment_date=FullMonthDateStr(statement_summary.get("latestStatementPaymentDate")), + due_date=FullMonthDateStr(statement_summary.get("dueDate")), + balance_forward=statement_summary.get("balanceForward"), + ) + + +@define +class SummariesGroupedByPaymentMethodDTO(Serializable): + """DTO for payment method summaries from database aggregation. + + Key: payment_method (e.g., 'EFT', 'PAD', 'INTERNAL'). + """ + + summaries: dict[str, PaymentMethodSummaryRawDTO] + + @classmethod + def from_db_result(cls, db_summaries: dict[str, PaymentMethodSummaryRawDTO]) -> SummariesGroupedByPaymentMethodDTO: + """Create from database aggregation result.""" + return cls(summaries=db_summaries) + + def get_summary(self, payment_method: str) -> PaymentMethodSummaryRawDTO: + """Get summary DTO for a specific payment method.""" + return self.summaries.get(payment_method) + + def get_all_payment_methods(self) -> list[str]: + """Get list of all payment methods in summaries.""" + return list(self.summaries.keys()) + + +@define +class StatementPDFContextDTO(Serializable): + """DTO for complete PDF statement rendering context.""" + + statement_summary: StatementSummaryDTO | None + grouped_invoices: list[GroupedInvoicesDTO] + account: dict | None + statement: StatementContextDTO + has_payment_instructions: bool = False diff --git a/pay-api/tests/conftest.py b/pay-api/tests/conftest.py index 155868679..27870f853 100755 --- a/pay-api/tests/conftest.py +++ b/pay-api/tests/conftest.py @@ -93,7 +93,7 @@ def db(app): # pylint: disable=redefined-outer-name, invalid-name with app.app_context(): # Create worker-specific database c = app.config - initial_url = f"postgresql+pg8000://{c['DB_USER']}:{c['DB_PASSWORD']}@{c['DB_HOST']}:{c['DB_PORT']}/pay-test" + initial_url = f"postgresql+psycopg://{c['DB_USER']}:{c['DB_PASSWORD']}@{c['DB_HOST']}:{c['DB_PORT']}/pay-test" engine = create_engine(initial_url, isolation_level="AUTOCOMMIT") with engine.connect() as conn: conn.execute(text(f'DROP DATABASE IF EXISTS "{c["DB_NAME"]}"')) diff --git a/pay-api/tests/unit/api/test_eft_payment_actions.py b/pay-api/tests/unit/api/test_eft_payment_actions.py index f738100f5..45513087f 100755 --- a/pay-api/tests/unit/api/test_eft_payment_actions.py +++ b/pay-api/tests/unit/api/test_eft_payment_actions.py @@ -31,7 +31,7 @@ from pay_api.models import PaymentAccount as PaymentAccountModel from pay_api.models import Statement as StatementModel from pay_api.services import EftService, EFTShortNameLinkService -from pay_api.services import Statement as StatementService +from pay_api.services.statement import Statement from pay_api.utils.enums import ( DisbursementStatus, EFTCreditInvoiceStatus, @@ -355,7 +355,7 @@ def test_eft_reverse_payment_action(db, session, client, jwt, app, admin_users_m assert rv.status_code == 400 assert rv.json["type"] == Error.EFT_PAYMENT_ACTION_UNPAID_STATEMENT.name - invoices = StatementService.find_all_payments_and_invoices_for_statement(statement.id).all() + invoices = Statement.find_all_payments_and_invoices_for_statement(statement.id).all() invoices[0].invoice_status_code = InvoiceStatus.PAID.value invoices[0].payment_date = datetime.now(tz=UTC) - relativedelta(days=61) invoices[0].save() diff --git a/pay-api/tests/unit/api/test_payment.py b/pay-api/tests/unit/api/test_payment.py index ae876643c..12e03bae2 100755 --- a/pay-api/tests/unit/api/test_payment.py +++ b/pay-api/tests/unit/api/test_payment.py @@ -124,7 +124,7 @@ def test_eft_consolidated_payments(session, client, jwt, app): ) invoice_with_reference.save() factory_payment_line_item(invoice_id=invoice_with_reference.id, fee_schedule_id=1).save() - factory_invoice_reference(invoice_with_reference.id, invoice_number=invoice_with_reference).save() + factory_invoice_reference(invoice_with_reference.id, invoice_number=invoice_with_reference.id).save() invoice_without_reference = factory_invoice( payment_account, paid=0, total=100, status_code=InvoiceStatus.APPROVED.value diff --git a/pay-api/tests/unit/api/test_refund_approval.py b/pay-api/tests/unit/api/test_refund_approval.py index 095fcb0d4..583e5c11b 100644 --- a/pay-api/tests/unit/api/test_refund_approval.py +++ b/pay-api/tests/unit/api/test_refund_approval.py @@ -531,6 +531,15 @@ def test_partial_refund_approval_flow( assert refund_result["decisionDate"] is None assert refund_result["declineReason"] is None assert refund_result["refundAmount"] == refund_amount + assert refund_result["partialRefundLines"] + assert len(refund_result["partialRefundLines"]) == 1 + refund_line = refund_result["partialRefundLines"][0] + assert refund_line["paymentLineItemId"] == invoice.payment_line_items[0].id + assert refund_line["description"] == invoice.payment_line_items[0].description + assert refund_line["futureEffectiveFeeAmount"] == 0 + assert refund_line["priorityFeeAmount"] == 0 + assert refund_line["serviceFeeAmount"] == 0 + assert refund_line["statutoryFeeAmount"] == refund_amount rv = client.get(f"/api/v1/refunds?refundStatus={RefundStatus.PENDING_APPROVAL.value}", headers=requester_headers) assert rv.status_code == 200 diff --git a/pay-api/tests/unit/api/test_statement.py b/pay-api/tests/unit/api/test_statement.py index 6433eef54..7b89d17be 100755 --- a/pay-api/tests/unit/api/test_statement.py +++ b/pay-api/tests/unit/api/test_statement.py @@ -19,19 +19,26 @@ import json from datetime import UTC, datetime +from unittest.mock import patch from dateutil.relativedelta import relativedelta from pay_api.models import PaymentAccount +from pay_api.models.fee_schedule import FeeSchedule from pay_api.models.invoice import Invoice +from pay_api.services.report_service import ReportService from pay_api.utils.enums import ContentType, InvoiceStatus, PaymentMethod, StatementFrequency from tests.utilities.base_test import ( + factory_applied_credits, + factory_credit, factory_eft_credit, factory_eft_file, factory_eft_shortname, factory_eft_shortname_link, factory_invoice, factory_payment_account, + factory_payment_line_item, + factory_refunds_partial, factory_statement, factory_statement_invoices, factory_statement_settings, @@ -558,3 +565,115 @@ def test_statement_summary_multi_eft_under_payment(session, client, jwt, app): assert rv.json.get("totalDue") == invoice2.total assert rv.json.get("shortNameLinksCount") == 2 assert rv.json.get("isEftUnderPayment") is True + + +def test_statement_pad_totals_with_credits(session, client, jwt, app): + """Assert that PAD statement with credits shows creditsApplied and credited transactions.""" + token = jwt.create_jwt(get_claims(), token_header) + headers = { + "Authorization": f"Bearer {token}", + "content-type": "application/json", + "Accept": ContentType.PDF.value, + } + + rv = client.post( + "/api/v1/payment-requests", + data=json.dumps( + get_payment_request_with_payment_method( + business_identifier="CP0002000", + payment_method=PaymentMethod.PAD.value, + ) + ), + headers=headers, + ) + + invoice: Invoice = Invoice.find_by_id(rv.json.get("id")) + pay_account: PaymentAccount = PaymentAccount.find_by_id(invoice.payment_account_id) + + statement_from_date = datetime(2025, 12, 1, tzinfo=UTC) + statement_to_date = datetime(2025, 12, 31, tzinfo=UTC) + + invoice.paid = 50.00 + invoice.created_on = datetime(2025, 12, 5, tzinfo=UTC) + invoice.payment_date = datetime(2025, 12, 10, tzinfo=UTC) + invoice.invoice_status_code = "PAID" + invoice.save() + + credit = factory_credit( + account_id=pay_account.id, + cfs_identifier="TEST_CREDIT_PAD", + amount=10.00, + remaining_amount=10.00, + ) + + applied_credit = factory_applied_credits( + invoice_id=invoice.id, + credit_id=credit.id, + invoice_number=f"INV_{invoice.id}", + amount_applied=10.00, + invoice_amount=invoice.total, + cfs_identifier="TEST_CREDIT_PAD", + ) + + applied_credit.created_on = datetime(2025, 12, 6, tzinfo=UTC) + applied_credit.save() + + fee_schedule = FeeSchedule.find_by_filing_type_and_corp_type("CP", "OTANN") + line_item = factory_payment_line_item( + invoice_id=invoice.id, + fee_schedule_id=fee_schedule.fee_schedule_id, + ) + line_item.save() + + factory_refunds_partial( + invoice_id=invoice.id, + payment_line_item_id=line_item.id, + refund_amount=10.00, + created_on=datetime(2025, 12, 7, tzinfo=UTC), + ) + invoice.refund = 10.00 + invoice.refund_date = datetime(2025, 12, 7, tzinfo=UTC) + + settings_model = factory_statement_settings( + payment_account_id=pay_account.id, + frequency=StatementFrequency.MONTHLY.value, + from_date=statement_from_date, + ) + statement_model = factory_statement( + payment_account_id=pay_account.id, + frequency=StatementFrequency.MONTHLY.value, + payment_methods=PaymentMethod.PAD.value, + statement_settings_id=settings_model.id, + from_date=statement_from_date, + to_date=statement_to_date, + ) + factory_statement_invoices(statement_id=statement_model.id, invoice_id=invoice.id) + + with patch.object(ReportService, "get_report_response", return_value=None) as mock_report: + rv = client.get( + f"/api/v1/accounts/{pay_account.auth_account_id}/statements/{statement_model.id}", + headers=headers, + ) + + assert rv.status_code == 200 + + call_args = mock_report.call_args[0][0] + template_vars = call_args.template_vars + + assert len(template_vars["groupedInvoices"]) == 1 + + grouped_invoice = template_vars["groupedInvoices"][0] + + assert grouped_invoice["paymentMethod"] == PaymentMethod.PAD.value + + assert "creditsApplied" in grouped_invoice + assert grouped_invoice["creditsApplied"] == "10.00" + + assert "totals" in grouped_invoice + assert grouped_invoice["totals"] == "30.00" + + assert "paid" in grouped_invoice + assert grouped_invoice["paid"] == "30.00" + + assert "countedRefund" in grouped_invoice + assert grouped_invoice["countedRefund"] == "10.00" diff --git a/pay-api/tests/unit/api/test_transaction.py b/pay-api/tests/unit/api/test_transaction.py index 55b9cb747..7fa19bd7d 100755 --- a/pay-api/tests/unit/api/test_transaction.py +++ b/pay-api/tests/unit/api/test_transaction.py @@ -470,6 +470,7 @@ def test_transaction_post_for_nsf_payment(mock_payment_system_factory, mock_unlo mock_payment_system = mock_payment_system_factory.return_value mock_payment_system.get_receipt.return_value = ("12345", "2024-01-01", 100.00) mock_payment_system.get_pay_system_reason_code.return_value = None + mock_payment_system.get_payment_system_url_for_payment.return_value = "http://localhost:8080/pay-web" inv_number_1 = "REG00001" payment_account = factory_payment_account(payment_method_code=PaymentMethod.PAD.value) diff --git a/pay-api/tests/unit/services/test_payment.py b/pay-api/tests/unit/services/test_payment.py index 291b12d55..2b8c778b9 100644 --- a/pay-api/tests/unit/services/test_payment.py +++ b/pay-api/tests/unit/services/test_payment.py @@ -26,17 +26,16 @@ from pay_api.services.csv_service import CsvService from pay_api.services.invoice_search import InvoiceSearch from pay_api.services.payment import Payment as PaymentService -from pay_api.services.payment import PaymentReportInput -from pay_api.services.payment_calculations import ( - build_grouped_invoice_context, - build_statement_context, - build_statement_summary_context, - build_transaction_rows, - calculate_invoice_summaries, - determine_service_provision_status, -) from pay_api.utils.dataclasses import PurchaseHistorySearch -from pay_api.utils.enums import ContentType, InvoiceReferenceStatus, InvoiceStatus, PaymentMethod +from pay_api.utils.enums import InvoiceReferenceStatus, InvoiceStatus, PaymentMethod +from pay_api.utils.statement_dtos import ( + GroupedInvoicesDTO, + PaymentMethodSummaryDTO, + PaymentMethodSummaryRawDTO, + StatementContextDTO, + StatementSummaryDTO, + StatementTransactionDTO, +) from pay_api.utils.util import current_local_time from tests.utilities.base_test import ( factory_invoice, @@ -335,7 +334,7 @@ def test_search_payment_history_for_all(session): factory_invoice_reference(invoice.id).save() results = InvoiceSearch.search_all_purchase_history( - auth_account_id=auth_account_id, search_filter={}, content_type=ContentType.PDF.value + auth_account_id=auth_account_id, search_filter={}, query_only=False ) assert results is not None assert results.get("items") is not None @@ -368,7 +367,7 @@ def test_create_payment_report_csv(session): factory_payment_line_item(invoice_id=invoice.id, fee_schedule_id=1, description=f"Test Description {_i}").save() search_results = InvoiceSearch.search_all_purchase_history( - auth_account_id=auth_account_id, search_filter={}, content_type=ContentType.CSV.value + auth_account_id=auth_account_id, search_filter={}, query_only=True ) assert search_results is not None assert search_results.count() == 10 @@ -442,7 +441,7 @@ def test_csv_service_create_report(session): factory_payment_line_item(invoice_id=invoice.id, fee_schedule_id=1, description="Test Description").save() search_results = InvoiceSearch.search_all_purchase_history( - auth_account_id=auth_account_id, search_filter={}, content_type=ContentType.CSV.value + auth_account_id=auth_account_id, search_filter={}, query_only=True ) csv_data = CsvService.prepare_csv_data(search_results) @@ -904,140 +903,123 @@ def test_get_invoice_totals_for_statements(session): assert totals["due"] == 650 - 200 - 100 -def test_build_grouped_invoice_context_basic(): - """Test grouped invoices.""" - invoices = [ - { - "payment_method": PaymentMethod.EFT.value, - "paid": 100, - "total": 200, - "line_items": [], - "details": [], - "status_code": InvoiceStatus.PAID.value, - }, - { - "payment_method": PaymentMethod.CC.value, - "paid": 50, - "total": 50, - "line_items": [], - "details": [], - "status_code": InvoiceStatus.PAID.value, - }, - # FUTURE - Partial refunds - { - "payment_method": PaymentMethod.CC.value, - "paid": 20, - "refund": 10, - "total": 50, - "line_items": [], - "details": [], - "status_code": InvoiceStatus.PAID.value, - }, - ] - statement = {"amount_owing": 100, "to_date": "2024-06-01"} - summary = {"latestStatementPaymentDate": "2024-06-01", "dueDate": "2024-06-10"} - - grouped = build_grouped_invoice_context(invoices, statement, summary) - - assert any(item["payment_method"] == PaymentMethod.EFT.value for item in grouped) - assert any(item["payment_method"] == PaymentMethod.CC.value for item in grouped) - - eft_item = next(item for item in grouped if item["payment_method"] == PaymentMethod.EFT.value) - assert eft_item["total_paid"] == "100.00" - assert "transactions" in eft_item - - cc_item = next(item for item in grouped if item["payment_method"] == PaymentMethod.CC.value) - # 50 + 20 paid, 10 refund, total 2 invoices - assert cc_item["total_paid"] == "70.00" - # FUTURE - Partial refunds: check due/paid/total summary - assert "paid_summary" in cc_item - assert "due_summary" in cc_item - assert "totals_summary" in cc_item - # Partial refund: paid_summary/due_summary/total_summary basic check - assert float(cc_item["paid_summary"]) >= 0 - assert float(cc_item["totals_summary"]) >= 0 - - -def test_calculate_invoice_summaries(session): - """Test invoice summaries.""" +def test_statement_transaction_dto_from_orm(session): + """Test StatementTransactionDTO.from_orm creates correct DTO.""" + payment_account = factory_payment_account() + payment_account.save() + + invoice = factory_invoice( + payment_account, + payment_method_code=PaymentMethod.EFT.value, + status_code=InvoiceStatus.APPROVED.value, + total=100, + service_fees=10, + paid=0, + ) + invoice.save() + factory_payment_line_item(invoice_id=invoice.id, fee_schedule_id=1).save() + + statement_to_date = datetime.now(tz=UTC) + dto = StatementTransactionDTO.from_orm(invoice, PaymentMethod.EFT.value, statement_to_date) + + assert dto.invoice_id == invoice.id + assert dto.products == ["test"] + assert dto.folio == invoice.folio_number + assert dto.fee == 90 # 100 - 10 service_fees - 0 gst + assert dto.service_fee == 10 + assert dto.total == 100 + assert dto.status_code == "APPROVED" + assert dto.service_provided is True + assert len(dto.line_items) == 1 + + +def test_grouped_invoices_dto_from_invoices_and_summary(session): + """Test GroupedInvoicesDTO.from_invoices_and_summary creates correct DTO.""" payment_account = factory_payment_account() payment_account.save() invoice1 = factory_invoice( payment_account, - paid=0.00, - refund=100.00, - total=100.00, payment_method_code=PaymentMethod.EFT.value, - refund_date="2024-06-01", + status_code=InvoiceStatus.APPROVED.value, + total=100, + paid=0, ) invoice1.save() + factory_payment_line_item(invoice_id=invoice1.id, fee_schedule_id=1).save() invoice2 = factory_invoice( payment_account, - paid=100.00, - refund=0.00, - total=100.00, payment_method_code=PaymentMethod.EFT.value, - payment_date="2024-05-31", + status_code=InvoiceStatus.PAID.value, + total=50, + paid=50, ) invoice2.save() + factory_payment_line_item(invoice_id=invoice2.id, fee_schedule_id=1).save() + + db_summary = PaymentMethodSummaryRawDTO( + totals=150, + fees=150, + service_fees=0, + gst=0, + paid=50, + due=100, + invoice_count=2, + credits_applied=0, + counted_refund=0, + ) + + statement_to_date = datetime.now(tz=UTC) + statement = factory_statement( + payment_account_id=payment_account.id, + from_date=statement_to_date - timedelta(days=30), + to_date=statement_to_date, + ) + statement.amount_owing = 100 + statement_summary = {"dueDate": "2024-07-01"} + + dto = GroupedInvoicesDTO.from_invoices_and_summary( + payment_method=PaymentMethod.EFT.value, + invoices_orm=[invoice1, invoice2], + db_summary=db_summary, + statement=statement, + statement_summary=statement_summary, + statement_to_date=statement_to_date, + is_first=True, + ) + + assert dto.payment_method == PaymentMethod.EFT.value + assert dto.totals == 150.00 + assert dto.paid == 50.00 + assert dto.due == 100.00 + assert dto.is_index_0 is True + assert len(dto.transactions) == 2 + assert dto.include_service_provided is True + + +def test_statement_context_dto_from_dict(): + """Test StatementContextDTO.from_dict creates correct DTO.""" + statement = { + "from_date": "June 01, 2024", + "to_date": "June 30, 2024", + "frequency": "MONTHLY", + "amount_owing": 123.45, + "id": 1, + "is_interim_statement": False, + } - invoices = [ - { - "id": invoice1.id, - "payment_method": PaymentMethod.EFT.value, - "paid": 0, - "refund": 100, - "total": 100, - "refund_date": "2024-06-01", - }, - { - "id": invoice2.id, - "payment_method": PaymentMethod.EFT.value, - "paid": 100, - "refund": 0, - "total": 100, - "refund_date": None, - }, - ] - statement = {"to_date": "2024-06-01"} - summary = calculate_invoice_summaries(invoices, PaymentMethod.EFT.value, statement) - assert summary["paid_summary"] == 100.00 - assert summary["due_summary"] == 0.00 - assert summary["totals_summary"] == 100.00 - - -def test_build_transaction_rows(): - """Test transaction rows.""" - invoices = [ - { - "line_items": [{"description": "Service Fee"}], - "details": [{"label": "Folio", "value": "123"}], - "folio_number": "F123", - "created_on": datetime.now().isoformat(), - "total": 100, - "service_fees": 10, - "gst": 5, - "status_code": InvoiceStatus.PAID.value, - } - ] - rows = build_transaction_rows(invoices) - assert rows[0]["products"] == ["Service Fee"] - assert rows[0]["details"][0].startswith("Folio") - assert rows[0]["fee"] == "85.00" - - -def test_build_statement_context(): - """Test statement.""" - statement = {"from_date": "2024-06-01", "to_date": "2024-06-30", "frequency": "MONTHLY", "amount_owing": 123.45} - ctx = build_statement_context(statement) - assert "duration" in ctx - assert ctx["amount_owing"] == "123.45" - - -def test_build_statement_summary_context(): - """Test statement summary.""" + dto = StatementContextDTO.from_dict(statement) + + assert dto.duration == "June 01, 2024 - June 30, 2024" + assert dto.amount_owing == 123.45 + assert dto.frequency == "MONTHLY" + assert dto.id == 1 + assert dto.is_interim_statement is False + + +def test_statement_summary_dto_from_dict(): + """Test StatementSummaryDTO.from_dict creates correct DTO.""" summary = { "lastStatementTotal": 100, "lastStatementPaidAmount": 50, @@ -1045,12 +1027,14 @@ def test_build_statement_summary_context(): "latestStatementPaymentDate": "2024-06-01", "dueDate": "2024-06-10", } - ctx = build_statement_summary_context(summary) - assert ctx["lastStatementTotal"] == "100.00" - assert ctx["lastStatementPaidAmount"] == "50.00" - assert ctx["cancelledTransactions"] == "10.00" - assert "latestStatementPaymentDate" in ctx - assert "dueDate" in ctx + + dto = StatementSummaryDTO.from_dict(summary) + + assert dto.last_statement_total == 100 + assert dto.last_statement_paid_amount == 50 + assert dto.cancelled_transactions == 10 + assert dto.latest_statement_payment_date is not None + assert dto.due_date is not None @pytest.mark.parametrize( @@ -1075,304 +1059,277 @@ def test_build_statement_summary_context(): (InvoiceStatus.OVERDUE.value, PaymentMethod.PAD.value, False), (InvoiceStatus.SETTLEMENT_SCHEDULED.value, PaymentMethod.CC.value, False), (InvoiceStatus.PARTIAL.value, PaymentMethod.EFT.value, False), - ("settlement scheduled", PaymentMethod.PAD.value, True), - ("Settlement Scheduled", PaymentMethod.EFT.value, False), - ("approved", PaymentMethod.EJV.value, True), - ("overdue", PaymentMethod.EFT.value, True), - ("overdue", PaymentMethod.PAD.value, False), ], ) def test_determine_service_provision_status(status_code, payment_method, expected): """Test service provision status determination based on status and payment method.""" - assert determine_service_provision_status(status_code, payment_method) == expected + assert StatementTransactionDTO.determine_service_provision_status(status_code, payment_method) == expected + + +def test_payment_method_summary_raw_dto_from_db_row(): + """Test PaymentMethodSummaryRawDTO.from_db_row() calculates due correctly.""" + + class MockRow: + def __init__(self): + self.totals = 500.00 + self.fees = 450.00 + self.service_fees = 25.00 + self.gst = 25.00 + self.paid = 100.00 + self.counted_refund = 0.00 + self.invoice_count = 5 + self.credits_applied = 2.00 + + row = MockRow() + dto = PaymentMethodSummaryRawDTO.from_db_row(row) + + assert dto.totals == 498.00 + assert dto.fees == 450.00 + assert dto.service_fees == 25.00 + assert dto.gst == 25.00 + assert dto.paid == 98.00 + assert dto.due == 400.00 + assert dto.invoice_count == 5 + assert dto.credits_applied == 2.00 + + +def test_payment_method_summary_dto_from_db_summary(): + """Test PaymentMethodSummaryDTO.from_db_summary() formats currency correctly.""" + raw_dto = PaymentMethodSummaryRawDTO( + totals=500.00, + fees=450.00, + service_fees=25.00, + gst=25.00, + paid=100.00, + due=400.00, + credits_applied=0.00, + invoice_count=5, + counted_refund=0, + ) + dto = PaymentMethodSummaryDTO.from_db_summary(raw_dto) -def test_generate_payment_report_template_vars_structure(session, monkeypatch): - """Test that generate_payment_report creates correct templateVars structure.""" - payment_account = factory_payment_account().save() + assert dto.totals == 500.00 + assert dto.fees == 450.00 + assert dto.service_fees == 25.00 + assert dto.gst == 25.00 + assert dto.paid == 100.00 + assert dto.due == 400.00 - pad_invoice = factory_invoice( - payment_account, - status_code="SETTLEMENT_SCHED", - payment_method_code=PaymentMethod.PAD.value, - total=6.00, - service_fees=1.50, - business_identifier="SA5393", - corp_type_code="CSO", - created_name="PAUL ANTHONY", - details=[{"label": "View File", "value": "VIC-S-S-251093"}], - ).save() - - factory_payment_line_item( - invoice_id=pad_invoice.id, - fee_schedule_id=1, - filing_fees=4.50, - service_fees=1.50, - total=6.00, - description="View Supreme File", - ).save() - - cc_invoice = factory_invoice( - payment_account, - status_code="CREATED", - payment_method_code=PaymentMethod.CC.value, - total=30.00, - service_fees=0.00, - corp_type_code="BCR", - created_name="SYSTEM", - details=[], - ).save() - - factory_payment_line_item( - invoice_id=cc_invoice.id, - fee_schedule_id=1, - filing_fees=30.00, - service_fees=0.00, - total=30.00, - description="NSF", - ).save() - - factory_invoice_reference(pad_invoice.id, invoice_number="REG08145451").save() - factory_invoice_reference(cc_invoice.id, invoice_number="REG08172926").save() - - invoices = [pad_invoice, cc_invoice] - results = {"items": []} - - for invoice in invoices: - invoice_dict = { - "id": invoice.id, - "business_identifier": invoice.business_identifier, - "corp_type_code": invoice.corp_type_code, - "created_by": invoice.created_by, - "created_name": invoice.created_name, - "created_on": invoice.created_on.isoformat(), - "paid": float(invoice.paid or 0), - "refund": float(invoice.refund or 0), - "status_code": invoice.invoice_status_code, - "total": float(invoice.total), - "gst": float(invoice.gst or 0), - "service_fees": float(invoice.service_fees or 0), - "payment_method": invoice.payment_method_code, - "folio_number": invoice.folio_number, - "details": invoice.details or [], - "line_items": [], - } - - for line_item in invoice.payment_line_items: - invoice_dict["line_items"].append( - { - "total": float(line_item.total), - "pst": float(line_item.pst or 0), - "statutory_fees_gst": float(line_item.statutory_fees_gst or 0), - "service_fees_gst": float(line_item.service_fees_gst or 0), - "service_fees": float(line_item.service_fees or 0), - "description": line_item.description, - "filing_type_code": "CSBSRCH" if "View" in line_item.description else "NSF", - } - ) + dto_none = PaymentMethodSummaryDTO.from_db_summary(None) + assert dto_none.totals == 0 + assert dto_none.fees == 0 + assert dto_none.service_fees == 0 + assert dto_none.gst == 0 + assert dto_none.paid == 0 + assert dto_none.due == 0 - results["items"].append(invoice_dict) - statement = { - "from_date": "May 04, 2025", - "to_date": "May 10, 2025", - "is_overdue": False, - "payment_methods": ["PAD", "CC"], - "amount_owing": "0.00", - "statement_total": 36.0, - "id": 10289501, - "frequency": "WEEKLY", - "is_interim_statement": False, - } +def test_grouped_invoices_dto_with_internal_payment_method(session): + """Test GroupedInvoicesDTO handles INTERNAL payment method with staff payments.""" + payment_account = factory_payment_account() + payment_account.save() - auth_data = {"account": {"id": payment_account.auth_account_id, "name": "PAUL", "accountType": "PREMIUM"}} + invoice = factory_invoice( + payment_account, + payment_method_code=PaymentMethod.INTERNAL.value, + status_code=InvoiceStatus.APPROVED.value, + total=100, + paid=0, + ) + invoice.save() + factory_payment_line_item(invoice_id=invoice.id, fee_schedule_id=1).save() + + db_summary = PaymentMethodSummaryRawDTO( + totals=100, + fees=100, + service_fees=0, + gst=0, + paid=0, + due=100, + invoice_count=1, + credits_applied=0, + counted_refund=0, + ) - class MockUser: - """Mock user class.""" + statement = factory_statement( + payment_account_id=payment_account.id, + to_date="2024-06-01", + ) + statement_summary = {} + statement_to_date = datetime.now(tz=UTC) + + dto = GroupedInvoicesDTO.from_invoices_and_summary( + payment_method=PaymentMethod.INTERNAL.value, + invoices_orm=[invoice], + db_summary=db_summary, + statement=statement, + statement_summary=statement_summary, + statement_to_date=statement_to_date, + is_first=True, + ) - bearer_token = "mock_token" # noqa: S105 + assert dto.payment_method == PaymentMethod.INTERNAL.value + assert dto.is_staff_payment is True + assert "STAFF" in dto.statement_header_text - class MockResponse: - """Mock response class.""" - def json(self): - """Return mock contact data.""" - return { - "contacts": [ - { - "city": "Victoria", - "country": "CA", - "postalCode": "V8P2P2", - "region": "BC", - "street": "123 Main St", - } - ] - } +def test_grouped_invoices_dto_with_eft_interim_statement(session): + """Test GroupedInvoicesDTO handles EFT interim statement correctly.""" + payment_account = factory_payment_account() + payment_account.save() - monkeypatch.setattr("pay_api.services.oauth_service.OAuthService.get", lambda *args, **kwargs: MockResponse()) # noqa: ARG005 + invoice = factory_invoice( + payment_account, + payment_method_code=PaymentMethod.EFT.value, + status_code=InvoiceStatus.APPROVED.value, + total=100, + paid=0, + ) + invoice.save() + factory_payment_line_item(invoice_id=invoice.id, fee_schedule_id=1).save() + + db_summary = PaymentMethodSummaryRawDTO( + totals=100, + fees=100, + service_fees=0, + gst=0, + paid=0, + due=100, + invoice_count=1, + credits_applied=0, + counted_refund=0, + ) + + statement = factory_statement( + payment_account_id=payment_account.id, + to_date="2024-06-01", + is_interim_statement=True, + amount_owing=100, + ) + statement_summary = {"latestStatementPaymentDate": "2024-05-15"} + statement_to_date = datetime.now(tz=UTC) + + dto = GroupedInvoicesDTO.from_invoices_and_summary( + payment_method=PaymentMethod.EFT.value, + invoices_orm=[invoice], + db_summary=db_summary, + statement=statement, + statement_summary=statement_summary, + statement_to_date=statement_to_date, + is_first=True, + ) - captured_template_vars = {} + assert dto.payment_method == PaymentMethod.EFT.value + assert dto.amount_owing == 100 + assert dto.latest_payment_date == "2024-05-15" + assert dto.due_date is None # Not set for interim statements - def mock_get_report_response(request): - """Mock report service and capture template vars.""" - captured_template_vars.update(request.template_vars) - return "mock_report_response" - monkeypatch.setattr("pay_api.services.report_service.ReportService.get_report_response", mock_get_report_response) +def test_grouped_invoices_dto_with_eft_regular_statement(session): + """Test GroupedInvoicesDTO handles EFT regular statement with due date.""" + payment_account = factory_payment_account() + payment_account.save() - report_inputs = PaymentReportInput( - content_type="application/pdf", - report_name="test-statement.pdf", - template_name="statement_report", - results=results, - statement_summary=None, + invoice = factory_invoice( + payment_account, + payment_method_code=PaymentMethod.EFT.value, + status_code=InvoiceStatus.APPROVED.value, + total=100, + paid=0, + ) + invoice.save() + factory_payment_line_item(invoice_id=invoice.id, fee_schedule_id=1).save() + + db_summary = PaymentMethodSummaryRawDTO( + totals=100, + fees=100, + service_fees=0, + gst=0, + paid=0, + due=100, + invoice_count=1, + credits_applied=0, + counted_refund=0, ) - response = InvoiceSearch.generate_payment_report( - report_inputs, auth=auth_data, user=MockUser(), statement=statement + statement = factory_statement( + payment_account_id=payment_account.id, + to_date="2024-06-01", + is_interim_statement=False, + amount_owing=100, + ) + statement_summary = {"dueDate": "2024-07-01"} + statement_to_date = datetime.now(tz=UTC) + + dto = GroupedInvoicesDTO.from_invoices_and_summary( + payment_method=PaymentMethod.EFT.value, + invoices_orm=[invoice], + db_summary=db_summary, + statement=statement, + statement_summary=statement_summary, + statement_to_date=statement_to_date, + is_first=True, ) - assert response == "mock_report_response" - - assert "groupedInvoices" in captured_template_vars - assert "account" in captured_template_vars - assert "statement" in captured_template_vars - assert "total" in captured_template_vars - - grouped_invoices = captured_template_vars["groupedInvoices"] - assert len(grouped_invoices) == 2 # PAD and CC - - pad_group = next(g for g in grouped_invoices if g["payment_method"] == "PAD") - assert pad_group["total_paid"] == "0.00" - assert len(pad_group["transactions"]) == 1 - assert "ACCOUNT STATEMENT - PRE-AUTHORIZED DEBIT" in pad_group["statement_header_text"] - - cc_group = next(g for g in grouped_invoices if g["payment_method"] == "CC") - assert cc_group["total_paid"] == "0.00" - assert len(cc_group["transactions"]) == 1 - assert "ACCOUNT STATEMENT - CREDIT CARD" in cc_group["statement_header_text"] - - account = captured_template_vars["account"] - assert account["name"] == "PAUL" - assert account["id"] == payment_account.auth_account_id - assert "contact" in account - - statement_info = captured_template_vars["statement"] - assert statement_info["from_date"] == "May 04, 2025" - assert statement_info["to_date"] == "May 10, 2025" - - totals = captured_template_vars["total"] - assert "fees" in totals - assert "paid" in totals - assert "due" in totals - - -def test_build_grouped_invoice_context_with_additional_notes(): - """Test that grouped invoice context includes additional notes based on invoice statuses.""" - invoices = [ - { - "payment_method": PaymentMethod.PAD.value, - "paid": 0, - "total": 6, - "line_items": [{"description": "View Supreme File"}], - "details": [], - "status_code": InvoiceStatus.SETTLEMENT_SCHEDULED.value, - }, - { - "payment_method": PaymentMethod.PAD.value, - "paid": 0, - "total": 6, - "line_items": [{"description": "File Summary Report"}], - "details": [], - "status_code": InvoiceStatus.SETTLEMENT_SCHEDULED.value, - }, - { - "payment_method": PaymentMethod.CC.value, - "paid": 0, - "total": 30, - "line_items": [{"description": "NSF"}], - "details": [], - "status_code": InvoiceStatus.CREATED.value, - }, - { - "payment_method": PaymentMethod.EFT.value, - "paid": 100, - "total": 100, - "line_items": [{"description": "Business Registration"}], - "details": [], - "status_code": InvoiceStatus.PAID.value, - }, - { - "payment_method": PaymentMethod.EFT.value, - "paid": 0, - "total": 50, - "line_items": [{"description": "Name Change"}], - "details": [], - "status_code": InvoiceStatus.CANCELLED.value, - }, - ] - - statement = {"amount_owing": 0, "to_date": "2025-05-10"} - statement_summary = {} + assert dto.payment_method == PaymentMethod.EFT.value + assert dto.amount_owing == 100 + assert dto.latest_payment_date is None # Not set for regular statements + assert dto.due_date is not None # Should be formatted date + + +def test_statement_transaction_dto_with_multiple_line_items(session): + """Test StatementTransactionDTO handles multiple line items correctly.""" + payment_account = factory_payment_account() + payment_account.save() + + invoice = factory_invoice( + payment_account, + payment_method_code=PaymentMethod.EFT.value, + status_code=InvoiceStatus.APPROVED.value, + total=200, + service_fees=20, + paid=0, + ) + invoice.save() + + factory_payment_line_item(invoice_id=invoice.id, fee_schedule_id=1, description="Filing Fee", total=100).save() + factory_payment_line_item(invoice_id=invoice.id, fee_schedule_id=2, description="Service Fee", total=100).save() + + statement_to_date = datetime.now(tz=UTC) + dto = StatementTransactionDTO.from_orm(invoice, PaymentMethod.EFT.value, statement_to_date) + + assert len(dto.products) == 2 + assert "Filing Fee" in dto.products + assert "Service Fee" in dto.products + assert len(dto.line_items) == 2 + + +def test_statement_context_dto_with_daily_frequency(): + """Test StatementContextDTO handles DAILY frequency correctly.""" + statement = { + "from_date": "June 01, 2024", + "to_date": "June 01, 2024", + "frequency": "DAILY", + "amount_owing": 100, + } + + dto = StatementContextDTO.from_dict(statement) + + assert dto.duration == "June 01, 2024" + assert dto.frequency == "DAILY" + + +def test_statement_summary_dto_with_zero_cancelled_transactions(): + """Test StatementSummaryDTO handles zero cancelled transactions correctly.""" + summary = { + "lastStatementTotal": 100, + "lastStatementPaidAmount": 50, + "cancelledTransactions": 0, # Should be None in output + } + + dto = StatementSummaryDTO.from_dict(summary) - grouped = build_grouped_invoice_context(invoices, statement, statement_summary) - - # Should have 3 payment method groups: EFT, PAD, CC (in PaymentMethod.Order) - assert len(grouped) == 3 - - eft_group = next(g for g in grouped if g["payment_method"] == PaymentMethod.EFT.value) - pad_group = next(g for g in grouped if g["payment_method"] == PaymentMethod.PAD.value) - cc_group = next(g for g in grouped if g["payment_method"] == PaymentMethod.CC.value) - - assert "include_service_provided" in eft_group - assert eft_group["include_service_provided"] is True - - assert "include_service_provided" in pad_group - assert pad_group["include_service_provided"] is True - - assert "include_service_provided" in cc_group - assert cc_group["include_service_provided"] is False - - for group in grouped: - assert "include_service_provided" in group - assert isinstance(group["include_service_provided"], bool) - - for transaction in group["transactions"]: - assert "service_provided" in transaction - assert isinstance(transaction["service_provided"], bool) - - -def test_build_transaction_rows_includes_service_provided(): - """Test that build_transaction_rows includes service_provided for each transaction.""" - invoices = [ - { - "status_code": InvoiceStatus.PAID.value, - "line_items": [{"description": "Service 1"}], - "details": [], - "folio_number": "F001", - "created_on": "2025-05-07T00:00:00", - "total": 100, - "service_fees": 10, - "gst": 5, - }, - { - "status_code": InvoiceStatus.CANCELLED.value, - "line_items": [{"description": "Service 2"}], - "details": [], - "folio_number": "F002", - "created_on": "2025-05-08T00:00:00", - "total": 50, - "service_fees": 5, - "gst": 2.5, - }, - ] - - transactions = build_transaction_rows(invoices, PaymentMethod.PAD.value) - - assert len(transactions) == 2 - - assert transactions[0]["service_provided"] is True - assert "Service 1" in transactions[0]["products"][0] - - assert transactions[1]["service_provided"] is True - assert "(Cancelled) Service 2" in transactions[1]["products"][0] + assert dto.last_statement_total == 100 + assert dto.last_statement_paid_amount == 50 + assert dto.cancelled_transactions is None # Zero should become None diff --git a/pay-api/tests/unit/services/test_refund.py b/pay-api/tests/unit/services/test_refund.py index 28c4d8f43..3d8c8727f 100644 --- a/pay-api/tests/unit/services/test_refund.py +++ b/pay-api/tests/unit/services/test_refund.py @@ -150,6 +150,14 @@ def test_create_refund_for_paid_invoice( factory_receipt(invoice_id=i.id, receipt_number="1234569546456").save() mock_publish = Mock() mocker.patch("pay_api.services.gcp_queue.GcpQueue.publish", mock_publish) + + def mock_executor_submit(func): + """Mock executor submit to run synchronously in tests.""" + func() + return Mock() + + mocker.patch("pay_api.services.base_payment_system._executor.submit", side_effect=mock_executor_submit) + message = RefundService.create_refund(invoice_id=i.id, request={"reason": "Test"}, products=None) i = InvoiceModel.find_by_id(i.id) diff --git a/pay-api/tests/unit/services/test_statement.py b/pay-api/tests/unit/services/test_statement.py index a4314c85d..0dad4a849 100644 --- a/pay-api/tests/unit/services/test_statement.py +++ b/pay-api/tests/unit/services/test_statement.py @@ -19,7 +19,7 @@ from datetime import UTC, datetime, timedelta from decimal import Decimal -from unittest.mock import ANY, patch +from unittest.mock import patch import pytz from dateutil.relativedelta import relativedelta @@ -36,6 +36,7 @@ from pay_api.services.report_service import ReportRequest, ReportService from pay_api.services.statement import Statement as StatementService from pay_api.utils.constants import DT_SHORT_FORMAT +from pay_api.utils.converter import FullMonthDateStr from pay_api.utils.enums import ( ActivityAction, ContentType, @@ -44,7 +45,6 @@ StatementFrequency, StatementTemplate, ) -from pay_api.utils.util import get_statement_date_string from tests.utilities.base_test import ( factory_eft_shortname, factory_eft_shortname_link, @@ -586,7 +586,7 @@ def test_get_eft_statement_for_empty_invoices(session): ) assert report_name == expected_report_name - date_string_now = get_statement_date_string(datetime.now(tz=UTC)) + date_string_now = FullMonthDateStr(datetime.now(tz=UTC)) expected_template_vars = { "account": { "accountType": "PREMIUM", @@ -611,36 +611,29 @@ def test_get_eft_statement_for_empty_invoices(session): }, }, "groupedInvoices": [], + "hasPaymentInstructions": False, "statement": { - "amount_owing": "0.00", - "created_on": date_string_now, + "amountOwing": "0.00", + "createdOn": date_string_now, "frequency": "MONTHLY", - "from_date": get_statement_date_string(statement_from_date), - "to_date": get_statement_date_string(statement_to_date), + "fromDate": FullMonthDateStr(statement_from_date), + "toDate": FullMonthDateStr(statement_to_date), "id": statement_model.id, - "is_interim_statement": False, - "is_overdue": False, - "notification_date": None, - "overdue_notification_date": None, - "payment_methods": ["EFT"], - "statement_total": 0.0, - "duration": ( - f"{get_statement_date_string(statement_from_date)} - " - f"{get_statement_date_string(statement_to_date)}" - ), + "isInterimStatement": False, + "isOverdue": False, + "notificationDate": None, + "overdueNotificationDate": None, + "paymentMethods": ["EFT"], + "statementTotal": "0.00", + "duration": (f"{FullMonthDateStr(statement_from_date)} - " f"{FullMonthDateStr(statement_to_date)}"), }, "statementSummary": { - "dueDate": get_statement_date_string(StatementService.calculate_due_date(statement_to_date.date())), # pylint: disable=protected-access + "cancelledTransactions": None, + "dueDate": FullMonthDateStr(StatementService.calculate_due_date(statement_to_date.date())), # pylint: disable=protected-access "lastStatementTotal": "0.00", "lastStatementPaidAmount": "0.00", "latestStatementPaymentDate": None, - }, - "total": { - "due": "0.00", - "fees": "0.00", - "paid": "0.00", - "serviceFees": "0.00", - "statutoryFees": "0.00", + "balanceForward": "0.00", }, } expected_report_inputs = ReportRequest( @@ -802,7 +795,8 @@ def test_get_eft_statement_with_invoices(session): assert report_name == expected_report_name - date_string_now = get_statement_date_string(datetime.now(tz=UTC)) + date_string_now = FullMonthDateStr(datetime.now(tz=UTC)) + due_date_value = FullMonthDateStr(StatementService.calculate_due_date(statement_to_date.date())) # pylint: disable=protected-access expected_template_vars = { "account": { "accountType": "PREMIUM", @@ -828,237 +822,225 @@ def test_get_eft_statement_with_invoices(session): }, "groupedInvoices": [ { - "amount_owing": "400.00", - "credits_total": 0.0, - "due_date": get_statement_date_string( - StatementService.calculate_due_date(statement_to_date.date()) - ), # pylint: disable=protected-access - "due_summary": 450.0, - "fees_total": 468.75, - "gst_total": 6.25, - "include_service_provided": True, - "is_index_0": True, - "paid_summary": 50.0, - "payment_method": "EFT", - "refunds_total": 0.0, - "service_fees_total": 25.0, - "statement_header_text": "ACCOUNT STATEMENT - ELECTRONIC FUNDS TRANSFER", - "total_paid": "100.00", - "totals_summary": 500.0, + "amountOwing": "400.00", + "dueDate": due_date_value, + "due": "450.00", + "fees": "468.75", + "gst": "6.25", + "includeServiceProvided": True, + "isIndex0": True, + "isStaffPayment": None, + "latestPaymentDate": None, + "paid": "50.00", + "creditsApplied": "0.00", + "countedRefund": "0.00", + "paymentMethod": "EFT", + "serviceFees": "25.00", + "statementHeaderText": "ACCOUNT STATEMENT - ELECTRONIC FUNDS TRANSFER", + "totals": "500.00", "transactions": [ { - "bcol_account": "TEST", - "business_identifier": "CP0001234", - "corp_type_code": "CP", - "created_by": "test", - "created_name": "test name", - "created_on": ANY, + "invoiceId": invoice_1.id, + "isFullAppliedCredits": False, + "createdOn": FullMonthDateStr(datetime.now(UTC)), + "appliedCreditsAmount": "0.00", "details": ["label value"], "fee": "200.00", "folio": "1234567890", "gst": "0.00", - "id": invoice_1.id, - "invoice_number": "10021", - "line_items": [ + "lineItems": [ { "description": "test", - "filing_type_code": "OTANN", + "filingTypeCode": "OTANN", "gst": 0.0, "pst": 0.0, - "service_fees": 0.0, + "serviceFees": 0.0, + "serviceFeesGst": None, + "statutoryFeesGst": None, "total": 10.0, }, ], - "paid": 0.0, - "payment_account": { - "account_id": "1234", - "billable": True, - }, - "payment_method": "EFT", - "product": "BUSINESS", + "paymentDate": None, + "refundFee": "0.00", + "refundGst": "0.00", + "refundId": None, + "refundServiceFee": "0.00", + "refundTotal": "0.00", "products": ["test"], - "refund": 0.0, - "service_fee": "0.00", - "service_provided": False, - "status_code": "Invoice Approved", + "refundDate": None, + "serviceFee": "0.00", + "serviceProvided": True, + "statusCode": "APPROVED", "total": "200.00", + "appliedCredits": None, }, { - "bcol_account": "TEST", - "business_identifier": "CP0001234", - "corp_type_code": "CP", - "created_by": "test", - "created_name": "test name", - "created_on": ANY, + "invoiceId": invoice_2.id, + "isFullAppliedCredits": False, + "createdOn": FullMonthDateStr(datetime.now(UTC)), + "appliedCreditsAmount": "0.00", "details": ["label value"], "fee": "50.00", "folio": "1234567890", "gst": "0.00", - "id": invoice_2.id, - "invoice_number": "10021", - "line_items": [ + "lineItems": [ { "description": "test", - "filing_type_code": "OTANN", + "filingTypeCode": "OTANN", "gst": 0.0, "pst": 0.0, - "service_fees": 0.0, + "serviceFees": 0.0, + "serviceFeesGst": None, + "statutoryFeesGst": None, "total": 10.0, }, ], - "paid": 0.0, - "payment_account": {"account_id": "1234", "billable": True}, - "payment_method": "EFT", - "product": "BUSINESS", + "paymentDate": None, "products": ["test"], - "refund": 0.0, - "service_fee": "0.00", - "service_provided": False, - "status_code": "Invoice Approved", + "refundDate": None, + "refundFee": "0.00", + "refundGst": "0.00", + "refundId": None, + "refundServiceFee": "0.00", + "refundTotal": "0.00", + "serviceFee": "0.00", + "serviceProvided": True, + "statusCode": "APPROVED", "total": "50.00", + "appliedCredits": None, }, { - "bcol_account": "TEST", - "business_identifier": "CP0001234", - "corp_type_code": "CP", - "created_by": "test", - "created_name": "test name", - "created_on": ANY, + "invoiceId": invoice_3.id, + "isFullAppliedCredits": False, + "createdOn": FullMonthDateStr(datetime.now(UTC)), + "appliedCreditsAmount": "0.00", "details": ["label value"], "fee": "50.00", "folio": "1234567890", "gst": "0.00", - "id": invoice_3.id, - "invoice_number": "10021", - "line_items": [ + "lineItems": [ { "description": "test", - "filing_type_code": "OTANN", + "filingTypeCode": "OTANN", "gst": 0.0, "pst": 0.0, - "service_fees": 0.0, + "serviceFees": 0.0, + "serviceFeesGst": None, + "statutoryFeesGst": None, "total": 10.0, }, ], - "paid": 50.0, - "payment_account": {"account_id": "1234", "billable": True}, - "payment_date": datetime.strftime(invoice_3.payment_date, "%Y-%m-%dT%H:%M:%S.%f"), - "payment_method": "EFT", - "product": "BUSINESS", + "paymentDate": FullMonthDateStr(invoice_3.payment_date), "products": ["test"], - "refund": 0.0, - "service_fee": "0.00", - "service_provided": True, - "status_code": "COMPLETED", + "refundDate": None, + "refundFee": "0.00", + "refundGst": "0.00", + "refundId": None, + "refundServiceFee": "0.00", + "refundTotal": "0.00", + "serviceFee": "0.00", + "serviceProvided": True, + "statusCode": "APPROVED", "total": "50.00", + "appliedCredits": None, }, { - "bcol_account": "TEST", - "business_identifier": "CP0001234", - "corp_type_code": "CP", - "created_by": "test", - "created_name": "test name", - "created_on": ANY, + "invoiceId": invoice_4.id, + "isFullAppliedCredits": False, + "createdOn": FullMonthDateStr(datetime.now(UTC)), + "appliedCreditsAmount": "0.00", "details": ["label value"], "fee": "50.00", "folio": "1234567890", "gst": "0.00", - "id": invoice_4.id, - "invoice_number": "10021", - "line_items": [ + "lineItems": [ { "description": "test", - "filing_type_code": "OTANN", + "filingTypeCode": "OTANN", "gst": 0.0, "pst": 0.0, - "service_fees": 0.0, + "serviceFees": 0.0, + "serviceFeesGst": None, + "statutoryFeesGst": None, "total": 10.0, }, ], - "paid": 50.0, - "payment_account": {"account_id": "1234", "billable": True}, - "payment_date": datetime.strftime(invoice_4.payment_date, "%Y-%m-%dT%H:%M:%S"), - "payment_method": "EFT", - "product": "BUSINESS", + "paymentDate": FullMonthDateStr(invoice_4.payment_date), "products": ["test"], - "refund": 0.0, - "service_fee": "0.00", - "service_provided": True, - "status_code": "COMPLETED", + "refundDate": None, + "refundFee": "0.00", + "refundGst": "0.00", + "refundId": None, + "refundServiceFee": "0.00", + "refundTotal": "0.00", + "serviceFee": "0.00", + "serviceProvided": True, + "statusCode": "PAID", "total": "50.00", + "appliedCredits": None, }, { - "bcol_account": "TEST", - "business_identifier": "CP0001234", - "corp_type_code": "CP", - "created_by": "test", - "created_name": "test name", - "created_on": ANY, + "invoiceId": invoice_5.id, + "isFullAppliedCredits": False, + "createdOn": FullMonthDateStr(datetime.now(UTC)), + "appliedCreditsAmount": "0.00", "details": ["label value"], "fee": "118.75", "folio": "1234567890", "gst": "6.25", - "id": invoice_5.id, - "invoice_number": "10021", - "line_items": [ + "lineItems": [ { "description": "test", - "filing_type_code": "GSTTEST", + "filingTypeCode": "GSTTEST", "gst": 6.25, "pst": 0.0, - "service_fees": 25.0, - "service_fees_gst": 1.25, - "statutory_fees_gst": 5.0, + "serviceFees": 25.0, + "serviceFeesGst": 1.25, + "statutoryFeesGst": 5.0, "total": 150.0, }, ], - "paid": 0.0, - "payment_account": {"account_id": "1234", "billable": True}, - "payment_method": "EFT", - "product": "BUSINESS", + "paymentDate": None, + "refundFee": "0.00", + "refundGst": "0.00", + "refundId": None, + "refundServiceFee": "0.00", + "refundTotal": "0.00", "products": ["test"], - "refund": 0.0, - "service_fee": "25.00", - "service_provided": False, - "status_code": "Invoice Approved", + "refundDate": None, + "serviceFee": "25.00", + "serviceProvided": True, + "statusCode": "APPROVED", "total": "150.00", + "appliedCredits": None, }, ], } ], + "hasPaymentInstructions": True, "statement": { - "amount_owing": "400.00", - "created_on": date_string_now, - "duration": ( - f"{get_statement_date_string(statement_from_date)} - " - f"{get_statement_date_string(statement_to_date)}" - ), + "amountOwing": "400.00", + "createdOn": date_string_now, + "duration": (f"{FullMonthDateStr(statement_from_date)} - " f"{FullMonthDateStr(statement_to_date)}"), "frequency": "MONTHLY", - "from_date": get_statement_date_string(statement_from_date), - "to_date": get_statement_date_string(statement_to_date), + "fromDate": FullMonthDateStr(statement_from_date), + "toDate": FullMonthDateStr(statement_to_date), "id": statement_model.id, - "is_interim_statement": False, - "is_overdue": False, - "notification_date": None, - "overdue_notification_date": None, - "payment_methods": ["EFT"], - "statement_total": 500.0, + "isInterimStatement": False, + "isOverdue": False, + "notificationDate": None, + "overdueNotificationDate": None, + "paymentMethods": ["EFT"], + "statementTotal": "500.00", }, "statementSummary": { - "dueDate": get_statement_date_string(StatementService.calculate_due_date(statement_to_date.date())), # pylint: disable=protected-access + "cancelledTransactions": None, + "dueDate": due_date_value, "lastStatementTotal": "0.00", "lastStatementPaidAmount": "0.00", - "latestStatementPaymentDate": get_statement_date_string(invoice_3.payment_date.strftime("%Y-%m-%d")), - }, - # 2 are paid - looking with reference to the "statement", 1 is paid ($50) within the statement period - "total": { - "due": "450.00", - "fees": "500.00", - "paid": "50.00", - "serviceFees": "25.00", - "statutoryFees": "475.00", + "latestStatementPaymentDate": FullMonthDateStr(invoice_3.payment_date), + "balanceForward": "0.00", }, - "hasPaymentInstructions": True, } expected_report_inputs = ReportRequest( report_name=report_name, @@ -1069,7 +1051,28 @@ def test_get_eft_statement_with_invoices(session): stream=True, ) - mock_report.assert_called_with(expected_report_inputs) + call_args = mock_report.call_args[0][0] + + assert call_args.report_name == expected_report_inputs.report_name + assert call_args.template_name == expected_report_inputs.template_name + assert call_args.populate_page_number == expected_report_inputs.populate_page_number + assert call_args.content_type == expected_report_inputs.content_type + assert call_args.stream == expected_report_inputs.stream + + # Compare template_vars as dictionaries (order-independent) + expected_vars = expected_report_inputs.template_vars + actual_vars = call_args.template_vars + + # Sort transactions in grouped invoices for order-independent comparison + for grouped in expected_vars.get("groupedInvoices", []): + if "transactions" in grouped: + grouped["transactions"] = sorted(grouped["transactions"], key=lambda x: x.get("invoiceId", 0)) + + for grouped in actual_vars.get("groupedInvoices", []): + if "transactions" in grouped: + grouped["transactions"] = sorted(grouped["transactions"], key=lambda x: x.get("invoiceId", 0)) + + assert actual_vars == expected_vars def localize_date(date: datetime): diff --git a/pay-api/tests/utilities/base_test.py b/pay-api/tests/utilities/base_test.py index 78befca20..e204ef8e4 100644 --- a/pay-api/tests/utilities/base_test.py +++ b/pay-api/tests/utilities/base_test.py @@ -661,9 +661,11 @@ def factory_statement( statement_settings_id: str = None, created_on: datetime = datetime.now(tz=UTC), payment_methods: str = PaymentMethod.EFT.value, + is_interim_statement: bool = False, + amount_owing: float = None, ): """Return Factory.""" - return Statement( + stmt = Statement( frequency=frequency, statement_settings_id=statement_settings_id, payment_account_id=payment_account_id, @@ -671,8 +673,15 @@ def factory_statement( to_date=to_date, created_on=created_on, payment_methods=payment_methods, + is_interim_statement=is_interim_statement, ).save() + # Set amount_owing as an attribute (not a DB field, but used in schema) + if amount_owing is not None: + stmt.amount_owing = amount_owing + + return stmt + def factory_statement_invoices(statement_id: str, invoice_id: str): """Return Factory.""" diff --git a/pay-queue/poetry.lock b/pay-queue/poetry.lock index ff0ce2403..e12c7464e 100644 --- a/pay-queue/poetry.lock +++ b/pay-queue/poetry.lock @@ -156,18 +156,6 @@ typing-extensions = ">=4" [package.extras] tz = ["backports.zoneinfo ; python_version < \"3.9\""] -[[package]] -name = "asn1crypto" -version = "1.5.1" -description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, - {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, -] - [[package]] name = "attrs" version = "23.2.0" @@ -2101,9 +2089,9 @@ marshmallow = "3.21.1" marshmallow-sqlalchemy = "1.0.0" opentracing = "2.4.0" packaging = "24.0" -pg8000 = "^1.31.5" proto-plus = "1.23.0" protobuf = "4.25.8" +psycopg = "^3.3.1" psycopg2-binary = "2.9.9" pyasn1 = "0.5.1" pyasn1-modules = "0.3.0" @@ -2127,33 +2115,17 @@ structured-logging = {git = "https://github.com/bcgov/sbc-connect-common.git", r threadloop = "1.0.2" thrift = "0.16.0" tornado = "^6.5.1" -typing-extensions = "4.10.0" -urllib3 = "2.5.0" -werkzeug = "^3.0.3" +typing-extensions = "4.12.0" +urllib3 = "2.6.0" +werkzeug = "^3.1.4" [package.source] type = "git" -url = "https://github.com/ochiu/sbc-pay.git" -reference = "30357-Refund-Approval-Flow-2" -resolved_reference = "12733e1e54a2360598ca29a5ea86b42599f9019e" +url = "https://github.com/seeker25/sbc-pay.git" +reference = "fix_deadlock" +resolved_reference = "1775ca2cc4391d7cfea73b1a5d28649e7ad72cc5" subdirectory = "pay-api" -[[package]] -name = "pg8000" -version = "1.31.5" -description = "PostgreSQL interface library" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "pg8000-1.31.5-py3-none-any.whl", hash = "sha256:0af2c1926b153307639868d2ee5cef6cd3a7d07448e12736989b10e1d491e201"}, - {file = "pg8000-1.31.5.tar.gz", hash = "sha256:46ebb03be52b7a77c03c725c79da2ca281d6e8f59577ca66b17c9009618cae78"}, -] - -[package.dependencies] -python-dateutil = ">=2.8.2" -scramp = ">=1.4.5" - [[package]] name = "pluggy" version = "1.5.0" @@ -2317,6 +2289,30 @@ files = [ {file = "protobuf-4.25.8.tar.gz", hash = "sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd"}, ] +[[package]] +name = "psycopg" +version = "3.3.1" +description = "PostgreSQL database adapter for Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "psycopg-3.3.1-py3-none-any.whl", hash = "sha256:e44d8eae209752efe46318f36dd0fdf5863e928009338d736843bb1084f6435c"}, + {file = "psycopg-3.3.1.tar.gz", hash = "sha256:ccfa30b75874eef809c0fbbb176554a2640cc1735a612accc2e2396a92442fc6"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.6", markers = "python_version < \"3.13\""} +tzdata = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +binary = ["psycopg-binary (==3.3.1) ; implementation_name != \"pypy\""] +c = ["psycopg-c (==3.3.1) ; implementation_name != \"pypy\""] +dev = ["ast-comments (>=1.1.2)", "black (>=24.1.0)", "codespell (>=2.2)", "cython-lint (>=0.16)", "dnspython (>=2.1)", "flake8 (>=4.0)", "isort-psycopg", "isort[colors] (>=6.0)", "mypy (>=1.19.0)", "pre-commit (>=4.0.1)", "types-setuptools (>=57.4)", "types-shapely (>=2.0)", "wheel (>=0.37)"] +docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] +pool = ["psycopg-pool"] +test = ["anyio (>=4.0)", "mypy (>=1.19.0) ; implementation_name != \"pypy\"", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] + [[package]] name = "psycopg2-binary" version = "2.9.9" @@ -2782,21 +2778,6 @@ reference = "HEAD" resolved_reference = "d280cd1f9cba1ff1a24f5fb823ba089a8ad66e2a" subdirectory = "python" -[[package]] -name = "scramp" -version = "1.4.6" -description = "An implementation of the SCRAM protocol." -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "scramp-1.4.6-py3-none-any.whl", hash = "sha256:a0cf9d2b4624b69bac5432dd69fecfc55a542384fe73c3a23ed9b138cda484e1"}, - {file = "scramp-1.4.6.tar.gz", hash = "sha256:fe055ebbebf4397b9cb323fcc4b299f219cd1b03fd673ca40c97db04ac7d107e"}, -] - -[package.dependencies] -asn1crypto = ">=1.5.1" - [[package]] name = "semver" version = "3.0.2" @@ -3115,48 +3096,61 @@ files = [ [[package]] name = "typing-extensions" -version = "4.10.0" +version = "4.12.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, - {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, + {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, + {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, +] + +[[package]] +name = "tzdata" +version = "2025.2" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +groups = ["main"] +markers = "sys_platform == \"win32\"" +files = [ + {file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"}, + {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, ] [[package]] name = "urllib3" -version = "2.5.0" +version = "2.6.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, - {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, + {file = "urllib3-2.6.0-py3-none-any.whl", hash = "sha256:c90f7a39f716c572c4e3e58509581ebd83f9b59cced005b7db7ad2d22b0db99f"}, + {file = "urllib3-2.6.0.tar.gz", hash = "sha256:cb9bcef5a4b345d5da5d145dc3e30834f58e8018828cbc724d30b4cb7d4d49f1"}, ] [package.extras] -brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "werkzeug" -version = "3.1.3" +version = "3.1.4" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e"}, - {file = "werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746"}, + {file = "werkzeug-3.1.4-py3-none-any.whl", hash = "sha256:2ad50fb9ed09cc3af22c54698351027ace879a0b60a3b5edf5730b2f7d876905"}, + {file = "werkzeug-3.1.4.tar.gz", hash = "sha256:cd3cd98b1b92dc3b7b3995038826c68097dcb16f9baa63abe35f20eafeb9fe5e"}, ] [package.dependencies] -MarkupSafe = ">=2.1.1" +markupsafe = ">=2.1.1" [package.extras] watchdog = ["watchdog (>=2.3)"] @@ -3303,4 +3297,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "cf0b2bed2729ec7bfff8232f63b704e3390969641fcd2f4fc39a2dd20c97cee6" +content-hash = "6e22e50a2a49c9531f4c2e53666339860463d1800f0c3497a51b877678b00f51" diff --git a/pay-queue/pyproject.toml b/pay-queue/pyproject.toml index 793ba43b3..67730dd63 100644 --- a/pay-queue/pyproject.toml +++ b/pay-queue/pyproject.toml @@ -17,7 +17,7 @@ sqlalchemy = "^2.0.28" itsdangerous = "^2.1.2" launchdarkly-server-sdk = "^8.2.1" cachecontrol = "^0.14.0" -pay-api = { git = "https://github.com/ochiu/sbc-pay.git", branch = "30357-Refund-Approval-Flow-2", subdirectory = "pay-api" } +pay-api = { git = "https://github.com/seeker25/sbc-pay.git", branch = "fix_deadlock", subdirectory = "pay-api" } [tool.poetry.group.dev.dependencies] diff --git a/pay-queue/src/pay_queue/config.py b/pay-queue/src/pay_queue/config.py index b07ea50d0..e3bebdb70 100644 --- a/pay-queue/src/pay_queue/config.py +++ b/pay-queue/src/pay_queue/config.py @@ -76,10 +76,10 @@ class _Config: # pylint: disable=too-few-public-methods,protected-access DB_PORT = os.getenv("DATABASE_PORT", "5432") if DB_UNIX_SOCKET := os.getenv("DATABASE_UNIX_SOCKET", None): SQLALCHEMY_DATABASE_URI = ( - f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?unix_sock={DB_UNIX_SOCKET}/.s.PGSQL.5432" + f"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?host={DB_UNIX_SOCKET}&port={DB_PORT}" ) else: - SQLALCHEMY_DATABASE_URI = f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}" + SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}" # CFS API Settings CFS_BASE_URL = os.getenv("CFS_BASE_URL") @@ -158,7 +158,7 @@ class TestConfig(_Config): # pylint: disable=too-few-public-methods DB_PORT = os.getenv("DATABASE_TEST_PORT", "5432") SQLALCHEMY_DATABASE_URI = os.getenv( "DATABASE_TEST_URL", - default=f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}", + default=f"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}", ) USE_DOCKER_MOCK = os.getenv("USE_DOCKER_MOCK", None)