feat(cat-gateway): Catalyst signed docs simple upsert and select quer…

…ies templates (#1365)

* add integration tests setup for event-db queries

* add two queries, and integration test setup

* update insert_signed_documents query

* wip

* fix sqlfluff

* fix sqlfluff

* fix Earthfile

* fix

* remove unused old `test` target

* try

* try

* try

* setup poetry pyproject for event-db integration tests

* add pybars dependency

* add privileged mode for event-db test target

* make select query 2 into hbs template

* wip

* wip

* replace pybars with jinja2 templates

* update first select query

* fix spelling

.config/dictionaries/project.dic

@@ -216,6 +216,7 @@ projectcatalyst
 Prokhorenko
 proptest
 psql
+psycopg
 Ptarget
 pubkey
 PUBLICKEY

catalyst-gateway/event-db/Earthfile

@@ -4,6 +4,7 @@
 VERSION 0.8
 
 IMPORT github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.2.24 AS postgresql-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/python:v3.2.24 AS python-ci
 
 # cspell: words
 
@@ -41,24 +42,19 @@ local:
         SAVE IMAGE --push --insecure $local_registry/event-db:latest
     END
 
-
-# test the event db database schema
-#   CI target : true
-#test:
-#    FROM github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.9.2+postgres-base
-
-#    COPY github.com/input-output-hk/catalyst-ci/earthly/utils:v2.9.2+shell-assert/assert.sh .
-
-#    COPY ./docker-compose.yml .
-#    WITH DOCKER \
-#        --compose docker-compose.yml \
-#        --load event-db:latest=(+build --with_historic_data=false) \
-#        --service event-db \
-#        --allow-privileged
-#        RUN sleep 65;\
-#            res=$(psql postgresql://catalyst-event-dev:[email protected]:5432/CatalystEventDev -c "SELECT COUNT(*) FROM event");\
-
-#            source assert.sh;\
-#            expected=$(printf " count \n-------\n     5\n(1 row)");\
-#            assert_eq "$expected" "$res"
-#    END
+# Run the queries_tests.py
+test:
+    FROM python-ci+python-base
+
+    DO python-ci+BUILDER
+    COPY --dir tests .
+    COPY --dir queries .
+
+    WITH DOCKER \
+        --compose "./tests/docker-compose.yml" \
+        --load event-db:latest=+build \
+        --pull alpine:3.20.3 \
+        --service event-db-is-running \
+        --allow-privileged
+        RUN poetry run pytest -s -m ci
+    END

catalyst-gateway/event-db/blueprint.cue

@@ -1,2 +1,9 @@
 version: "1.0.0"
-project: name: "gateway-event-db"
+project: {
+  name: "gateway-event-db"
+  ci: {
+    targets: {
+      test: privileged: true
+    }
+  }
+}

catalyst-gateway/event-db/pyproject.toml

@@ -0,0 +1,23 @@
+# cspell: words bitcoinlib
+
+[tool.poetry]
+name = "tests"
+version = "0.1.0"
+description = ""
+authors = []
+readme = "Readme.md"
+license = "MIT or Apache-2.0"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+loguru = "^0.7.2"
+pytest = "^8.0.0"
+psycopg = "3.2.3"
+psycopg-binary = "^3.2.3"
+jinja2 = "^3.1.4"
+
+[tool.pytest.ini_options]
+markers = [
+    "ci: marks tests to be run in ci",
+    "nightly: marks tests to be run nightly",
+]

catalyst-gateway/event-db/queries/insert_signed_documents.sql

@@ -0,0 +1,22 @@
+INSERT INTO signed_docs
+(
+  id,
+  ver,
+  type,
+  author,
+  metadata,
+  payload,
+  raw
+)
+VALUES
+($1, $2, $3, $4, $5, $6, $7)
+ON CONFLICT (id, ver) DO UPDATE
+SET
+type = signed_docs.type
+
+WHERE
+signed_docs.type = excluded.type
+AND signed_docs.author = excluded.author
+AND signed_docs.metadata = excluded.metadata
+AND signed_docs.payload = excluded.payload
+AND signed_docs.raw = excluded.raw

catalyst-gateway/event-db/queries/select_signed_documents.sql.jinja

@@ -0,0 +1,12 @@
+SELECT
+  signed_docs.type,
+  signed_docs.author,
+  signed_docs.metadata,
+  signed_docs.payload,
+  signed_docs.raw
+FROM signed_docs
+WHERE
+  signed_docs.id = '{{ id }}'
+  {% if ver %} AND signed_docs.ver = '{{ ver }}' {% endif %}
+ORDER BY signed_docs.ver DESC
+LIMIT 1

catalyst-gateway/event-db/queries/select_signed_documents_2.sql.jinja

@@ -0,0 +1,12 @@
+SELECT
+  signed_docs.id,
+  signed_docs.ver,
+  signed_docs.type,
+  signed_docs.author,
+  signed_docs.metadata
+FROM signed_docs
+WHERE
+  {{ conditions }}
+ORDER BY signed_docs.type DESC, signed_docs.id DESC, signed_docs.ver DESC
+{% if limit %} LIMIT {{ limit }} {% endif %}
+{% if offset %} OFFSET {{ offset }} {% endif %}

catalyst-gateway/event-db/tests/docker-compose.yml

@@ -0,0 +1,30 @@
+services:
+  event-db:
+    image: event-db:latest
+    environment:
+      - DB_HOST=localhost
+      - DB_PORT=5432
+      - DB_NAME=CatalystEventDev
+      - DB_DESCRIPTION="Catalyst Event DB"
+      - DB_SUPERUSER=postgres
+      - DB_SUPERUSER_PASSWORD=postgres
+      - DB_USER=catalyst-event-dev
+      - DB_USER_PASSWORD=CHANGE_ME
+
+      - INIT_AND_DROP_DB=true
+      - WITH_MIGRATIONS=true
+    ports:
+      - 5432:5432
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${DB_SUPERUSER} -d $${DB_SUPERUSER_PASSWORD}"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+# it is a helper service to wait until the event-db will be ready
+# mainly its a trick for Earthly how to wait until service will be fully functional
+  event-db-is-running:
+    image: alpine:3.20.3
+    depends_on:
+      event-db:
+        condition: service_healthy

catalyst-gateway/event-db/tests/test_signed_docs_queries.py

@@ -0,0 +1,147 @@
+import psycopg
+import pytest
+import jinja2
+
+
+jinja_env = jinja2.Environment(
+    loader=jinja2.FileSystemLoader("./queries/"),
+)
+EVENT_DB_URL = "postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev"
+
+
+class SignedData:
+    def __init__(
+        self,
+        id: str,
+        ver: str,
+        doc_type: str,
+        author: str,
+        metadata: str,
+        payload: str,
+        raw: bytes,
+    ):
+        self.id = id
+        self.ver = ver
+        self.doc_type = doc_type
+        self.author = author
+        self.metadata = metadata
+        self.payload = payload
+        self.raw = raw
+
+
+@pytest.mark.ci
+def test_signed_docs_queries():
+    with psycopg.connect(EVENT_DB_URL) as conn:
+        docs = [
+            SignedData(
+                id="3764e30b-9bb5-4a34-906e-f4de1845e8bf",
+                ver="299255cb-9f53-46ec-8e24-4360b9d374bd",
+                doc_type="c17f59b2-1304-4a75-923e-00795add70af",
+                author="Alex",
+                metadata="{}",
+                payload="{}",
+                raw=b"bytes1",
+            ),
+            SignedData(
+                id="4ee05138-e85b-48b4-91c2-2a45a74a0a82",
+                ver="3cd78d6c-9388-47ab-9a65-f3a84c76190f",
+                doc_type="fec1b996-ad89-4fab-a787-6568f42b00b1",
+                author="Steven",
+                metadata="{}",
+                payload="{}",
+                raw=b"bytes2",
+            ),
+        ]
+
+        insert_signed_documents_query(conn, docs)
+        # try insert the same values
+        insert_signed_documents_query(conn, docs)
+        select_signed_documents_query(conn, docs)
+        select_signed_documents_2_query(conn, docs)
+
+        # try insert the same id and ver, but with the different other data
+        docs[0].author = "Sasha"
+        docs[1].author = "Sasha"
+        should_panic(
+            lambda: insert_signed_documents_query(conn, docs),
+            "insert_signed_documents_query should fail",
+        )
+        should_panic(
+            lambda: select_signed_documents_query(conn, docs),
+            "select_signed_documents_query should fail",
+        )
+        should_panic(
+            lambda: select_signed_documents_2_query(conn, docs),
+            "select_signed_documents_2_query should fail",
+        )
+
+
+def insert_signed_documents_query(conn, docs: [SignedData]):
+    sql_stmt = open("./queries/insert_signed_documents.sql", "r").read()
+    sql_stmt = (
+        sql_stmt.replace("$1", "%s")
+        .replace("$2", "%s")
+        .replace("$3", "%s")
+        .replace("$4", "%s")
+        .replace("$5", "%s")
+        .replace("$6", "%s")
+        .replace("$7", "%s")
+    )
+    for doc in docs:
+        conn.execute(
+            sql_stmt,
+            (
+                doc.id,
+                doc.ver,
+                doc.doc_type,
+                doc.author,
+                doc.metadata,
+                doc.payload,
+                doc.raw,
+            ),
+        )
+
+
+def select_signed_documents_query(conn, docs: [SignedData]):
+    template = jinja_env.get_template("select_signed_documents.sql.jinja")
+    for doc in docs:
+        sql_stmt = template.render(
+            {
+                "id": doc.id,
+                "ver": doc.ver,
+            }
+        )
+        cur = conn.execute(sql_stmt)
+        (doc_type, author, metadata, payload, raw) = cur.fetchone()
+        assert str(doc_type) == doc.doc_type
+        assert author == doc.author
+        assert str(metadata) == doc.metadata
+        assert str(payload) == doc.payload
+        assert raw == doc.raw
+
+
+def select_signed_documents_2_query(conn, docs: [SignedData]):
+    template = jinja_env.get_template("select_signed_documents_2.sql.jinja")
+    for doc in docs:
+        sql_stmt = template.render(
+            {
+                "conditions": f"signed_docs.id = '{doc.id}' AND signed_docs.ver = '{doc.ver}'",
+                "limit": 1,
+                "offset": 0,
+            }
+        )
+        cur = conn.execute(sql_stmt)
+        (id, ver, doc_type, author, metadata) = cur.fetchone()
+        assert str(id) == doc.id
+        assert str(ver) == doc.ver
+        assert str(doc_type) == doc.doc_type
+        assert author == doc.author
+        assert str(metadata) == doc.metadata
+
+
+def should_panic(func, msg: str):
+    try:
+        func()
+        assert False, msg
+    except:
+        pass