feat: add e2e tests

.github/workflows/test-pr-e2e.yml

@@ -92,17 +92,49 @@ jobs:
       - name: Wait for database to be ready
         run: |
           # Add commands to wait for the database to be ready
+          echo "Waiting for Database to be ready..."
           if [ "${{ matrix.db_type }}" == "mysql" ]; then
-            until docker exec $(docker ps -qf "name=keep-database") mysqladmin ping -h "localhost" --silent; do
+            until docker exec $(docker ps -qf "name=keep-database-1") mysqladmin ping -h "localhost" --silent; do
               echo "Waiting for MySQL to be ready..."
               sleep 2
             done
           elif [ "${{ matrix.db_type }}" == "postgres" ]; then
-            until docker exec $(docker ps -qf "name=keep-database") pg_isready -h localhost -U keepuser; do
+            until docker exec $(docker ps -qf "name=keep-database-1") pg_isready -h localhost -U keepuser; do
               echo "Waiting for Postgres to be ready..."
               sleep 2
             done
           fi
+          echo "Database is ready!"
+
+          echo "Waiting for Database (DB AUTH) to be ready..."
+          if [ "${{ matrix.db_type }}" == "mysql" ]; then
+            until docker exec $(docker ps -qf "name=keep-database-db-auth-1") mysqladmin ping -h "localhost" --silent; do
+              echo "Waiting for MySQL (DB AUTH) to be ready..."
+              sleep 3
+            done
+          elif [ "${{ matrix.db_type }}" == "postgres" ]; then
+            until docker exec $(docker ps -qf "name=keep-database-db-auth-1") pg_isready -h localhost -U keepuser; do
+              echo "Waiting for Postgres (DB AUTH) to be ready..."
+              sleep 2
+            done
+          fi      
+          echo "Database (DB AUTH) is ready!"
+
+          attempt=0
+          max_attempts=10
+          echo "Waiting for Keep backend (DB AUTH) to be ready..."
+          until $(curl --output /dev/null --silent --fail http://localhost:8081/healthcheck); do
+            if [ "$attempt" -ge "$max_attempts" ]; then
+              echo "Max attempts reached, exiting... Sometimes Keep can't start because of double-headed migrations, use: 'alembic -c keep/alembic.ini history' to investigate, or check artifacts."
+              exit 1
+            fi
+            echo "Waiting for Keep backend (DB AUTH) to be ready... (Attempt: $((attempt+1)))"
+            attempt=$((attempt+1))
+            sleep 4
+          done
+          echo "Keep backend (DB AUTH) is ready!"
+          # wait to the backend
+
 
           # wait to keep backend on port 8080
           echo "Waiting for Keep backend to be ready..."
@@ -116,11 +148,11 @@ jobs:
             fi
             echo "Waiting for Keep backend to be ready... (Attempt: $((attempt+1)))"
             attempt=$((attempt+1))
-            sleep 2
+            sleep 4
           done
-
           echo "Keep backend is ready!"
-          # wait to the backend
+
+
           echo "Waiting for Keep frontend to be ready..."
           attempt=0
           max_attempts=10
@@ -134,6 +166,36 @@ jobs:
             attempt=$((attempt+1))
             sleep 2
           done
+          echo "Keep frontend is ready"
+
+          echo "Waiting for Keep frontend (DB AUTH) to be ready..."
+          attempt=0
+          max_attempts=10
+          until $(curl --output /dev/null --silent --fail http://localhost:3001/); do
+            if [ "$attempt" -ge "$max_attempts" ]; then
+              echo "Max attempts reached, exiting..."
+              exit 1
+            fi
+            echo "Waiting for Keep frontend (DB AUTH) to be ready... (Attempt: $((attempt+1)))"
+            attempt=$((attempt+1))
+            sleep 2
+          done
+          echo "Keep frontend (DB AUTH) is ready"
+
+
+          echo "Waiting for Grafana to be ready..."
+          attempt=0
+          max_attempts=10
+          until $(curl --output /dev/null --silent --fail http://localhost:3002/api/health); do
+            if [ "$attempt" -ge "$max_attempts" ]; then
+              echo "Max attempts reached, exiting... "
+              exit 1
+            fi
+            echo "Waiting for Grafana to be ready... (Attempt: $((attempt+1)))"
+            attempt=$((attempt+1))
+            sleep 2
+          done
+          echo "Grafana is ready..."
 
           # create the state directory
           # mkdir -p ./state && chown -R root:root ./state && chmod -R 777 ./state
@@ -157,6 +219,8 @@ jobs:
         run: |
           docker compose --project-directory . -f tests/e2e_tests/docker-compose-e2e-${{ matrix.db_type }}.yml logs keep-backend > backend_logs-${{ matrix.db_type }}.txt
           docker compose --project-directory . -f tests/e2e_tests/docker-compose-e2e-${{ matrix.db_type }}.yml logs keep-frontend > frontend_logs-${{ matrix.db_type }}.txt
+          docker compose --project-directory . -f tests/e2e_tests/docker-compose-e2e-${{ matrix.db_type }}.yml logs keep-backend-db-auth > backend_logs-${{ matrix.db_type }}-db-auth.txt
+          docker compose --project-directory . -f tests/e2e_tests/docker-compose-e2e-${{ matrix.db_type }}.yml logs keep-frontend-db-auth > frontend_logs-${{ matrix.db_type }}-db-auth.txt
         continue-on-error: true
 
       - name: Upload test artifacts on failure
@@ -169,6 +233,8 @@ jobs:
             playwright_dump_*.png
             backend_logs-${{ matrix.db_type }}.txt
             frontend_logs-${{ matrix.db_type }}.txt
+            backend_logs-${{ matrix.db_type }}-db-auth.txt
+            frontend_logs-${{ matrix.db_type }}-db-auth.txt
         continue-on-error: true
 
       - name: Tear down environment

tests/e2e_tests/docker-compose-e2e-mysql.yml

@@ -1,4 +1,6 @@
 services:
+  ## Keep Services with NO_AUTH
+  # Database Service
   keep-database:
     image: mysql:latest
     environment:
@@ -14,6 +16,7 @@ services:
       timeout: 5s
       retries: 5
 
+  # Frontend Services
   keep-frontend:
     extends:
       file: docker-compose.common.yml
@@ -28,6 +31,7 @@ services:
       - FRIGADE_DISABLED=true
       - SENTRY_DISABLED=true
 
+  # Backend Services
   keep-backend:
     extends:
       file: docker-compose.common.yml
@@ -46,6 +50,75 @@ services:
       keep-database:
         condition: service_healthy
 
+
+  ## Keep Services with DB
+  # Database Service (5433) 
+  keep-database-db-auth:
+    image: mysql:latest
+    environment:
+      - MYSQL_ROOT_PASSWORD=keep
+      - MYSQL_DATABASE=keep
+    volumes:
+      - mysql-data:/var/lib/mysql-auth-db
+    ports:
+      - "3307:3306"
+    healthcheck:
+      test: ["CMD-SHELL", "mysqladmin ping -h localhost"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # Frontend Services (3001)
+  keep-frontend-db-auth:
+    build:
+      context: ./keep-ui/
+      dockerfile: ../docker/Dockerfile.ui
+    ports:
+      - "3001:3000"
+    environment:
+      - NEXTAUTH_SECRET=secret
+      - NEXTAUTH_URL=http://localhost:3001
+      - NEXT_PUBLIC_API_URL=http://localhost:8081
+      - POSTHOG_DISABLED=true
+      - AUTH_TYPE=DB
+      - API_URL=http://keep-backend-db-auth:8080
+      - POSTHOG_DISABLED=true
+      - FRIGADE_DISABLED=true
+      - SENTRY_DISABLED=true
+
+  # Backend Services (8081)
+  keep-backend-db-auth:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.api
+    ports:
+      - "8081:8080"
+    environment:
+      - PORT=8080
+      - SECRET_MANAGER_TYPE=FILE
+      - SECRET_MANAGER_DIRECTORY=/state
+      - OPENAI_API_KEY=$OPENAI_API_KEY
+      - PUSHER_APP_ID=1
+      - PUSHER_APP_KEY=keepappkey
+      - PUSHER_APP_SECRET=keepappsecret
+      - PUSHER_HOST=keep-websocket-server
+      - PUSHER_PORT=6001
+      - USE_NGROK=false
+      - AUTH_TYPE=DB
+      - DATABASE_CONNECTION_STRING=mysql+pymysql://root:keep@keep-database-db-auth:3306/keep
+      - POSTHOG_DISABLED=true
+      - FRIGADE_DISABLED=true
+      - SECRET_MANAGER_DIRECTORY=/app
+      - SQLALCHEMY_WARN_20=1
+      - KEEP_JWT_SECRET=verysecretkey
+      - KEEP_DEFAULT_USERNAME=keep
+      - KEEP_DEFAULT_PASSWORD=keep
+    depends_on:
+      keep-database-db-auth:
+        condition: service_healthy
+
+
+  # Other Services (Common)
   keep-websocket-server:
     extends:
       file: docker-compose.common.yml
@@ -59,5 +132,20 @@ services:
     ports:
       - "9090:9090"
 
+  grafana:
+    image: grafana/grafana-enterprise:11.4.0
+    user: "472" # Grafana's default user ID
+    ports:
+      - "3002:3000"
+    volumes:
+      - ./keep/providers/grafana_provider/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./tests/e2e_tests/grafana.ini:/etc/grafana/grafana.ini:ro
+      - grafana-storage:/var/lib/grafana
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    depends_on:
+      - prometheus-server-for-test-target
+
 volumes:
   mysql-data:
+  grafana-storage: {}

tests/e2e_tests/docker-compose-e2e-postgres.yml

@@ -1,4 +1,6 @@
 services:
+  ## Keep Services with NO_AUTH
+  # Database Service
   keep-database:
     image: postgres:13
     environment:
@@ -12,6 +14,7 @@ services:
       - ./postgres-custom.conf:/etc/postgresql/conf.d/custom.conf
       - ./docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
 
+  # Frontend Services
   keep-frontend:
     extends:
       file: docker-compose.common.yml
@@ -26,6 +29,7 @@ services:
       - FRIGADE_DISABLED=true
       - SENTRY_DISABLED=true
 
+  # Backend Services
   keep-backend:
     extends:
       file: docker-compose.common.yml
@@ -43,6 +47,70 @@ services:
     depends_on:
       - keep-database
 
+  ## Keep Services with DB
+  # Database Service (5433)
+  keep-database-db-auth:
+    image: postgres:13
+    environment:
+      POSTGRES_USER: keepuser
+      POSTGRES_PASSWORD: keeppassword
+      POSTGRES_DB: keepdb
+    ports:
+      - "5433:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql-auth-db/data
+      - ./postgres-custom.conf:/etc/postgresql-auth-db/conf.d/custom.conf
+      - ./docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+
+  # Frontend Services (3001)
+  keep-frontend-db-auth:
+    build:
+      context: ./keep-ui/
+      dockerfile: ../docker/Dockerfile.ui
+    ports:
+      - "3001:3000"
+    environment:
+      - NEXTAUTH_SECRET=secret
+      - NEXTAUTH_URL=http://localhost:3001
+      - NEXT_PUBLIC_API_URL=http://localhost:8080
+      - POSTHOG_DISABLED=true
+      - AUTH_TYPE=DB
+      - API_URL=http://keep-backend-db-auth:8080
+      - POSTHOG_DISABLED=true
+      - FRIGADE_DISABLED=true
+      - SENTRY_DISABLED=true
+
+  # Backend Services (8081)
+  keep-backend-db-auth:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.api
+    ports:
+      - "8081:8080"
+    environment:
+      - PORT=8080
+      - SECRET_MANAGER_TYPE=FILE
+      - SECRET_MANAGER_DIRECTORY=/state
+      - OPENAI_API_KEY=$OPENAI_API_KEY
+      - PUSHER_APP_ID=1
+      - PUSHER_APP_KEY=keepappkey
+      - PUSHER_APP_SECRET=keepappsecret
+      - PUSHER_HOST=keep-websocket-server
+      - PUSHER_PORT=6001
+      - USE_NGROK=false
+      - AUTH_TYPE=DB
+      - DATABASE_CONNECTION_STRING=postgresql+psycopg2://keepuser:keeppassword@keep-database-db-auth:5432/keepdb
+      - POSTHOG_DISABLED=true
+      - FRIGADE_DISABLED=true
+      - SECRET_MANAGER_DIRECTORY=/app
+      - SQLALCHEMY_WARN_20=1
+      - KEEP_JWT_SECRET=verysecretkey
+      - KEEP_DEFAULT_USERNAME=keep
+      - KEEP_DEFAULT_PASSWORD=keep
+    depends_on:
+      - keep-database-db-auth
+
+  # Other Services (Common)
   keep-websocket-server:
     extends:
       file: docker-compose.common.yml
@@ -56,5 +124,20 @@ services:
     ports:
       - "9090:9090"
 
+  grafana:
+    image: grafana/grafana-enterprise:11.4.0
+    user: "472" # Grafana's default user ID
+    ports:
+      - "3002:3000"
+    volumes:
+      - ./keep/providers/grafana_provider/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./tests/e2e_tests/grafana.ini:/etc/grafana/grafana.ini:ro
+      - grafana-storage:/var/lib/grafana
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    depends_on:
+      - prometheus-server-for-test-target
+
 volumes:
   postgres-data:
+  grafana-storage: {}

tests/e2e_tests/grafana.ini

@@ -0,0 +1,9 @@
+[unified_alerting]
+enabled = true
+
+[database]
+wal = true
+url = sqlite3:///var/lib/grafana/grafana.db?_busy_timeout=500
+
+[service_accounts]
+enabled = true