Improve test speed

Jenkinsfile

@@ -1,5 +1,6 @@
 @Library('podTemplateLib')
 import net.santiment.utils.podTemplates
+import java.util.UUID
 
 properties([
   buildDiscarder(
@@ -11,34 +12,136 @@ properties([
 
 slaveTemplates = new podTemplates()
 
+def numPartitions = 4
+
 slaveTemplates.dockerTemplate { label ->
   node(label) {
-    stage('Run Tests') {
+    stage('Build Test Image') {
       container('docker') {
         def scmVars = checkout scm
         def gitHead = scmVars.GIT_COMMIT.substring(0, 7)
         def sanitizedBranchName = env.BRANCH_NAME.replaceAll('/', '-')
+        def changeId = env.CHANGE_ID ?: 'none'
+        def imageTag = "sanbase-test:${scmVars.GIT_COMMIT}-${env.BUILD_ID}-${sanitizedBranchName}-${changeId}"
 
         sh "docker build \
-          -t sanbase-test:${scmVars.GIT_COMMIT}-${env.BUILD_ID}-${sanitizedBranchName}-${env.CHANGE_ID} \
+          -t ${imageTag} \
           -f Dockerfile-test . \
           --progress plain"
 
-        sh "docker run \
-            --rm --name test-postgres-${scmVars.GIT_COMMIT}-${env.BUILD_ID}-${sanitizedBranchName}-${env.CHANGE_ID} \
-            -e POSTGRES_PASSWORD=password \
-            -d pgvector/pgvector:pg15"
-
-        try {
-          sh "docker run --rm \
-            --link test-postgres-${scmVars.GIT_COMMIT}-${env.BUILD_ID}-${sanitizedBranchName}-${env.CHANGE_ID}:test-db \
-            --env DATABASE_URL=postgres://postgres:password@test-db:5432/postgres \
-            -t sanbase-test:${scmVars.GIT_COMMIT}-${env.BUILD_ID}-${sanitizedBranchName}-${env.CHANGE_ID}"
-        } finally {
-          sh "docker kill test-postgres-${scmVars.GIT_COMMIT}-${env.BUILD_ID}-${sanitizedBranchName}-${env.CHANGE_ID}"
+        // Store variables for later stages
+        env.TEST_IMAGE_TAG = imageTag
+        env.GIT_HEAD = gitHead
+        env.GIT_COMMIT_FULL = scmVars.GIT_COMMIT
+      }
+    }
+
+    stage('Run Tests') {
+      container('docker') {
+        catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+          def imageTag = env.TEST_IMAGE_TAG
+          def runToken = UUID.randomUUID().toString().substring(0, 8)
+          def buildSuffix = "${env.GIT_COMMIT_FULL}-${env.BUILD_ID}-${runToken}"
+          env.TEST_RUN_SUFFIX = buildSuffix
+          def failuresDir = "${pwd()}/test_failures_${buildSuffix}"
+
+          sh "mkdir -p ${failuresDir}"
+
+          // Start postgres containers for each partition
+          for (int i = 1; i <= numPartitions; i++) {
+            def postgresContainerName = "test-postgres-${buildSuffix}-p${i}"
+            sh "docker rm -f ${postgresContainerName} >/dev/null 2>&1 || true"
+            sh "docker run \
+              --rm --name ${postgresContainerName} \
+              -e POSTGRES_PASSWORD=password \
+              --health-cmd 'pg_isready -U postgres' \
+              --health-interval 2s \
+              --health-timeout 5s \
+              --health-retries 10 \
+              -d pgvector/pgvector:pg15"
+          }
+
+          // Wait for each Postgres container to be ready
+          sh """
+            set -e
+            for i in \$(seq 1 ${numPartitions}); do
+              name="test-postgres-${buildSuffix}-p\$i"
+              echo "Waiting for \$name to be ready..."
+              for attempt in \$(seq 1 30); do
+                if docker exec "\$name" pg_isready -U postgres 2>/dev/null; then
+                  echo "\$name is ready"
+                  break
+                fi
+                if [ \$attempt -eq 30 ]; then
+                  echo "ERROR: \$name did not become ready within 60 seconds"
+                  docker logs "\$name" 2>&1 || true
+                  exit 1
+                fi
+                sleep 2
+              done
+            done
+          """
+
+          try {
+            def partitions = [:]
+
+            for (int i = 1; i <= numPartitions; i++) {
+              def partition = i
+              partitions["Partition ${partition}"] = {
+                sh "docker run --rm \
+                  --link test-postgres-${buildSuffix}-p${partition}:test-db \
+                  --env DATABASE_URL=postgres://postgres:password@test-db:5432/postgres \
+                  --env MIX_TEST_PARTITION=${partition} \
+                  -v ${failuresDir}:/app/_build/test/failures \
+                  -t ${imageTag} \
+                  mix test --partitions ${numPartitions} \
+                    --formatter Sanbase.FailedTestFormatter \
+                    --formatter ExUnit.CLIFormatter \
+                    --slowest 20"
+              }
+            }
+
+            parallel partitions
+          } finally {
+            for (int i = 1; i <= numPartitions; i++) {
+              sh "docker kill test-postgres-${buildSuffix}-p${i} || true"
+            }
+          }
         }
+      }
+    }
+
+    stage('Summarize Test Failures') {
+      container('docker') {
+        def buildSuffix = env.TEST_RUN_SUFFIX ?: "${env.GIT_COMMIT_FULL}-${env.BUILD_ID}"
+        def failuresDir = "${pwd()}/test_failures_${buildSuffix}"
+
+        sh """
+          set +x
+          echo ''
+          echo '========================================'
+          echo '  Combined results from all partitions'
+          echo '========================================'
+
+          if ls ${failuresDir}/partition_*.txt 1>/dev/null 2>&1; then
+            echo 'Failing tests across all partitions:'
+            cut -f2 ${failuresDir}/partition_*.txt | sort
+            echo ''
+            echo 'Re-run with:'
+            echo "  mix test \$(cut -f2 ${failuresDir}/partition_*.txt | tr '\\n' ' ')"
+            rm -rf ${failuresDir}
+            exit 1
+          fi
+
+          echo 'All partitions passed!'
+          rm -rf ${failuresDir}
+        """
+      }
+    }
 
-        if (env.BRANCH_NAME == 'master') {
+    if (env.BRANCH_NAME == 'master') {
+      stage('Build & Push') {
+        container('docker') {
           withCredentials([
             string(
               credentialsId: 'SECRET_KEY_BASE',
@@ -57,15 +160,15 @@ slaveTemplates.dockerTemplate { label ->
             docker.withRegistry("https://${awsRegistry}", 'ecr:eu-central-1:ecr-credentials') {
               sh "docker build \
                 -t ${awsRegistry}/sanbase:${env.BRANCH_NAME} \
-                -t ${awsRegistry}/sanbase:${scmVars.GIT_COMMIT} \
+                -t ${awsRegistry}/sanbase:${env.GIT_COMMIT_FULL} \
                 --build-arg SECRET_KEY_BASE=${env.SECRET_KEY_BASE} \
                 --build-arg PARITY_URL=${env.PARITY_URL} \
-                --build-arg GIT_HEAD=${gitHead} \
-                --build-arg GIT_COMMIT=${scmVars.GIT_COMMIT} . \
+                --build-arg GIT_HEAD=${env.GIT_HEAD} \
+                --build-arg GIT_COMMIT=${env.GIT_COMMIT_FULL} . \
                 --progress plain"
 
               sh "docker push ${awsRegistry}/sanbase:${env.BRANCH_NAME}"
-              sh "docker push ${awsRegistry}/sanbase:${scmVars.GIT_COMMIT}"
+              sh "docker push ${awsRegistry}/sanbase:${env.GIT_COMMIT_FULL}"
             }
           }
         }

config/test.exs

@@ -14,11 +14,17 @@ config :sanbase, Sanbase.EventBus,
 
 config :sanbase, Sanbase, url: {:system, "SANBASE_URL", ""}
 
+test_port =
+  case System.get_env("MIX_TEST_PARTITION") do
+    nil -> 4001
+    partition -> 4001 + String.to_integer(partition)
+  end
+
 config :sanbase, SanbaseWeb.Endpoint,
-  http: [port: 4001],
+  http: [port: test_port],
   server: true,
-  website_url: {:system, "WEBSITE_URL", "http://localhost:4001"},
-  backend_url: {:system, "BACKEND_URL", "http://localhost:4001"}
+  website_url: {:system, "WEBSITE_URL", "http://localhost:#{test_port}"},
+  backend_url: {:system, "BACKEND_URL", "http://localhost:#{test_port}"}
 
 config :ex_aws,
   access_key_id: "test_id",
@@ -39,8 +45,8 @@ config :logger, :console,
 config :sanbase, Sanbase.RepoReader, projects_data_endpoint_secret: "no_secret"
 
 config :sanbase, Sanbase.ApiCallLimit,
-  quota_size: 10,
-  quota_size_max_offset: 10
+  quota_size: 5,
+  quota_size_max_offset: 5
 
 config :sanbase, Sanbase.Accounts.Interaction,
   interaction_cooldown_seconds: 0,
@@ -73,7 +79,7 @@ config :sanbase, Sanbase.Repo,
   pool: Ecto.Adapters.SQL.Sandbox,
   username: "postgres",
   password: "postgres",
-  database: "sanbase_test",
+  database: "sanbase_test#{System.get_env("MIX_TEST_PARTITION")}",
   pool_size: 5,
   ssl: false,
   ssl_opts: []

lib/mix/failure_test_formatter.ex

@@ -2,7 +2,9 @@ defmodule Sanbase.FailedTestFormatter do
   @moduledoc false
   use GenServer
 
-  ## Callbacks
+  @failures_dir "_build/test/failures"
+
+  def failures_dir, do: @failures_dir
 
   def init(_opts) do
     {:ok,
@@ -17,23 +19,19 @@ defmodule Sanbase.FailedTestFormatter do
       case test do
         %ExUnit.Test{state: {:failed, [{kind, error, _stacktrace}]}} = test
         when kind in [:error, :invalid, :exit] ->
-          # Add a leading dot so the file:line string can be copy-pasted in the
-          # terminal to directly execute it
           file = String.replace_leading(test.tags.file, File.cwd!() <> "/", "")
           line = test.tags.line
 
-          # In case of :exit, the error tuple can contain more info, like :timeout, etc.
-          # In case of :error, the error is not a tuple, but a struct
-          reason = if kind == :exit and is_tuple(error), do: " (#{elem(error, 0)})", else: ""
+          # Do not add the reason for now
+          _reason = if kind == :exit and is_tuple(error), do: " (#{elem(error, 0)})", else: ""
 
-          test_identifier = "#{file}:#{line}#{reason}"
+          test_identifier = "#{file}:#{line}"
 
           config
           |> Map.update(kind, %{counter: 1, list: [test_identifier]}, fn map ->
             map |> Map.update!(:counter, &(&1 + 1)) |> Map.update!(:list, &[test_identifier | &1])
           end)
 
-        # TODO: Support ExUnit.MultiError
         test ->
           IO.warn("Unexpected failed test format. Got: #{inspect(test)}")
           config
@@ -44,6 +42,7 @@ defmodule Sanbase.FailedTestFormatter do
 
   def handle_cast({:suite_finished, _times_us}, config) do
     print_suite(config)
+    write_failures_to_file(config)
     {:noreply, config}
   end
 
@@ -54,7 +53,6 @@ defmodule Sanbase.FailedTestFormatter do
   defp print_suite(config) do
     for kind <- Map.keys(config) do
       if (get_in(config, [kind, :counter]) || 0) > 0 do
-        # All tests that failed an assert will have `kind = :error`
         error_tests_message =
           get_in(config, [kind, :list]) |> Enum.map(&(" " <> &1)) |> Enum.join("\n")
 
@@ -67,4 +65,23 @@ defmodule Sanbase.FailedTestFormatter do
       end
     end
   end
+
+  defp write_failures_to_file(config) do
+    lines =
+      for kind <- Map.keys(config),
+          (get_in(config, [kind, :counter]) || 0) > 0,
+          test_id <- get_in(config, [kind, :list]) do
+        "#{kind}\t#{test_id}"
+      end
+
+    if lines != [] do
+      partition = System.get_env("MIX_TEST_PARTITION") || "0"
+      File.mkdir_p!(@failures_dir)
+
+      File.write!(
+        Path.join(@failures_dir, "partition_#{partition}.txt"),
+        Enum.join(lines, "\n") <> "\n"
+      )
+    end
+  end
 end

lib/sanbase_web/graphql/resolvers/entity_resolver.ex

@@ -191,9 +191,6 @@ defmodule SanbaseWeb.Graphql.Resolvers.EntityResolver do
           {:ok, %{query: :get_most_similar, args: args_with_embedding}}
 
         {:error, reason} ->
-          # Preserve the previous GraphQL shape expected by tests:
-          # - keep the top-level field non-null
-          # - surface the error via the nested data/stats resolvers
           {:error, reason}
       end
     end

test/sanbase/accounts/access_attempt_test.exs

@@ -1,5 +1,5 @@
 defmodule Sanbase.Accounts.AccessAttemptTest do
-  use Sanbase.DataCase, async: false
+  use Sanbase.DataCase, async: true
   alias Sanbase.Accounts.{AccessAttempt, EmailLoginAttempt}
   alias Sanbase.Repo
   import Sanbase.Factory

test/sanbase/accounts/user/user_email_test.exs

@@ -1,5 +1,5 @@
 defmodule Sanbase.Accounts.User.EmailTest do
-  use SanbaseWeb.ConnCase, async: false
+  use SanbaseWeb.ConnCase, async: true
 
   alias Sanbase.Accounts.User.Email
 

test/sanbase/ai/academy_ai_service_test.exs

@@ -1,5 +1,5 @@
 defmodule Sanbase.AI.AcademyAIServiceTest do
-  use Sanbase.DataCase, async: false
+  use Sanbase.DataCase, async: true
 
   import Sanbase.Factory
   import Mox

test/sanbase/ai/openai_client_test.exs

@@ -1,6 +1,5 @@
 defmodule Sanbase.AI.OpenAIClientTest do
-  use ExUnit.Case, async: false
-  use SanbaseWeb.ConnCase
+  use ExUnit.Case, async: true
 
   import Mox
 

test/sanbase/alerts/trigger_restrictions_test.exs

@@ -1,5 +1,5 @@
 defmodule Sanbase.Alert.TriggerRestrictionsTest do
-  use SanbaseWeb.ConnCase, async: false
+  use SanbaseWeb.ConnCase, async: true
 
   import Sanbase.Factory
   import SanbaseWeb.Graphql.TestHelpers

test/sanbase/alerts/trigger_test.exs

@@ -1,5 +1,5 @@
 defmodule Sanbase.Alert.TriggersTest do
-  use Sanbase.DataCase, async: false
+  use Sanbase.DataCase, async: true
 
   import Sanbase.Factory
   import ExUnit.CaptureLog

test/sanbase/alerts/trigger_voting_test.exs

@@ -1,5 +1,5 @@
 defmodule Sanbase.Alert.TriggerVotingTest do
-  use SanbaseWeb.ConnCase, async: false
+  use SanbaseWeb.ConnCase, async: true
 
   import Sanbase.Factory
   import SanbaseWeb.Graphql.TestHelpers

test/sanbase/app_notifications/app_notifications_test.exs

@@ -641,8 +641,8 @@ defmodule Sanbase.AppNotificationsTest do
       {:ok, _vote} = Vote.create(%{user_id: voter.id, watchlist_id: watchlist.id})
       {:ok, _vote} = Vote.create(%{user_id: voter.id, watchlist_id: watchlist.id})
 
-      subscriber = Sanbase.EventBus.AppNotificationsSubscriber
-      Sanbase.EventBus.drain_topics(subscriber.topics(), 10_000)
+      Sanbase.EventBus.AppNotificationsSubscriber.topics()
+      |> Sanbase.EventBus.drain_topics()
 
       owner_notifications = AppNotifications.list_notifications_for_user(author.id, limit: 20)
 

test/sanbase/auth/apikey/apikey_test.exs

@@ -1,5 +1,5 @@
 defmodule Sanbase.Accounts.ApiKeyTest do
-  use Sanbase.DataCase, async: false
+  use Sanbase.DataCase, async: true
 
   alias Sanbase.Accounts.{
     User,

test/sanbase/auth/user_permissions_test.exs

@@ -1,5 +1,5 @@
 defmodule Sanbase.Accounts.UserPermissionsTest do
-  use Sanbase.DataCase, async: false
+  use Sanbase.DataCase, async: true
 
   import Sanbase.Factory
 

test/sanbase/comments/comment_test.exs

@@ -1,5 +1,5 @@
 defmodule Sanbase.CommentTest do
-  use Sanbase.DataCase, async: false
+  use Sanbase.DataCase, async: true
 
   import Sanbase.Factory
   alias Sanbase.Comments.EntityComment

test/sanbase/comments/notification_test.exs

@@ -1,5 +1,5 @@
 defmodule Sanbase.Comments.NotificationTest do
-  use Sanbase.DataCase, async: false
+  use Sanbase.DataCase, async: true
 
   import Sanbase.Factory
   alias Sanbase.Comments.{EntityComment, Notification}