feat(#2724): move drep list pagination directly to sql

CHANGELOG.md

@@ -29,6 +29,7 @@ changes.
 - Change votes representation on Governance Actions [Issue 2880](https://github.com/IntersectMBO/govtool/issues/2880)
 - Change vote rationale character limit to 10000 [Issue 2891](https://github.com/IntersectMBO/govtool/issues/2891)
 - Move ratification threshold label below the voter type [Issue 2893](https://github.com/IntersectMBO/govtool/issues/2893)
+- Move DRep list pagination directly to SQL [Issue 2724](https://github.com/IntersectMBO/govtool/issues/2724)
 
 ### Removed
 

govtool/backend/app/Main.hs

@@ -144,6 +144,7 @@ startApp vvaConfig sentryService = do
 
 exceptionHandler :: VVAConfig -> SentryService -> Maybe Request -> SomeException -> IO ()
 exceptionHandler vvaConfig sentryService mRequest exception = do
+  print exception
   let isNotTimeoutThread x = case fromException x of
         Just TimeoutThread -> False
         _                  -> True

govtool/backend/example-config.json

@@ -1,11 +1,11 @@
 {
   "dbsyncconfig" : {
-      "host"     : "localhost",
-      "dbname"   : "cexplorer",
-      "user"     : "postgres",
-      "password" : "postgres",
-      "port"     : 5432
-    },
+    "host"     : "localhost",
+    "dbname"   : "cexplorer",
+    "user"     : "postgres",
+    "password" : "postgres",
+    "port"     : 5432
+  },
   "port" : 9999,
   "host" : "localhost",
   "cachedurationseconds": 20,

govtool/backend/sql/list-dreps.sql

@@ -117,19 +117,29 @@ DRepData AS (
     leva.metadata_hash,
     COALESCE(dr_deposit.deposit, 0) as deposit,
     DRepDistr.amount,
-    (DRepActivity.epoch_no - GREATEST(COALESCE(voting_procedure_block.epoch_no, block_first_register.epoch_no), lve.epoch_no, newestRegister.epoch_no)) <= DRepActivity.drep_activity AS active,
     RankedDRepRegistration.tx_hash,
-    newestRegister.time AS last_register_time,
-    COALESCE(RankedDRepRegistration.deposit, 0) as latest_deposit,
-    hndva.value AS has_non_deregister_voting_anchor,
+    newestRegister.time AT TIME ZONE 'UTC' AS last_register_time,
     fetch_error.message AS fetch_error,
     off_chain_vote_drep_data.payment_address,
     off_chain_vote_drep_data.given_name,
     off_chain_vote_drep_data.objectives,
     off_chain_vote_drep_data.motivations,
     off_chain_vote_drep_data.qualifications,
     off_chain_vote_drep_data.image_url,
-    off_chain_vote_drep_data.image_hash
+    off_chain_vote_drep_data.image_hash,
+    -- drep type
+    CASE
+      WHEN COALESCE(RankedDRepRegistration.deposit, 0) >= 0 AND leva.url IS NULL THEN 'SoleVoter'
+      WHEN COALESCE(RankedDRepRegistration.deposit, 0) >= 0 AND leva.url IS NOT NULL THEN 'DRep'
+      WHEN COALESCE(RankedDRepRegistration.deposit, 0) < 0 AND hndva.value = true THEN 'SoleVoter'
+      WHEN COALESCE(RankedDRepRegistration.deposit, 0) < 0 AND hndva.value = false THEN 'DRep'
+    END AS drep_type,
+    -- status
+    CASE
+      WHEN COALESCE(RankedDRepRegistration.deposit, 0) < 0 THEN 'Retired'
+      WHEN COALESCE(RankedDRepRegistration.deposit, 0) >= 0 AND (DRepActivity.epoch_no - GREATEST(COALESCE(voting_procedure_block.epoch_no, block_first_register.epoch_no), lve.epoch_no, newestRegister.epoch_no)) <= DRepActivity.drep_activity THEN 'Active'
+      WHEN COALESCE(RankedDRepRegistration.deposit, 0) >= 0 AND NOT (DRepActivity.epoch_no - GREATEST(COALESCE(voting_procedure_block.epoch_no, block_first_register.epoch_no), lve.epoch_no, newestRegister.epoch_no)) <= DRepActivity.drep_activity THEN 'Inactive'
+    END AS status
   FROM
     drep_hash dh
     JOIN RankedDRepRegistration ON RankedDRepRegistration.drep_hash_id = dh.id AND RankedDRepRegistration.rn = 1
@@ -195,33 +205,57 @@ DRepData AS (
     leva.metadata_hash,
     dr_deposit.deposit,
     DRepDistr.amount,
-    DRepActivity.epoch_no,
-    voting_procedure_block.epoch_no,
-    block_first_register.epoch_no,
-    lve.epoch_no, newestRegister.epoch_no,
-    DRepActivity.drep_activity,
     RankedDRepRegistration.tx_hash,
     newestRegister.time,
-    RankedDRepRegistration.deposit,
-    hndva.value,
     fetch_error.message,
     off_chain_vote_drep_data.payment_address,
     off_chain_vote_drep_data.given_name,
     off_chain_vote_drep_data.objectives,
     off_chain_vote_drep_data.motivations,
     off_chain_vote_drep_data.qualifications,
     off_chain_vote_drep_data.image_url,
-    off_chain_vote_drep_data.image_hash
-)
-SELECT * FROM DRepData
-WHERE
+    off_chain_vote_drep_data.image_hash,
+    RankedDRepRegistration.deposit,
+    hndva.value,
+    DRepActivity.epoch_no,
+    DRepActivity.drep_activity,
+    voting_procedure_block.epoch_no,
+    block_first_register.epoch_no,
+    lve.epoch_no,
+    newestRegister.epoch_no
+),
+FilteredDRepData AS (
+  SELECT * FROM DRepData
+  WHERE
   (
-    COALESCE(?, '') = '' OR
-    (CASE WHEN LENGTH(?) % 2 = 0 AND ? ~ '^[0-9a-fA-F]+$' THEN drep_hash = ? ELSE false END) OR
-    view ILIKE ? OR
-    given_name ILIKE ? OR
-    payment_address ILIKE ? OR
-    objectives ILIKE ? OR
-    motivations ILIKE ? OR
-    qualifications ILIKE ?
-  )
+    COALESCE(?, '') = '' 
+    OR (
+      CASE 
+        WHEN LENGTH(?) % 2 = 0 AND ? ~ '^[0-9a-fA-F]+$' THEN drep_hash = ? 
+        ELSE false 
+      END
+    )
+    OR (
+      ? ILIKE ANY(ARRAY[view, given_name, payment_address, objectives, motivations, qualifications])
+    )
+  )
+  AND (?::TEXT = '' OR status = ANY(?))
+  AND (drep_type != 'SoleVoter' OR (COALESCE(?, '') <> '' AND view ILIKE ?)) 
+)
+SELECT 
+  (SELECT COUNT(*) FROM FilteredDRepData) AS total,
+  COALESCE(jsonb_agg(elements), '[]'::jsonb) AS elements
+FROM (
+  SELECT *
+  FROM FilteredDRepData
+  ORDER BY 
+    CASE ? 
+      WHEN 'VotingPower' THEN amount::TEXT
+      WHEN 'RegistrationDate' THEN last_register_time::TEXT 
+      WHEN 'Status' THEN status::TEXT
+      ELSE NULL 
+    END DESC,
+    CASE WHEN ? = 'Random' THEN RANDOM() END
+  LIMIT ?
+  OFFSET ?
+) AS elements

govtool/backend/src/VVA/API.hs

@@ -43,6 +43,7 @@ import           VVA.Network              as Network
 import qualified VVA.Proposal             as Proposal
 import qualified VVA.Transaction          as Transaction
 import qualified VVA.Types                as Types
+import           VVA.Common.Types
 import           VVA.Types                (App, AppEnv (..),
                                            AppError (CriticalError, InternalError, ValidationError),
                                            CacheEnv (..))
@@ -140,60 +141,24 @@ delegationToResponse Types.Delegation {..} =
 drepList :: App m => Maybe Text -> [DRepStatus] -> Maybe DRepSortMode -> Maybe Natural -> Maybe Natural -> m ListDRepsResponse
 drepList mSearchQuery statuses mSortMode mPage mPageSize = do
   CacheEnv {dRepListCache} <- asks vvaCache
-  dreps <- cacheRequest dRepListCache (fromMaybe "" mSearchQuery) (DRep.listDReps mSearchQuery)
-
-  let filterDRepsByQuery = case mSearchQuery of
-        Nothing -> filter $ \Types.DRepRegistration {..} ->
-          dRepRegistrationType /= Types.SoleVoter
-        Just query -> filter $ \Types.DRepRegistration {..} ->
-          let searchLower = Text.toLower query
-              viewLower = Text.toLower dRepRegistrationView
-              hashLower = Text.toLower dRepRegistrationDRepHash
-          in case dRepRegistrationType of
-              Types.SoleVoter -> 
-                searchLower == viewLower || searchLower == hashLower
-              Types.DRep -> 
-                True
-
-
-  let filterDRepsByStatus = case statuses of
-        [] -> id
-        _  -> filter $ \Types.DRepRegistration {..} ->
-          mapDRepStatus dRepRegistrationStatus `elem` statuses
-
-  randomizedOrderList <- mapM (\_ -> randomRIO (0, 1 :: Double)) dreps
-
-  let sortDReps = case mSortMode of
-        Nothing -> id
-        Just Random -> fmap snd . sortOn fst . Prelude.zip randomizedOrderList
-        Just VotingPower -> sortOn $ \Types.DRepRegistration {..} ->
-          Down dRepRegistrationVotingPower
-        Just RegistrationDate -> sortOn $ \Types.DRepRegistration {..} ->
-          Down dRepRegistrationLatestRegistrationDate
-        Just Status -> sortOn $ \Types.DRepRegistration {..} ->
-          dRepRegistrationStatus
-
-  appEnv <- ask
-
-  allValidDReps <- liftIO $ mapConcurrently
-    (\[email protected]{..} -> do
-        let drep = drepRegistrationToDrep d
-        return drep)
-    $ sortDReps $ filterDRepsByQuery $ filterDRepsByStatus dreps
 
-  let page = (fromIntegral $ fromMaybe 0 mPage) :: Int
-      pageSize = (fromIntegral $ fromMaybe 10 mPageSize) :: Int
+  let page = fromMaybe 0 mPage
+  let pageSize = fromMaybe 10 mPageSize
+  let offset = page * pageSize
+  let sortMode = fromMaybe VotingPower mSortMode
 
-      total = length allValidDReps :: Int
+  let cacheKey = pack (show (mSearchQuery, statuses, sortMode, page, pageSize))
 
-  let elements = take pageSize $ drop (page * pageSize) allValidDReps
-  return $ ListDRepsResponse
-    { listDRepsResponsePage = fromIntegral page
-    , listDRepsResponsePageSize = fromIntegral pageSize
-    , listDRepsResponseTotal = fromIntegral total
-    , listDRepsResponseElements = elements
-    }
+  (totalCount, cachedDReps) <- cacheRequest dRepListCache cacheKey $
+      DRep.listDReps mSearchQuery statuses (Just sortMode) (fromIntegral pageSize) (fromIntegral offset)
 
+  let response = ListDRepsResponse
+        { listDRepsResponsePage = fromIntegral page
+        , listDRepsResponsePageSize = fromIntegral pageSize
+        , listDRepsResponseTotal = fromIntegral totalCount
+        , listDRepsResponseElements = cachedDReps
+        }
+  return response
 
 getVotingPower :: App m => HexText -> m Integer
 getVotingPower (unHexText -> dRepId) = do