feat: add order_by and order params to /extended/v1/tx/mempool (#1810)

* feat: sort mempool

* test: age sort

* fix: add enum types

* chore: move v2 to proper folder

* docs: add to openapi

Author: rafaelcr

docs/openapi.yaml

@@ -263,6 +263,22 @@ paths:
           required: false
           schema:
             type: string
+        - name: order_by
+          in: query
+          description: Option to sort results by transaction age, size, or fee rate.
+          required: false
+          schema:
+            type: string
+            enum: [age, size, fee]
+            example: fee
+        - name: order
+          in: query
+          description: Option to sort results in ascending or descending order.
+          required: false
+          schema:
+            type: string
+            enum: [asc, desc]
+            example: asc
         - name: limit
           in: query
           description: max number of mempool transactions to fetch

src/api/init.ts

@@ -52,7 +52,7 @@ import {
 import { createV2BlocksRouter } from './routes/v2/blocks';
 import { getReqQuery } from './query-helpers';
 import { createV2BurnBlocksRouter } from './routes/v2/burn-blocks';
-import { createMempoolRouter } from './v2/mempool';
+import { createMempoolRouter } from './routes/v2/mempool';
 
 export interface ApiServer {
   expressApp: express.Express;

src/api/query-helpers.ts

@@ -5,6 +5,17 @@ import { InvalidRequestError, InvalidRequestErrorType } from '../errors';
 import { DbEventTypeId } from './../datastore/common';
 import { has0xPrefix, hexToBuffer } from '@hirosystems/api-toolkit';
 
+export enum MempoolOrderByParam {
+  fee = 'fee',
+  size = 'size',
+  age = 'age',
+}
+
+export enum OrderParam {
+  asc = 'asc',
+  desc = 'desc',
+}
+
 function handleBadRequest(res: Response, next: NextFunction, errorMessage: string): never {
   const error = new InvalidRequestError(errorMessage, InvalidRequestErrorType.bad_request);
   res.status(400).json({ error: errorMessage });

src/api/routes/tx.ts

@@ -16,6 +16,8 @@ import {
   validateRequestHexInput,
   parseAddressOrTxId,
   parseEventTypeFilter,
+  MempoolOrderByParam,
+  OrderParam,
 } from '../query-helpers';
 import { getPagingQueryLimit, parsePagingQueryInput, ResourceType } from '../pagination';
 import { validate } from '../validate';
@@ -162,10 +164,33 @@ export function createTxRouter(db: PgStore): express.Router {
           InvalidRequestErrorType.invalid_param
         );
       }
+
+      const orderBy = req.query.order_by;
+      if (
+        orderBy !== undefined &&
+        orderBy != MempoolOrderByParam.fee &&
+        orderBy != MempoolOrderByParam.age &&
+        orderBy != MempoolOrderByParam.size
+      ) {
+        throw new InvalidRequestError(
+          `The "order_by" param can only be 'fee', 'age', or 'size'`,
+          InvalidRequestErrorType.invalid_param
+        );
+      }
+      const order = req.query.order;
+      if (order !== undefined && order != OrderParam.asc && order != OrderParam.desc) {
+        throw new InvalidRequestError(
+          `The "order" param can only be 'asc' or 'desc'`,
+          InvalidRequestErrorType.invalid_param
+        );
+      }
+
       const { results: txResults, total } = await db.getMempoolTxList({
         offset,
         limit,
         includeUnanchored,
+        orderBy,
+        order,
         senderAddress,
         recipientAddress,
         address,

src/api/v2/mempool.ts renamed to src/api/routes/v2/mempool.ts

@@ -1,13 +1,13 @@
 import * as express from 'express';
-import { asyncHandler } from '../async-handler';
+import { asyncHandler } from '../../async-handler';
 import {
   ETagType,
   getETagCacheHandler,
   setETagCacheHeaders,
-} from '../controllers/cache-controller';
-import { PgStore } from '../../datastore/pg-store';
-import { DbMempoolFeePriority, DbTxTypeId } from '../../datastore/common';
-import { MempoolFeePriorities } from '../../../docs/generated';
+} from '../../controllers/cache-controller';
+import { PgStore } from '../../../datastore/pg-store';
+import { DbMempoolFeePriority, DbTxTypeId } from '../../../datastore/common';
+import { MempoolFeePriorities } from '../../../../docs/generated';
 
 function parseMempoolFeePriority(fees: DbMempoolFeePriority[]): MempoolFeePriorities {
   const out: MempoolFeePriorities = {

src/datastore/pg-store.ts

@@ -101,6 +101,7 @@ import {
 } from './connection';
 import * as path from 'path';
 import { PgStoreV2 } from './pg-store-v2';
+import { MempoolOrderByParam, OrderParam } from '../api/query-helpers';
 
 export const MIGRATIONS_DIR = path.join(REPO_DIR, 'migrations');
 
@@ -1289,13 +1290,17 @@ export class PgStore extends BasePgStore {
     limit,
     offset,
     includeUnanchored,
+    orderBy,
+    order,
     senderAddress,
     recipientAddress,
     address,
   }: {
     limit: number;
     offset: number;
     includeUnanchored: boolean;
+    orderBy?: MempoolOrderByParam;
+    order?: OrderParam;
     senderAddress?: string;
     recipientAddress?: string;
     address?: string;
@@ -1310,6 +1315,9 @@ export class PgStore extends BasePgStore {
         senderAddress || recipientAddress || address
           ? sql`(COUNT(*) OVER())::int AS count`
           : sql`(SELECT mempool_tx_count FROM chain_tip) AS count`;
+      const orderBySql =
+        orderBy == 'fee' ? sql`fee_rate` : orderBy == 'size' ? sql`tx_size` : sql`receipt_time`;
+      const orderSql = order == 'asc' ? sql`ASC` : sql`DESC`;
       const resultQuery = await sql<(MempoolTxQueryResult & { count: number })[]>`
         SELECT ${unsafeCols(sql, [...MEMPOOL_TX_COLUMNS, abiColumn('mempool_txs')])}, ${count}
         FROM mempool_txs
@@ -1333,7 +1341,7 @@ export class PgStore extends BasePgStore {
               ? sql`OR tx_id IN ${sql(unanchoredTxs)}`
               : sql``
           })
-        ORDER BY receipt_time DESC
+        ORDER BY ${orderBySql} ${orderSql}
         LIMIT ${limit}
         OFFSET ${offset}
       `;

src/tests/mempool-tests.ts

@@ -1141,6 +1141,89 @@ describe('mempool tests', () => {
     expect(JSON.parse(searchResult7.text)).toEqual(expectedResp7);
   });
 
+  test('fetch mempool-tx list sorted', async () => {
+    const sendAddr = 'SP25YGP221F01S9SSCGN114MKDAK9VRK8P3KXGEMB';
+    const recvAddr = 'SP10EZK56MB87JYF5A704K7N18YAT6G6M09HY22GC';
+
+    const block = new TestBlockBuilder().addTx().build();
+    await db.update(block);
+    const txs: DbMempoolTxRaw[] = [];
+    for (let index = 0; index < 5; index++) {
+      const paddedIndex = ('00' + index).slice(-2);
+      const mempoolTx: DbMempoolTxRaw = {
+        pruned: false,
+        tx_id: `0x89120000000000000000000000000000000000000000000000000000000000${paddedIndex}`,
+        anchor_mode: 3,
+        nonce: 0,
+        raw_tx: bufferToHex(Buffer.from('x'.repeat(index + 1))),
+        type_id: DbTxTypeId.TokenTransfer,
+        receipt_time: (new Date(`2020-07-09T15:14:${paddedIndex}Z`).getTime() / 1000) | 0,
+        status: 1,
+        post_conditions: '0x01f5',
+        fee_rate: 100n * BigInt(index + 1),
+        sponsored: false,
+        sponsor_address: undefined,
+        origin_hash_mode: 1,
+        sender_address: sendAddr,
+        token_transfer_recipient_address: recvAddr,
+        token_transfer_amount: 1234n,
+        token_transfer_memo: '',
+      };
+      txs.push(mempoolTx);
+    }
+    await db.updateMempoolTxs({ mempoolTxs: txs });
+
+    let result = await supertest(api.server).get(`/extended/v1/tx/mempool?order_by=fee&order=desc`);
+    let json = JSON.parse(result.text);
+    expect(json.results[0].fee_rate).toBe('500');
+    expect(json.results[1].fee_rate).toBe('400');
+    expect(json.results[2].fee_rate).toBe('300');
+    expect(json.results[3].fee_rate).toBe('200');
+    expect(json.results[4].fee_rate).toBe('100');
+
+    result = await supertest(api.server).get(`/extended/v1/tx/mempool?order_by=fee&order=asc`);
+    json = JSON.parse(result.text);
+    expect(json.results[0].fee_rate).toBe('100');
+    expect(json.results[1].fee_rate).toBe('200');
+    expect(json.results[2].fee_rate).toBe('300');
+    expect(json.results[3].fee_rate).toBe('400');
+    expect(json.results[4].fee_rate).toBe('500');
+
+    // Larger transactions were set with higher fees.
+    result = await supertest(api.server).get(`/extended/v1/tx/mempool?order_by=size&order=desc`);
+    json = JSON.parse(result.text);
+    expect(json.results[0].fee_rate).toBe('500');
+    expect(json.results[1].fee_rate).toBe('400');
+    expect(json.results[2].fee_rate).toBe('300');
+    expect(json.results[3].fee_rate).toBe('200');
+    expect(json.results[4].fee_rate).toBe('100');
+
+    result = await supertest(api.server).get(`/extended/v1/tx/mempool?order_by=size&order=asc`);
+    json = JSON.parse(result.text);
+    expect(json.results[0].fee_rate).toBe('100');
+    expect(json.results[1].fee_rate).toBe('200');
+    expect(json.results[2].fee_rate).toBe('300');
+    expect(json.results[3].fee_rate).toBe('400');
+    expect(json.results[4].fee_rate).toBe('500');
+
+    // Newer transactions were set with higher fees.
+    result = await supertest(api.server).get(`/extended/v1/tx/mempool?order_by=age&order=desc`);
+    json = JSON.parse(result.text);
+    expect(json.results[0].fee_rate).toBe('500');
+    expect(json.results[1].fee_rate).toBe('400');
+    expect(json.results[2].fee_rate).toBe('300');
+    expect(json.results[3].fee_rate).toBe('200');
+    expect(json.results[4].fee_rate).toBe('100');
+
+    result = await supertest(api.server).get(`/extended/v1/tx/mempool?order_by=age&order=asc`);
+    json = JSON.parse(result.text);
+    expect(json.results[0].fee_rate).toBe('100');
+    expect(json.results[1].fee_rate).toBe('200');
+    expect(json.results[2].fee_rate).toBe('300');
+    expect(json.results[3].fee_rate).toBe('400');
+    expect(json.results[4].fee_rate).toBe('500');
+  });
+
   test('mempool - contract_call tx abi details are retrieved', async () => {
     const block1 = new TestBlockBuilder()
       .addTx()