Skip to content

Commit ed6d0d1

Browse files
realfishsamrealfishsam
committed
feat: increase default limit to 10,000 and optimize search pagination
- Updated default limit to 10,000 for markets and events in all exchanges - Updated OpenAPI spec with new default limits - Improved Polymarket search with parallel pagination and robust filtering - Updated examples and JSDoc documentation - Added search verification script
1 parent 9c4374c commit ed6d0d1

10 files changed

Lines changed: 159 additions & 54 deletions

File tree

‎core/src/BaseExchange.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,14 +143,14 @@ export abstract class PredictionMarketExchange {
143143
* @returns Array of unified markets
144144
*
145145
* @example-ts Fetch markets
146-
* const markets = await exchange.fetchMarkets({ query: 'Trump', limit: 20 });
146+
* const markets = await exchange.fetchMarkets({ query: 'Trump', limit: 10000 });
147147
* console.log(markets[0].title);
148148
*
149149
* @example-ts Get market by slug
150150
* const markets = await exchange.fetchMarkets({ slug: 'will-trump-win' });
151151
*
152152
* @example-python Fetch markets
153-
* markets = exchange.fetch_markets(query='Trump', limit=20)
153+
* markets = exchange.fetch_markets(query='Trump', limit=10000)
154154
* print(markets[0].title)
155155
*
156156
* @example-python Get market by slug

‎core/src/exchanges/kalshi/fetchEvents.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ export async function fetchEvents(params: EventFetchParams): Promise<UnifiedEven
5050
return unifiedEvent;
5151
});
5252

53-
const limit = params?.limit || 20;
53+
const limit = params?.limit || 10000;
5454
return unifiedEvents.slice(0, limit);
5555

5656
} catch (error: any) {

‎core/src/exchanges/kalshi/fetchMarkets.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ async function fetchMarketsBySlug(eventTicker: string): Promise<UnifiedMarket[]>
156156

157157
async function searchMarkets(query: string, params?: MarketFetchParams): Promise<UnifiedMarket[]> {
158158
// We must fetch ALL markets to search them locally since we don't have server-side search
159-
const searchLimit = 5000;
159+
const searchLimit = 10000;
160160
const markets = await fetchMarketsDefault({ ...params, limit: searchLimit });
161161
const lowerQuery = query.toLowerCase();
162162
const searchIn = params?.searchIn || 'title'; // Default to title-only search
@@ -170,12 +170,12 @@ async function searchMarkets(query: string, params?: MarketFetchParams): Promise
170170
return titleMatch || descMatch; // 'both'
171171
});
172172

173-
const limit = params?.limit || 20;
173+
const limit = params?.limit || 10000;
174174
return filtered.slice(0, limit);
175175
}
176176

177177
async function fetchMarketsDefault(params?: MarketFetchParams): Promise<UnifiedMarket[]> {
178-
const limit = params?.limit || 50;
178+
const limit = params?.limit || 10000;
179179
const offset = params?.offset || 0;
180180
const now = Date.now();
181181
const status = params?.status || 'active'; // Default to 'active'

‎core/src/exchanges/limitless/fetchEvents.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ export async function fetchEvents(params: EventFetchParams): Promise<UnifiedEven
1212
const response = await axios.get(`${LIMITLESS_API_URL}/markets/search`, {
1313
params: {
1414
query: params.query,
15-
limit: params?.limit || 20
15+
limit: params?.limit || 10000
1616
}
1717
});
1818

‎core/src/exchanges/limitless/fetchMarkets.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ async function searchMarkets(
6262
const response = await axios.get(`${LIMITLESS_API_URL}/markets/search`, {
6363
params: {
6464
query: query,
65-
limit: params?.limit || 20,
65+
limit: params?.limit || 10000,
6666
page: params?.page || 1,
6767
similarityThreshold: params?.similarityThreshold || 0.5
6868
}
@@ -86,14 +86,14 @@ async function searchMarkets(
8686

8787
return allMarkets
8888
.filter((m: any): m is UnifiedMarket => m !== null && m.outcomes.length > 0)
89-
.slice(0, params?.limit || 20);
89+
.slice(0, params?.limit || 10000);
9090
}
9191

9292
async function fetchMarketsDefault(
9393
marketFetcher: MarketFetcher,
9494
params?: MarketFetchParams
9595
): Promise<UnifiedMarket[]> {
96-
const limit = params?.limit || 200;
96+
const limit = params?.limit || 10000;
9797
const offset = params?.offset || 0;
9898

9999
// Map sort parameter to SDK's sortBy

‎core/src/exchanges/polymarket/fetchEvents.ts‎

Lines changed: 30 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,56 @@
11
import { EventFetchParams } from '../../BaseExchange';
22
import { UnifiedEvent, UnifiedMarket } from '../../types';
3-
import { GAMMA_API_URL, mapMarketToUnified, paginateParallel } from './utils';
3+
import { GAMMA_SEARCH_URL, mapMarketToUnified, paginateSearchParallel } from './utils';
44
import { polymarketErrorMapper } from './errors';
55

66
export async function fetchEvents(params: EventFetchParams): Promise<UnifiedEvent[]> {
7-
const searchLimit = 100000; // Fetch all events for comprehensive search
8-
97
try {
10-
const status = params?.status || 'active';
8+
if (!params.query) {
9+
// If no query is provided, we can't use the search endpoint effectively.
10+
// However, the BaseExchange interface enforces query presence for fetchEvents.
11+
// Just in case, we return empty or throw.
12+
throw new Error("Query is required for Polymarket event search");
13+
}
14+
15+
const limit = params.limit || 10000;
16+
const status = params.status || 'active';
17+
1118
const queryParams: any = {
12-
limit: searchLimit
19+
q: params.query,
20+
limit_per_type: 50, // Fetch 50 per page for better efficiency
21+
events_status: status === 'all' ? undefined : status,
22+
sort: 'volume',
23+
ascending: false
1324
};
1425

26+
// If specific status requested
1527
if (status === 'active') {
16-
queryParams.active = 'true';
17-
queryParams.closed = 'false';
28+
queryParams.events_status = 'active';
1829
} else if (status === 'closed') {
19-
queryParams.active = 'false';
20-
queryParams.closed = 'true';
21-
} else {
22-
// 'all' - no filter, maybe handled by default or API behavior
30+
queryParams.events_status = 'closed';
2331
}
2432

25-
// Fetch events from Gamma API using parallel pagination
26-
const events = await paginateParallel(GAMMA_API_URL, queryParams);
33+
// Use parallel pagination to fetch all matching events
34+
const events = await paginateSearchParallel(GAMMA_SEARCH_URL, queryParams, limit * 10);
2735

28-
// Client-side text filtering
29-
const lowerQuery = (params?.query || '').toLowerCase();
30-
const searchIn = params?.searchIn || 'title';
36+
// Client-side filtering to ensure title matches (API does fuzzy search)
37+
const lowerQuery = params.query.toLowerCase();
38+
const searchIn = params.searchIn || 'title';
3139

32-
const filtered = events.filter((event: any) => {
40+
const filteredEvents = events.filter((event: any) => {
3341
const titleMatch = (event.title || '').toLowerCase().includes(lowerQuery);
3442
const descMatch = (event.description || '').toLowerCase().includes(lowerQuery);
3543

3644
if (searchIn === 'title') return titleMatch;
3745
if (searchIn === 'description') return descMatch;
38-
return titleMatch || descMatch;
46+
return titleMatch || descMatch; // 'both'
3947
});
4048

41-
// Map to UnifiedEvent
42-
const unifiedEvents: UnifiedEvent[] = filtered.map((event: any) => {
49+
// Map events to UnifiedEvent
50+
const unifiedEvents: UnifiedEvent[] = filteredEvents.map((event: any) => {
4351
const markets: UnifiedMarket[] = [];
4452

45-
if (event.markets) {
53+
if (event.markets && Array.isArray(event.markets)) {
4654
for (const market of event.markets) {
4755
const unifiedMarket = mapMarketToUnified(event, market, { useQuestionAsCandidateFallback: true });
4856
if (unifiedMarket) {
@@ -66,9 +74,7 @@ export async function fetchEvents(params: EventFetchParams): Promise<UnifiedEven
6674
return unifiedEvent;
6775
});
6876

69-
// Apply limit to filtered results
70-
const limit = params?.limit || 20;
71-
return unifiedEvents.slice(0, limit);
77+
return unifiedEvents;
7278

7379
} catch (error: any) {
7480
throw polymarketErrorMapper.mapError(error);

‎core/src/exchanges/polymarket/fetchMarkets.ts‎

Lines changed: 39 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import axios from 'axios';
22
import { MarketFetchParams } from '../../BaseExchange';
33
import { UnifiedMarket } from '../../types';
4-
import { GAMMA_API_URL, mapMarketToUnified, paginateParallel } from './utils';
4+
import { GAMMA_API_URL, GAMMA_SEARCH_URL, mapMarketToUnified, paginateParallel, paginateSearchParallel } from './utils';
55
import { polymarketErrorMapper } from './errors';
66

77
export async function fetchMarkets(params?: MarketFetchParams): Promise<UnifiedMarket[]> {
@@ -47,34 +47,53 @@ async function fetchMarketsBySlug(slug: string): Promise<UnifiedMarket[]> {
4747
}
4848

4949
async function searchMarkets(query: string, params?: MarketFetchParams): Promise<UnifiedMarket[]> {
50-
const searchLimit = 5000; // Fetch enough markets for a good search pool
50+
const limit = params?.limit || 10000;
51+
52+
// Use parallel pagination to fetch all matching events
53+
// Each event can contain multiple markets, so we need a larger pool
54+
const queryParams: any = {
55+
q: query,
56+
limit_per_type: 50, // Fetch 50 events per page
57+
events_status: params?.status === 'all' ? undefined : (params?.status || 'active'),
58+
sort: 'volume',
59+
ascending: false
60+
};
5161

52-
// Fetch markets with a higher limit
53-
const markets = await fetchMarketsDefault({
54-
...params,
55-
limit: searchLimit
56-
});
62+
// Fetch events with parallel pagination
63+
const events = await paginateSearchParallel(GAMMA_SEARCH_URL, queryParams, limit * 5);
5764

58-
// Client-side text filtering
65+
const unifiedMarkets: UnifiedMarket[] = [];
5966
const lowerQuery = query.toLowerCase();
60-
const searchIn = params?.searchIn || 'title'; // Default to title-only search
67+
const searchIn = params?.searchIn || 'title';
6168

62-
const filtered = markets.filter(market => {
63-
const titleMatch = (market.title || '').toLowerCase().includes(lowerQuery);
64-
const descMatch = (market.description || '').toLowerCase().includes(lowerQuery);
69+
// Flatten events into markets
70+
for (const event of events) {
71+
if (!event.markets) continue;
6572

66-
if (searchIn === 'title') return titleMatch;
67-
if (searchIn === 'description') return descMatch;
68-
return titleMatch || descMatch; // 'both'
69-
});
73+
for (const market of event.markets) {
74+
const unifiedMarket = mapMarketToUnified(event, market, { useQuestionAsCandidateFallback: true });
75+
if (!unifiedMarket) continue;
76+
77+
// Apply client-side filtering on market title
78+
const titleMatch = (unifiedMarket.title || '').toLowerCase().includes(lowerQuery);
79+
const descMatch = (unifiedMarket.description || '').toLowerCase().includes(lowerQuery);
80+
81+
let matches = false;
82+
if (searchIn === 'title') matches = titleMatch;
83+
else if (searchIn === 'description') matches = descMatch;
84+
else matches = titleMatch || descMatch;
85+
86+
if (matches) {
87+
unifiedMarkets.push(unifiedMarket);
88+
}
89+
}
90+
}
7091

71-
// Apply limit to filtered results
72-
const limit = params?.limit || 20;
73-
return filtered.slice(0, limit);
92+
return unifiedMarkets.slice(0, limit);
7493
}
7594

7695
async function fetchMarketsDefault(params?: MarketFetchParams): Promise<UnifiedMarket[]> {
77-
const limit = params?.limit || 200; // Higher default for better coverage
96+
const limit = params?.limit || 10000; // Higher default for better coverage
7897
const offset = params?.offset || 0;
7998

8099
// Map generic sort params to Polymarket Gamma API params

‎core/src/exchanges/polymarket/utils.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { UnifiedMarket, MarketOutcome, CandleInterval } from '../../types';
22
import { addBinaryOutcomes } from '../../utils/market-utils';
33

44
export const GAMMA_API_URL = 'https://gamma-api.polymarket.com/events';
5+
export const GAMMA_SEARCH_URL = 'https://gamma-api.polymarket.com/public-search';
56
export const CLOB_API_URL = 'https://clob.polymarket.com';
67
export const DATA_API_URL = 'https://data-api.polymarket.com';
78

@@ -149,3 +150,50 @@ export async function paginateParallel(url: string, params: any, maxResults: num
149150

150151
return [firstPage, ...remainingPages].flat();
151152
}
153+
154+
/**
155+
* Fetch all results from Gamma public-search API using parallel pagination.
156+
* Uses 'page' parameter instead of 'offset'.
157+
*/
158+
export async function paginateSearchParallel(url: string, params: any, maxResults: number = 10000): Promise<any[]> {
159+
const axios = (await import('axios')).default;
160+
161+
// 1. Fetch the first page to check pagination info
162+
const firstPageResponse = await axios.get(url, {
163+
params: { ...params, page: 1 }
164+
});
165+
166+
const data = firstPageResponse.data;
167+
const firstPageEvents = data.events || [];
168+
const pagination = data.pagination;
169+
170+
// If no more pages, return what we have
171+
if (!pagination?.hasMore || firstPageEvents.length === 0) {
172+
return firstPageEvents;
173+
}
174+
175+
// 2. Calculate how many pages to fetch based on totalResults and limit_per_type
176+
const limitPerType = params.limit_per_type || 20;
177+
const totalResults = Math.min(pagination.totalResults || 0, maxResults);
178+
const totalPages = Math.ceil(totalResults / limitPerType);
179+
180+
// Fetch remaining pages in parallel
181+
const pageNumbers = [];
182+
for (let i = 2; i <= totalPages; i++) {
183+
pageNumbers.push(i);
184+
}
185+
186+
const remainingPages = await Promise.all(pageNumbers.map(async (pageNum) => {
187+
try {
188+
const res = await axios.get(url, {
189+
params: { ...params, page: pageNum }
190+
});
191+
return res.data?.events || [];
192+
} catch (e) {
193+
return []; // Swallow individual page errors to be robust
194+
}
195+
}));
196+
197+
return [firstPageEvents, ...remainingPages].flat();
198+
}
199+

‎core/src/server/openapi.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1073,6 +1073,7 @@ components:
10731073
properties:
10741074
limit:
10751075
type: integer
1076+
default: 10000
10761077
offset:
10771078
type: integer
10781079
sort:
@@ -1101,6 +1102,7 @@ components:
11011102
type: string
11021103
limit:
11031104
type: integer
1105+
default: 10000
11041106
offset:
11051107
type: integer
11061108
status:

‎core/verify_search.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import { PolymarketExchange } from './src/exchanges/polymarket';
2+
3+
async function test() {
4+
const api = new PolymarketExchange();
5+
6+
try {
7+
const events = await api.fetchEvents({ query: 'Trump' });
8+
console.log(`Found ${events.length} events`);
9+
events.forEach(e => {
10+
console.log(`- Event: ${e.title} (ID: ${e.id})`);
11+
console.log(` Markets: ${e.markets.length}`);
12+
});
13+
} catch (e) {
14+
console.error('fetchEvents failed:', e);
15+
}
16+
17+
console.log('\nTesting fetchMarkets with query: "Fed"');
18+
try {
19+
const markets = await api.fetchMarkets({ query: 'Fed' });
20+
console.log(`Found ${markets.length} markets with "Fed" in the title`);
21+
console.log('Sample markets:');
22+
markets.slice(0, 5).forEach(m => {
23+
console.log(`- ${m.title}`);
24+
});
25+
} catch (e) {
26+
console.error('fetchMarkets failed:', e);
27+
}
28+
}
29+
30+
test();

0 commit comments

Comments
 (0)