From bdc2a1f720752af9a61a6f2574a10d10c3637767 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Fri, 8 Dec 2023 11:18:43 +0000 Subject: [PATCH 1/3] Add @vercel/kv and @upstage/ratelimit packages --- package-lock.json | 82 +++++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 ++ 2 files changed, 84 insertions(+) diff --git a/package-lock.json b/package-lock.json index 3499117..92ab7e8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,9 @@ "@heroicons/react": "^2.0.18", "@prisma/client": "^4.14.0", "@supabase/supabase-js": "^2.24.0", + "@upstash/ratelimit": "^1.0.0", "@vercel/analytics": "^1.0.1", + "@vercel/kv": "^1.0.1", "@vercel/og": "^0.5.4", "@vercel/postgres": "^0.3.0", "eslint": "8.29.0", @@ -899,11 +901,49 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@upstash/core-analytics": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/@upstash/core-analytics/-/core-analytics-0.0.6.tgz", + "integrity": "sha512-cpPSR0XJAJs4Ddz9nq3tINlPS5aLfWVCqhhtHnXt4p7qr5+/Znlt1Es736poB/9rnl1hAHrOsOvVj46NEXcVqA==", + "dependencies": { + "@upstash/redis": "^1.19.3" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@upstash/ratelimit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@upstash/ratelimit/-/ratelimit-1.0.0.tgz", + "integrity": "sha512-IUlc+UKuHJQqXGQi7E3UKV0CKfRZ0g6qeNnbSve940k3xLoM+X3H45MrSwDaH7meMWemztLs0Q4yjkd3ItddCw==", + "dependencies": { + "@upstash/core-analytics": "^0.0.6" + } + }, + "node_modules/@upstash/redis": { + "version": "1.25.1", + "resolved": "https://registry.npmjs.org/@upstash/redis/-/redis-1.25.1.tgz", + "integrity": "sha512-ACj0GhJ4qrQyBshwFgPod6XufVEfKX2wcaihsEvSdLYnY+m+pa13kGt1RXm/yTHKf4TQi/Dy2A8z/y6WUEOmlg==", + "dependencies": { + "crypto-js": "^4.2.0" + } + }, "node_modules/@vercel/analytics": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@vercel/analytics/-/analytics-1.0.1.tgz", "integrity": "sha512-Ux0c9qUfkcPqng3vrR0GTrlQdqNJ2JREn/2ydrVuKwM3RtMfF2mWX31Ijqo1opSjNAq6rK76PwtANw6kl6TAow==" }, + "node_modules/@vercel/kv": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@vercel/kv/-/kv-1.0.1.tgz", + "integrity": "sha512-uTKddsqVYS2GRAM/QMNNXCTuw9N742mLoGRXoNDcyECaxEXvIHG0dEY+ZnYISV4Vz534VwJO+64fd9XeSggSKw==", + "dependencies": { + "@upstash/redis": "1.25.1" + }, + "engines": { + "node": ">=14.6" + } + }, "node_modules/@vercel/og": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/@vercel/og/-/og-0.5.4.tgz", @@ -1504,6 +1544,11 @@ "node": ">= 8" } }, + "node_modules/crypto-js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==" + }, "node_modules/css-background-parser": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/css-background-parser/-/css-background-parser-0.1.0.tgz", @@ -5531,11 +5576,43 @@ "eslint-visitor-keys": "^3.3.0" } }, + "@upstash/core-analytics": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/@upstash/core-analytics/-/core-analytics-0.0.6.tgz", + "integrity": "sha512-cpPSR0XJAJs4Ddz9nq3tINlPS5aLfWVCqhhtHnXt4p7qr5+/Znlt1Es736poB/9rnl1hAHrOsOvVj46NEXcVqA==", + "requires": { + "@upstash/redis": "^1.19.3" + } + }, + "@upstash/ratelimit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@upstash/ratelimit/-/ratelimit-1.0.0.tgz", + "integrity": "sha512-IUlc+UKuHJQqXGQi7E3UKV0CKfRZ0g6qeNnbSve940k3xLoM+X3H45MrSwDaH7meMWemztLs0Q4yjkd3ItddCw==", + "requires": { + "@upstash/core-analytics": "^0.0.6" + } + }, + "@upstash/redis": { + "version": "1.25.1", + "resolved": "https://registry.npmjs.org/@upstash/redis/-/redis-1.25.1.tgz", + "integrity": "sha512-ACj0GhJ4qrQyBshwFgPod6XufVEfKX2wcaihsEvSdLYnY+m+pa13kGt1RXm/yTHKf4TQi/Dy2A8z/y6WUEOmlg==", + "requires": { + "crypto-js": "^4.2.0" + } + }, "@vercel/analytics": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@vercel/analytics/-/analytics-1.0.1.tgz", "integrity": "sha512-Ux0c9qUfkcPqng3vrR0GTrlQdqNJ2JREn/2ydrVuKwM3RtMfF2mWX31Ijqo1opSjNAq6rK76PwtANw6kl6TAow==" }, + "@vercel/kv": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@vercel/kv/-/kv-1.0.1.tgz", + "integrity": "sha512-uTKddsqVYS2GRAM/QMNNXCTuw9N742mLoGRXoNDcyECaxEXvIHG0dEY+ZnYISV4Vz534VwJO+64fd9XeSggSKw==", + "requires": { + "@upstash/redis": "1.25.1" + } + }, "@vercel/og": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/@vercel/og/-/og-0.5.4.tgz", @@ -5955,6 +6032,11 @@ "which": "^2.0.1" } }, + "crypto-js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==" + }, "css-background-parser": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/css-background-parser/-/css-background-parser-0.1.0.tgz", diff --git a/package.json b/package.json index 0371f36..23bd3a1 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,9 @@ "@heroicons/react": "^2.0.18", "@prisma/client": "^4.14.0", "@supabase/supabase-js": "^2.24.0", + "@upstash/ratelimit": "^1.0.0", "@vercel/analytics": "^1.0.1", + "@vercel/kv": "^1.0.1", "@vercel/og": "^0.5.4", "@vercel/postgres": "^0.3.0", "eslint": "8.29.0", From 9ee12726d15ccecb269ee99ab90e081c60fa3ee7 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Fri, 8 Dec 2023 11:19:04 +0000 Subject: [PATCH 2/3] Add ratelimiting for api routes --- middleware.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 middleware.ts diff --git a/middleware.ts b/middleware.ts new file mode 100644 index 0000000..5182850 --- /dev/null +++ b/middleware.ts @@ -0,0 +1,35 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { Ratelimit } from '@upstash/ratelimit'; +import { kv } from '@vercel/kv'; + +const ratelimit = new Ratelimit({ + redis: kv, + // 20 requests from the same IP within a 10 second sliding window + limiter: Ratelimit.slidingWindow(20, '10s'), + prefix: `v2/zoo/ratelimit/${process.env.VERCEL_ENV ?? 'local'}`, +}); + +// Rate limit the /api/predictions/[id] endpoint +export const config = { + matcher: ['/api/predictions/:path+'], +}; + +export default async function middleware(request: NextRequest) { + if (!process.env.VERCEL_ENV || !process.env.KV_REST_API_URL || !process.env.KV_REST_API_URL) { + console.warn('Skipping ratelimiting middleware'); + return NextResponse.next(); + } + + const ip = request.ip ?? '127.0.0.1'; + const { success, limit, remaining, reset } = await ratelimit.limit(ip); + const headers = { + 'X-Ratelimit-Hit': String(!success), + 'X-Ratelimit-Limit': String(limit), + 'X-Ratelimit-Remaining': String(remaining), + 'X-Ratelimit-Reset': String(reset), + } + + return success + ? NextResponse.next({headers}) + : NextResponse.json({}, { status: 429, headers }); +} From 3c00ccb9cd03f5a80be873c3c42b0988ffeb4729 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Fri, 8 Dec 2023 12:58:29 +0000 Subject: [PATCH 3/3] Improve requests to polling endpoint --- middleware.ts | 1 + pages/index.js | 33 +++++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/middleware.ts b/middleware.ts index 5182850..55019e4 100644 --- a/middleware.ts +++ b/middleware.ts @@ -7,6 +7,7 @@ const ratelimit = new Ratelimit({ // 20 requests from the same IP within a 10 second sliding window limiter: Ratelimit.slidingWindow(20, '10s'), prefix: `v2/zoo/ratelimit/${process.env.VERCEL_ENV ?? 'local'}`, + timeout: 500, }); // Rate limit the /api/predictions/[id] endpoint diff --git a/pages/index.js b/pages/index.js index e44bbfa..b1dcf1d 100644 --- a/pages/index.js +++ b/pages/index.js @@ -36,7 +36,9 @@ export default function Home({ baseUrl, submissionPredictions }) { const response = await fetch(`/api/submissions/${seed}`, { method: "GET", }); - submissionPredictions = await response.json(); + if (response.ok) { + submissionPredictions = await response.json(); + } setPredictions(submissionPredictions); // get the model names from the predictions, and update which ones are checked @@ -151,17 +153,36 @@ export default function Home({ baseUrl, submissionPredictions }) { throw new Error(prediction.detail); } + // Add incremental backoff for polling requests. + const backoff = [250, 500, 500, 750, 1000, 1500, 3000, 5000, 10000, 15000, 30000]; + while ( prediction.status !== "succeeded" && prediction.status !== "failed" ) { - await sleep(500); + const jitter = random(0, 100); // Don't make all requests at the same time. + const delay = backoff.shift(); + if (!delay) { + // We've exceeded our timeout. + // TODO: Better user facing messaging here. + break; + } + + await sleep(delay + jitter); const response = await fetch("/api/predictions/" + prediction.id); - prediction = await response.json(); - console.log(prediction); + + // Handle Rate Limiting + if (response.status === 429) { + const reset = response.headers.get('X-Ratelimit-Reset') ?? Date.now() + 10_000; + const wait = reset - Date.now(); + await sleep(wait) + continue; + } + if (response.status !== 200) { throw new Error(prediction.detail); } + prediction = await response.json(); } prediction.model = model.name; @@ -551,3 +572,7 @@ export async function getServerSideProps({ req }) { return { props: { baseUrl, submissionPredictions } }; } + +function random(min, max) { + return Math.floor(Math.random() * (max - min + 1) + min) +}