fix: handle google crawler

Author: velopert

package.json

@@ -20,6 +20,7 @@
     "@testing-library/react": "^9.3.2",
     "@testing-library/user-event": "^7.1.2",
     "@types/aws-lambda": "^8.10.39",
+    "@types/cidr-matcher": "^2.1.2",
     "@types/codemirror": "^0.0.84",
     "@types/date-fns": "^2.6.0",
     "@types/ioredis": "^4.14.3",
@@ -63,6 +64,7 @@
     "babel-preset-react-app": "^9.1.0",
     "camelcase": "^5.3.1",
     "case-sensitive-paths-webpack-plugin": "2.2.0",
+    "cidr-matcher": "^2.1.1",
     "codemirror": "^5.51.0",
     "copy-webpack-plugin": "^5.1.1",
     "core-js": "^3.6.4",

src/server/crawlerCheck.ts

@@ -0,0 +1,144 @@
+// googlebot-and-userfetcher-ip-cache.ts
+import axios from 'axios';
+import CidrMatcher from 'cidr-matcher';
+
+export interface IPRangeDoc {
+  prefixes: {
+    ipv4Prefix?: string;
+    ipv6Prefix?: string;
+    service: string; // e.g., "googlebot", "user-triggered-fetchers"
+    scope: string; // e.g., "global"
+  }[];
+  creationTime?: string;
+}
+
+type CachePack = {
+  fetchedAt: number;
+  googlebot: IPRangeDoc;
+  userFetchers: IPRangeDoc;
+  cidrs: string[];
+  matcher: CidrMatcher;
+};
+
+export default class GoogleCrawlIPCache {
+  private static readonly URL_GOOGLEBOT =
+    'https://developers.google.com/search/apis/ipranges/googlebot.json';
+  private static readonly URL_USER_FETCHERS =
+    'https://developers.google.com/search/apis/ipranges/user-triggered-fetchers-google.json';
+  private static readonly ONE_HOUR = 60 * 60 * 1000;
+
+  private cache: CachePack | null = null;
+  private inflight: Promise<CachePack> | null = null;
+  private cacheDurationMs: number;
+
+  constructor(cacheDurationMs: number = GoogleCrawlIPCache.ONE_HOUR) {
+    this.cacheDurationMs = cacheDurationMs;
+  }
+
+  /** 통합 CIDR 매처로 검사 (IPv4/IPv6 지원) */
+  public async isCrawlerIP(ip: string): Promise<boolean> {
+    const pack = await this.getCachePack();
+    return pack.matcher.contains(ip);
+  }
+
+  /** 통합 CIDR 목록 반환 */
+  public async getAllCIDRs(): Promise<string[]> {
+    const pack = await this.getCachePack();
+    return pack.cidrs.slice();
+  }
+
+  /** 원본 두 문서 반환 */
+  public async getRawDocs(): Promise<{
+    googlebot: IPRangeDoc;
+    userFetchers: IPRangeDoc;
+  }> {
+    const pack = await this.getCachePack();
+    return { googlebot: pack.googlebot, userFetchers: pack.userFetchers };
+  }
+
+  /** 강제 무효화 */
+  public invalidate(): void {
+    this.cache = null;
+  }
+
+  /** 마지막 갱신 시각(ms) */
+  public getLastFetched(): number | null {
+    return this.cache?.fetchedAt ?? null;
+  }
+
+  /** 남은 TTL(ms). 캐시 없으면 0 */
+  public getTtlRemainingMs(): number {
+    if (!this.cache) return 0;
+    const remain = this.cache.fetchedAt + this.cacheDurationMs - Date.now();
+    return Math.max(0, remain);
+  }
+
+  // 내부 구현부
+
+  private async getCachePack(): Promise<CachePack> {
+    const now = Date.now();
+
+    if (this.cache && now - this.cache.fetchedAt < this.cacheDurationMs) {
+      return this.cache;
+    }
+
+    if (this.inflight) {
+      return this.inflight;
+    }
+
+    this.inflight = this.fetchBoth()
+      .then((pack) => {
+        this.cache = pack;
+        return pack;
+      })
+      .finally(() => {
+        this.inflight = null;
+      });
+
+    return this.inflight;
+  }
+
+  private async fetchBoth(): Promise<CachePack> {
+    const headers = {
+      'User-Agent': 'GoogleCrawlIPCache/1.0',
+      Accept: 'application/json',
+    };
+
+    const [gbRes, ufRes] = await Promise.all([
+      axios.get<IPRangeDoc>(GoogleCrawlIPCache.URL_GOOGLEBOT, {
+        headers,
+        timeout: 15000,
+      }),
+      axios.get<IPRangeDoc>(GoogleCrawlIPCache.URL_USER_FETCHERS, {
+        headers,
+        timeout: 15000,
+      }),
+    ]);
+
+    const googlebot = gbRes.data;
+    const userFetchers = ufRes.data;
+
+    // 두 문서의 CIDR을 합쳐서 매처 구성
+    const cidrs: string[] = [];
+    for (const p of googlebot.prefixes || []) {
+      if (p.ipv4Prefix) cidrs.push(p.ipv4Prefix);
+      if (p.ipv6Prefix) cidrs.push(p.ipv6Prefix);
+    }
+    for (const p of userFetchers.prefixes || []) {
+      if (p.ipv4Prefix) cidrs.push(p.ipv4Prefix);
+      if (p.ipv6Prefix) cidrs.push(p.ipv6Prefix);
+    }
+
+    // 중복 제거(간단 Set)
+    const uniqueCidrs = Array.from(new Set(cidrs));
+    const matcher = new CidrMatcher(uniqueCidrs);
+
+    return {
+      fetchedAt: Date.now(),
+      googlebot,
+      userFetchers,
+      cidrs: uniqueCidrs,
+      matcher,
+    };
+  }
+}

src/server/rateLimitMiddleware.ts

@@ -1,5 +1,22 @@
 import { Middleware } from 'koa';
 import { redis } from './CacheManager';
+import GoogleBotIPCache from './crawlerCheck';
+
+const cache = new GoogleBotIPCache(); // 기본 TTL 1시간
+async function main() {
+  try {
+    const docs = await cache.getRawDocs();
+    console.log('[googlebot] prefixes:', docs.googlebot.prefixes.length);
+    console.log('[userFetchers] prefixes:', docs.userFetchers.prefixes.length);
+
+    const cidrs = await cache.getAllCIDRs();
+    console.log('Total unique CIDRs:', cidrs.length);
+  } catch (e) {
+    console.error('[GoogleCrawlIPCache] error:', e);
+  }
+}
+
+main();
 
 const parseNumber = (value: string | null) => {
   if (value === null) return null;
@@ -12,13 +29,25 @@ const WHITELIST_IPS = (process.env.REACT_APP_WHITELIST_IPS ?? '')
   .split(',')
   .map((ip) => ip.trim());
 
+console.log({ WHITELIST_IPS });
+
 const rateLimitMiddleware: Middleware = async (ctx, next) => {
   const ip = ctx.request.ips.slice(-1)[0] || ctx.request.ip;
 
-  if (WHITELIST_IPS.some((whitelistIp) => ip.includes(whitelistIp))) {
+  if (
+    WHITELIST_IPS.length > 0 &&
+    WHITELIST_IPS.some((whitelistIp) => ip.includes(whitelistIp))
+  ) {
     return next();
   }
 
+  const isCrawler = await cache.isCrawlerIP(ip);
+  if (isCrawler) {
+    return next();
+  } else {
+    console.log(`[${ip}] is not a crawler`);
+  }
+
   const isBlockedUrl = await redis.get(`${ctx.url}:blocked`);
   if (isBlockedUrl === '1') {
     ctx.status = 429;

yarn.lock

@@ -1709,6 +1709,11 @@
     "@types/connect" "*"
     "@types/node" "*"
 
+"@types/cidr-matcher@^2.1.2":
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/@types/cidr-matcher/-/cidr-matcher-2.1.2.tgz#0ac8f46a1b11040b237907cd0255c3179b9c302a"
+  integrity sha512-tuyfj7hfDCBolTk+TDPC6OA6YlKIS3dpvYDGNKzRMVq/wdrl6LD5MDqRoVKPhJ+4nuOrBz2yKnzclzy7t28NyA==
+
 "@types/codemirror@^0.0.84":
   version "0.0.84"
   resolved "https://registry.yarnpkg.com/@types/codemirror/-/codemirror-0.0.84.tgz#b0cfca79ccdfd45ffe1f737668276a31b3149ebd"
@@ -3751,6 +3756,13 @@ ci-info@^2.0.0:
   resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46"
   integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==
 
+cidr-matcher@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/cidr-matcher/-/cidr-matcher-2.1.1.tgz#01a489f291bfbc7a3a14358120a6d839a98b1c90"
+  integrity sha512-QPJRz4HDQxpB8AZWEqd6ejVp+siArXh3u1MYaUFV85cd293StGSMb87jVe0z9gS92KsFwxCxjb3utO3e5HKHTw==
+  dependencies:
+    ip6addr "^0.2.2"
+
 cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/cipher-base/-/cipher-base-1.0.4.tgz#8760e4ecc272f4c363532f926d874aae2c1397de"
@@ -7240,6 +7252,14 @@ ip-regex@^2.1.0:
   resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9"
   integrity sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=
 
+ip6addr@^0.2.2:
+  version "0.2.5"
+  resolved "https://registry.yarnpkg.com/ip6addr/-/ip6addr-0.2.5.tgz#06e134f44b4e1a684fd91b24035dca7a53b8f759"
+  integrity sha512-9RGGSB6Zc9Ox5DpDGFnJdIeF0AsqXzdH+FspCfPPaU/L/4tI6P+5lIoFUFm9JXs9IrJv1boqAaNCQmoDADTSKQ==
+  dependencies:
+    assert-plus "^1.0.0"
+    jsprim "^2.0.2"
+
 ip@^1.1.0, ip@^1.1.5:
   version "1.1.5"
   resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a"
@@ -8207,6 +8227,11 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13"
   integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=
 
+[email protected]:
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5"
+  integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==
+
 json-stable-stringify-without-jsonify@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651"
@@ -8265,6 +8290,16 @@ jsprim@^1.2.2:
     json-schema "0.2.3"
     verror "1.10.0"
 
+jsprim@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-2.0.2.tgz#77ca23dbcd4135cd364800d22ff82c2185803d4d"
+  integrity sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ==
+  dependencies:
+    assert-plus "1.0.0"
+    extsprintf "1.3.0"
+    json-schema "0.4.0"
+    verror "1.10.0"
+
 jsx-ast-utils@^2.2.1:
   version "2.2.3"
   resolved "https://registry.yarnpkg.com/jsx-ast-utils/-/jsx-ast-utils-2.2.3.tgz#8a9364e402448a3ce7f14d357738310d9248054f"