[WIP] Prerender-node v4: Typescript

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,62 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+labels: ["bug", "triage"]
+assignees:
+  - octocat
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: contact
+    attributes:
+      label: Contact Details
+      description: How can we get in touch with you if we need more info?
+      placeholder: ex. [email protected]
+    validations:
+      required: false
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+      placeholder: Tell us what you see!
+      value: "A bug happened!"
+    validations:
+      required: true
+  - type: dropdown
+    id: version
+    attributes:
+      label: Version
+      description: What version of our software are you running?
+      options:
+        - 3.x (Legacy)
+        - 4.x (Default)
+    validations:
+      required: true
+  - type: dropdown
+    id: browsers
+    attributes:
+      label: What browsers are you seeing the problem on?
+      multiple: true
+      options:
+        - Firefox
+        - Chrome
+        - Safari
+        - Microsoft Edge
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      render: Shell
+  - type: checkboxes
+    id: terms
+    attributes:
+      label: Code of Conduct
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://example.com)
+      options:
+        - label: I agree to follow this project's Code of Conduct
+          required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Prerender.io Support
+    url: https://prerender.io
+    about: Prerender.io customers can contact us for support.

.gitignore

@@ -1,2 +1,3 @@
 node_modules
-.DS_Store
+.DS_Store
+lib

.prettierrc

@@ -0,0 +1,4 @@
+{
+  "semi": true,
+  "singleQuote": true
+}

README.md

@@ -17,6 +17,16 @@ And when you set up your express app, add:
 app.use(require('prerender-node'));
 ```
 
+You can use ESM syntax as well:
+
+```js
+import prerender from 'prerender-node';
+import express from 'express';
+
+const app = express();
+app.use(prerender);
+```
+
 or if you have an account on [prerender.io](https://prerender.io/) and want to use your token:
 
 ```js

package.json

@@ -1,10 +1,15 @@
 {
   "name": "prerender-node",
-  "version": "3.5.0",
+  "version": "4.0.0-beta.1",
   "description": "express middleware for serving prerendered javascript-rendered pages for SEO",
   "author": "Todd Hooper",
+  "authors": [
+    "Todd Hooper",
+    "PoOwAa <ray@prerender.io>"
+  ],
   "license": "MIT",
-  "main": "index.js",
+  "main": "./lib/cjs/index.js",
+  "types": "./lib/cjs/types/index.d.ts",
   "repository": {
     "type": "git",
     "url": "git://github.com/prerender/prerender-node"
@@ -13,16 +18,46 @@
     "angular",
     "backbone",
     "emberjs",
-    "seo"
+    "seo",
+    "react",
+    "prerender",
+    "express",
+    "fastify"
   ],
   "devDependencies": {
+    "@types/express": "^4.17.17",
+    "@types/node": "^18.14.6",
+    "express": "^4.18.2",
+    "fastify": "^4.15.0",
     "mocha": "^6.2.0",
     "nock": "^11.9.1",
     "request": "^2.88.2",
+    "rimraf": "^4.3.1",
     "sinon": "^7.3.1",
-    "supertest": "^4.0.2"
+    "supertest": "^4.0.2",
+    "typescript": "^4.9.5"
   },
   "scripts": {
+    "clean": "rimraf lib",
+    "build": "npm run clean && npm run build:cjs && npm run build:esm",
+    "build:cjs": "tsc --project tsconfig.cjs.json",
+    "build:esm": "tsc --project tsconfig.esm.json && mv lib/esm/index.js lib/esm/index.mjs",
+    "prepack": "npm run build",
     "test": "echo 'Make sure to unset PRERENDER_SERVICE_URL when running locally' && ./node_modules/.bin/mocha"
-  }
+  },
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./lib/esm/types/index.d.ts",
+        "default": "./lib/esm/index.mjs"
+      },
+      "require": {
+        "types": "./lib/cjs/types/index.d.ts",
+        "default": "./lib/cjs/index.js"
+      }
+    }
+  },
+  "files": [
+    "lib/**/*"
+  ]
 }

src/config.interface.ts

@@ -0,0 +1,118 @@
+export interface PrerenderAgentOptions {
+  /**
+   * List of crawler user agents. If the user agent is in this list,
+   * the request will be forwarded to the prerender server.
+   *
+   * @example
+   * crawlerUserAgents: [
+   *  'googlebot',
+   *  'bingbot',
+   *  'yandex',
+   *  'baiduspider',
+   * ]
+   */
+  crawlerUserAgents?: string[];
+
+  /**
+   * List of extensions to ignore. If the request url contains one of these extensions,
+   * the request will not be forwarded to the prerender server.
+   *
+   * @example
+   * extensionsToIgnore: [
+   *  '.js',
+   *  '.css',
+   *  '.xml',
+   *  '.jpg',
+   *  '.pdf',
+   * ];
+   */
+  extensionsToIgnore?: string[];
+
+  /**
+   * Prerender.io token. If you don't have a token, you can get one
+   * for free at https://prerender.io/
+   *
+   * When token is set, the request will be forwarded to the Prerender.io server.
+   * Otherwise should use self-hosted prerender server.
+   */
+  token?: string;
+
+  /**
+   * Prerender service url. By default it is set to https://service.prerender.io
+   * If you want to use self-hosted prerender server, you can set this option.
+   *
+   * @example
+   * serviceUrl: 'http://localhost:3000'
+   */
+  serviceUrl?: string;
+
+  /**
+   * List of resources to whitelist. If the request url contains one of these resources,
+   * the request will be forwarded to the prerender server.
+   *
+   * If you want to prerender all resources except a few,
+   * you can use the {@link blackList} option.
+   *
+   * @example
+   * whiteList: [
+   *  '/about',
+   *  '/assets/images/logo.png',
+   *  'robots.txt',
+   * ];
+   */
+  whiteList?: string[];
+
+  /**
+   * List of resources to blacklist. If the request url contains one of these resources,
+   * the request will not be forwarded to the prerender server.
+   *
+   * If you want to prerender a few by default forbidden resources,
+   * you can use the {@link whiteList} option.
+   */
+  blackList?: string[];
+
+  /**
+   * TODO: do we want to use this instead of protocol?
+   * Force https. If this option is set to true, the request will be forwarded to the
+   * prerender server with https protocol.
+   *
+   * @example
+   * forceHttps: true
+   */
+  // forceHttps?: boolean;
+
+  /**
+   * Request protocol. Option to hard-set the protocol.
+   * Useful for sites that are available on both http and https.
+   */
+  protocol?: 'http' | 'https';
+
+  /**
+   * Useful for sites that are behind a load balancer or internal reverse proxy.
+   * For example, your internal URL looks like `http://internal-host.com/` and you
+   * might want it to instead send a request to Prerender.io with your real domain
+   * in place of `internal-host.com`.
+   */
+  host?: string;
+
+  /**
+   * Option to forward headers from request to prerender.
+   *
+   * @example
+   * forwardHeaders: true
+   */
+  forwardHeaders?: boolean;
+
+  /**
+   * Prerender server request options. If you want to set some additional options
+   * for the request to the prerender server, you can use this option.
+   *
+   * @example
+   * prerenderServerRequestOptions: {
+   *   timeout: 1000,
+   * }
+   */
+  prerenderServerRequestOptions?: {
+    [key: string]: any;
+  };
+}

src/crawlerAgent.ts

@@ -0,0 +1,37 @@
+export const crawlerUserAgents = [
+  'googlebot',
+  'Yahoo! Slurp',
+  'bingbot',
+  'yandex',
+  'baiduspider',
+  'facebookexternalhit',
+  'twitterbot',
+  'rogerbot',
+  'linkedinbot',
+  'embedly',
+  'quora link preview',
+  'showyoubot',
+  'outbrain',
+  'pinterest/0.',
+  'developers.google.com/+/web/snippet',
+  'slackbot',
+  'vkShare',
+  'W3C_Validator',
+  'redditbot',
+  'Applebot',
+  'WhatsApp',
+  'flipboard',
+  'tumblr',
+  'bitlybot',
+  'SkypeUriPreview',
+  'nuzzel',
+  'Discordbot',
+  'Google Page Speed',
+  'Qwantify',
+  'pinterestbot',
+  'Bitrix link preview',
+  'XING-contenttabreceiver',
+  'Chrome-Lighthouse',
+  'TelegramBot',
+  'SeznamBot',
+];

src/extensionsToIgnore.ts

@@ -0,0 +1,131 @@
+/**
+ * List of extensions to ignore. These extensions won't be served by prerender.
+ * If you want to serve this type of files, please consider to use a CDN.
+ *
+ * TODO: Add more exact description
+ */
+export const extensionsToIgnore = [
+  '.js',
+  '.css',
+  '.xml',
+  '.less',
+  '.pdf',
+  '.txt',
+  '.rss',
+  '.zip',
+  '.rar',
+  '.exe',
+  '.wmv',
+  '.mov',
+  '.psd',
+  '.ai',
+  '.xls',
+  '.swf',
+  '.dat',
+  '.dmg',
+  '.iso',
+  '.flv',
+  '.torrent',
+  '.webmanifest',
+
+  /**
+   * Office documents
+   */
+  '.doc',
+  '.docx',
+  '.docm',
+  '.dot',
+  '.dotx',
+  '.dotm',
+  '.xls',
+  '.xlsx',
+  '.xlsm',
+  '.xls',
+  '.xltx',
+  '.xltm',
+  '.xlsb',
+  '.xlam',
+  '.ppt',
+  '.pptx',
+  '.pptm',
+  '.pps',
+  '.ppsx',
+  '.ppsm',
+  '.pot',
+  '.potx',
+  '.potm',
+  '.ppam',
+  '.sld',
+  '.sldx',
+  '.sldm',
+  '.onetoc',
+  '.onetoc2',
+  '.onetmp',
+  '.onepkg',
+  '.thmx',
+
+  /**
+   * Fonts
+   */
+  '.eot',
+  '.otf',
+  '.ttc',
+  '.ttf',
+  '.woff2',
+  '.woff',
+
+  /**
+   * Browser compatible media types
+   * @see https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Containers
+   */
+  '.3gp',
+  '.3g2',
+  '.3gpp',
+  '.3gpp2',
+  '.aac',
+  '.adts',
+  '.amr',
+  '.avi',
+  '.awb',
+  '.drc',
+  '.flac',
+  '.m4a',
+  '.m4b',
+  '.m4p',
+  '.m4v',
+  '.mogg',
+  '.mp3',
+  '.mp4',
+  '.mp4a',
+  '.mp4b',
+  '.mpc',
+  '.mpg',
+  '.mpeg',
+  '.ogg',
+  '.oga',
+  '.opus',
+  '.wav',
+  '.webm',
+
+  /**
+   * Browser compatible image types
+   * @see https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Image_types
+   *
+   */
+  '.apng',
+  '.avif',
+  '.gif',
+  '.jpg',
+  '.jpeg',
+  '.jfif',
+  '.pjpeg',
+  '.pjp',
+  '.png',
+  '.svg',
+  '.webp',
+  '.bmp',
+  '.ico',
+  '.cur',
+  '.tif',
+  '.tiff',
+];

src/index.ts

@@ -0,0 +1 @@
+// TODO: export the package public API

src/middlewares/express.ts

@@ -0,0 +1,61 @@
+import { Request, Response, NextFunction } from 'express';
+import PrerenderAgent, {
+  CachedRender,
+  PrerenderedPageResponse,
+} from '../PrerenderAgent';
+
+export async function prerenderExpressMiddleware(
+  agent: PrerenderAgent,
+  beforeRender: (req: Request, res: Response) => Promise<CachedRender>,
+  afterRender: (
+    err: Error | null,
+    req: Request,
+    prerenderedPageResponse?: PrerenderedPageResponse
+  ) => { cancelRender: boolean }
+) {
+  return async function (req: Request, res: Response, next: NextFunction) {
+    if (!agent.shouldShowPrerenderedPage(req.url, req.method, req.headers))
+      return next();
+
+    try {
+      const cachedRender = await beforeRender(req, res);
+
+      if (cachedRender) {
+        if (typeof cachedRender === 'string') {
+          res.writeHead(200, { 'Content-Type': 'text/html' });
+          return res.end(cachedRender);
+        } else if (typeof cachedRender === 'object') {
+          res.writeHead(cachedRender.status || 200, {
+            'Content-Type': 'text/html',
+          });
+          return res.end(cachedRender.body || '');
+        }
+      }
+    } catch (error) {
+      // this is a user-defined function, so we don't want to
+      // do anything with the error
+      throw error;
+    }
+
+    try {
+      const prerenderedPageResponse = await agent.getPrerenderedPageResponse(
+        req.url,
+        req.headers
+      );
+
+      const options = afterRender(null, req, prerenderedPageResponse);
+      if (options && options.cancelRender) return next();
+
+      res.writeHead(
+        prerenderedPageResponse.statusCode,
+        prerenderedPageResponse.headers
+      );
+      return res.end(prerenderedPageResponse.body);
+    } catch (error) {
+      const options = afterRender(error, req);
+      if (options && options.cancelRender) return next();
+
+      next(error);
+    }
+  };
+}

src/middlewares/fastify.ts

@@ -0,0 +1 @@
+// TODO: implement fastify plugin

src/util.ts

@@ -0,0 +1,80 @@
+import url from "url";
+import { crawlerUserAgents } from "./crawlerAgent";
+
+const isResourceWhitelisted = (url: string, whitelist: string[] = []): boolean => {
+  return whitelist.every((whitelistedUrl: string) => {
+    return new RegExp(whitelistedUrl).test(url);
+  });
+}
+
+const isResourceBlacklisted = (url: string, blacklist: string[] = [], referer: string): boolean => {
+  return blacklist.some((blacklistedItem: string) => {
+    const regex = new RegExp(blacklistedItem);
+
+    const blacklistedUrl = regex.test(url);
+    const blacklistedReferer = regex.test(referer);
+
+    return blacklistedUrl || blacklistedReferer;
+  });
+}
+
+export const shouldShowPrerenderedPage = (request: {
+  method: 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'CONNECT' | 'OPTIONS' | 'TRACE' | 'PATCH',
+  headers: {
+    [key: string]: string,
+  },
+  url: string,
+}, options: {
+  extensionsToIgnore?: string[],
+  whitelist?: string[],
+  blacklist?: string[],
+}): boolean => {
+  const { method, headers } = request;
+  const { extensionsToIgnore, whitelist, blacklist } = options;
+  const userAgent = headers['user-agent'];
+  const bufferAgent = headers['x-bufferbot'] ?? '';
+
+  if (!userAgent) return false;
+  if (method !== 'GET' && method !== 'HEAD') return false;
+
+  // We don't want to serve prerender requests, it will cause an infinite loop
+  if (headers && headers['x-prerender']) return false;
+
+  let isRequestingPrerenderedPage = false;
+
+  const parsedUrl = url.parse(request.url, true)
+  const parsedQuery = parsedUrl.query;
+
+  // If the query contains _escaped_fragment_, show the prerendered page
+  if (parsedQuery && parsedQuery._escaped_fragment_) {
+    isRequestingPrerenderedPage = true;
+  }
+
+  // If the user agent is a bot, show the prerendered page
+  if (crawlerUserAgents.some((crawlerUserAgent) => userAgent.toLowerCase().includes(crawlerUserAgent.toLowerCase()))) {
+    isRequestingPrerenderedPage = true;
+  }
+
+  // If the user agent is Buffer, show the prerendered page
+  if (bufferAgent) {
+    isRequestingPrerenderedPage = true;
+  }
+
+  // If the request is for a resource file, don't show the prerendered page
+  const parsedPathName = parsedUrl.pathname?.toLowerCase();
+  if (extensionsToIgnore && extensionsToIgnore.some((extensionToIgnore: string) => parsedPathName?.endsWith(extensionToIgnore))) {
+    return false;
+  }
+
+  // If the request is for a whitelisted resource, show the prerendered page
+  if (whitelist && whitelist.length > 0) {
+    isRequestingPrerenderedPage = isResourceWhitelisted(request.url, whitelist);
+  }
+
+  // If the request is for a blacklisted resource, don't show the prerendered page
+  if (blacklist && blacklist.length > 0) {
+    isRequestingPrerenderedPage = !isResourceBlacklisted(request.url, blacklist, headers.referer);
+  }
+
+  return isRequestingPrerenderedPage;
+}

tsconfig.base.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "skipLibCheck": true,
+    "checkJs": true,
+    "allowJs": true,
+    "declaration": true,
+    "declarationMap": true,
+    "allowSyntheticDefaultImports": true
+  },
+  "files": ["./src/index.ts"]
+}

tsconfig.cjs.json

@@ -0,0 +1,11 @@
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "lib": ["ES6", "DOM"],
+    "target": "ES6",
+    "module": "CommonJS",
+    "moduleResolution": "Node",
+    "outDir": "./lib/cjs",
+    "declarationDir": "./lib/cjs/types"
+  }
+}

tsconfig.esm.json

@@ -0,0 +1,11 @@
+{
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {
+    "lib": ["ES2022", "DOM"],
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "NodeNext",
+    "outDir": "./lib/esm",
+    "declarationDir": "./lib/esm/types"
+  }
+}