feat: Support old and new puppeteer types (#555)

* docs: Add package testing example

* feat: Support old and new puppeteer types

* ci: Update puppeteer version matrix

* lint: Refactor if statement

* chore: TS fix

* feat: Disable hoisting for types, upgrade adblocker plugin

* lint: Make TS happy

* chore: Update deps

* fix: Add mising dev dep due to hoisting rule

* chore: Add @types/puppeteer as dev dependency

* ci: Add smaller fixes

* ci: Testing ci

* ci: Testing ci

* chore: Disable adblocker plugin build in old pptr versions

* ci: Testing ci

.github/workflows/test.yml

@@ -30,9 +30,11 @@ jobs:
           # - 14
           - 12
         puppeteer_version:
-          - 7.0.0 # Chromium 90.0.4403.0, Feb 3, 2021
+          - 10.2.0 # Chromium 93.0.4577.0
+          # - 7.0.0 # Chromium 90.0.4403.0, Feb 3, 2021
+          - 5.5.0 # Chromium 88.0.4298.0
           # - 5.0.0 # Chromium 83.0.4103.0, Jul 2, 2020
-          - 2.1.1 # Chromium 79.0.3942.0, Oct 24 2019
+          # - 2.1.1 # Chromium 79.0.3942.0, Oct 24 2019
           # - 2.0.0 # Chromium 79.0.3942.0, Oct 24 2019
           # - 1.20.0 # Chromium 78.0.3882.0, Sep 13 2019
           # - 1.15.0 # Chromium 75.0.3765.0, Apr 26 2019

README.md

@@ -72,6 +72,11 @@ yarn lernaupdate
 # If in doubt :-(
 yarn lerna exec "rm -f yarn.lock; rm -rf node_modules; echo 0"
 rm -f yarn.lock &&  rm -rf node_modules && yarn cache clean
+
+
+# Test a local monorepo package in an outside folder as it would've been installed from the registry
+# Change PACKAGE_DIR to the path of this monorepo and PACKAGE to the package you wish to install
+PACKAGE=puppeteer-extra PACKAGE_DIR=/Users/foo/puppeteer-extra/packages && yarn remove $(echo $PACKAGE); true && rm -f $(pwd)/$(echo $PACKAGE)-latest.tgz && yarn --cwd $(echo $PACKAGE_DIR)/$(echo $PACKAGE) pack --filename $(pwd)/$(echo $PACKAGE)-latest.tgz && YARN_CACHE_FOLDER=/tmp/yarn yarn add file:$(pwd)/$(echo $PACKAGE)-latest.tgz && rm -rf /tmp/yarn
 ```
 
 #### Publishing

package.json

@@ -17,9 +17,17 @@
     "prepare": "lerna run prepare",
     "release": "lerna publish --npm-client npm"
   },
-  "workspaces": [
-    "packages/*"
-  ],
+  "workspaces": {
+    "packages": [
+      "packages/*"
+    ],
+    "nohoist": [
+      "**/@types",
+      "**/@types/**",
+      "**/typescript",
+      "**/typescript/**"
+    ]
+  },
   "devDependencies": {
     "eslint": "^6.7.1",
     "eslint-config-prettier": "^6.7.0",

packages/puppeteer-extra-plugin-adblocker/build_version_check.js

@@ -0,0 +1,19 @@
+const pkg = require('./package.json')
+
+const isIncompatiblePuppeteerVersion = () => {
+  const version = pkg.devDependencies.puppeteer
+  const majorVersion = parseInt(version.split('.')[0])
+  if (majorVersion >= 6) {
+    return true
+  } else {
+    return false
+  }
+}
+
+const incompatible = isIncompatiblePuppeteerVersion()
+if (incompatible) {
+  console.warn(
+    'ERR: The adblocker plugin requires pptr >= 6',
+    process.env.PUPPETEER_VERSION
+  )
+}

packages/puppeteer-extra-plugin-adblocker/package.json

@@ -16,7 +16,7 @@
     "clean": "rimraf dist/*",
     "tscheck": "tsc --pretty --noEmit",
     "prebuild": "run-s clean",
-    "build": "run-s build:tsc build:rollup",
+    "build": "run-s build:tsc build:rollup; node build_version_check.js",
     "build:tsc": "tsc --module commonjs",
     "build:rollup": "rollup -c rollup.config.ts",
     "docs": "node -e 0",
@@ -44,20 +44,24 @@
   "devDependencies": {
     "@types/debug": "^4.1.5",
     "@types/node-fetch": "^2.5.4",
-    "@types/puppeteer": "5.4.3",
+    "@types/puppeteer": "*",
     "ava": "^2.4.0",
     "npm-run-all": "^4.1.5",
-    "puppeteer": "2.0.0",
+    "puppeteer": "^8.0.0",
     "rimraf": "^3.0.0",
     "ts-node": "^8.5.4",
     "tslint": "^5.20.1",
     "tslint-config-prettier": "^1.18.0",
     "tslint-config-standard": "^9.0.0",
-    "typescript": "^3.7.4"
+    "typescript": "4.1.2",
+    "rollup": "^1.27.5",
+    "rollup-plugin-commonjs": "^10.1.0",
+    "rollup-plugin-node-resolve": "^5.2.0",
+    "rollup-plugin-sourcemaps": "^0.4.2",
+    "rollup-plugin-typescript2": "^0.25.2"
   },
   "dependencies": {
-    "@cliqz/adblocker-puppeteer": "1.18.7",
-    "@types/chrome": "0.0.91",
+    "@cliqz/adblocker-puppeteer": "1.22.6",
     "debug": "^4.1.1",
     "node-fetch": "^2.6.0",
     "puppeteer-extra-plugin": "^3.1.9"

packages/puppeteer-extra-plugin-adblocker/tsconfig.json

@@ -17,7 +17,8 @@
     "noUnusedLocals": true,
     "noUnusedParameters": false,
     "pretty": true,
-    "stripInternal": true
+    "stripInternal": true,
+    "types": ["node"]
   },
   "include": [
     "./src/**/*.tsx",

packages/puppeteer-extra-plugin-recaptcha/package.json

@@ -48,20 +48,24 @@
   ],
   "devDependencies": {
     "@types/debug": "^4.1.5",
-    "@types/node": "^13.1.4",
-    "@types/puppeteer": "5.4.3",
+    "@types/node": "14.17.6",
+    "@types/puppeteer": "*",
     "ava": "2.4.0",
     "copyfiles": "^2.1.1",
     "npm-run-all": "^4.1.5",
-    "puppeteer": "^2.0.0",
+    "puppeteer": "9",
     "puppeteer-extra": "^3.1.18",
     "replace-in-files-cli": "^0.3.1",
     "rimraf": "^3.0.0",
     "ts-node": "^8.5.4",
     "tslint": "^5.20.1",
     "tslint-config-prettier": "^1.18.0",
     "tslint-config-standard": "^9.0.0",
-    "typescript": "^3.7.4"
+    "typescript": "4.4.3",
+    "rollup-plugin-commonjs": "^10.1.0",
+    "rollup-plugin-node-resolve": "^5.2.0",
+    "rollup-plugin-sourcemaps": "^0.4.2",
+    "rollup-plugin-typescript2": "^0.25.2"
   },
   "dependencies": {
     "debug": "^4.1.1",

packages/puppeteer-extra-plugin-recaptcha/src/index.ts

@@ -20,7 +20,6 @@ export const BuiltinSolutionProviders: types.SolutionProvider[] = [
  * @noInheritDoc
  */
 export class PuppeteerExtraPluginRecaptcha extends PuppeteerExtraPlugin {
-  public opts: types.PluginOptions
   private contentScriptDebug: debug.Debugger
 
   constructor(opts: Partial<types.PluginOptions>) {
@@ -44,6 +43,10 @@ export class PuppeteerExtraPluginRecaptcha extends PuppeteerExtraPlugin {
     }
   }
 
+  get opts(): types.PluginOptions {
+    return super.opts as any
+  }
+
   get contentScriptOpts(): types.ContentScriptOpts {
     const { visualFeedback } = this.opts
     return {

packages/puppeteer-extra-plugin-recaptcha/tsconfig.json

@@ -18,7 +18,8 @@
     "noUnusedLocals": true,
     "noUnusedParameters": false,
     "pretty": true,
-    "stripInternal": true
+    "stripInternal": true,
+    "types": ["node"]
   },
   "include": [
     "./src/**/*.tsx",

packages/puppeteer-extra-plugin/package.json

@@ -43,12 +43,12 @@
     "pupeteer"
   ],
   "devDependencies": {
-    "@types/node": "^13.1.4",
-    "@types/puppeteer": "5.4.3",
+    "@types/node": "14.14.34",
+    "@types/puppeteer": "*",
     "ava": "2.4.0",
     "documentation-markdown-themes": "^12.1.5",
     "npm-run-all": "^4.1.5",
-    "puppeteer": "^2.0.0",
+    "puppeteer": "9",
     "rimraf": "^3.0.0",
     "rollup": "^1.27.5",
     "rollup-plugin-commonjs": "^10.1.0",
@@ -59,7 +59,7 @@
     "tslint": "^5.12.1",
     "tslint-config-prettier": "^1.18.0",
     "tslint-config-standard": "^9.0.0",
-    "typescript": "^3.7.4"
+    "typescript": "4.4.3"
   },
   "dependencies": {
     "@types/debug": "^4.1.0",

packages/puppeteer-extra-plugin/src/index.ts

@@ -1,5 +1,5 @@
 import debug, { Debugger } from 'debug'
-import * as Puppeteer from 'puppeteer'
+import * as Puppeteer from './puppeteer'
 
 /** @private */
 const merge = require('merge-deep')
@@ -461,7 +461,7 @@ export abstract class PuppeteerExtraPlugin {
   _getMissingDependencies(plugins: any) {
     const pluginNames = new Set(plugins.map((p: any) => p.name))
     const missing = new Set(
-      Array.from(this.dependencies.values()).filter((x) => !pluginNames.has(x))
+      Array.from(this.dependencies.values()).filter(x => !pluginNames.has(x))
     )
     return missing
   }
@@ -534,9 +534,17 @@ export abstract class PuppeteerExtraPlugin {
     if (this.onTargetCreated) await this.onTargetCreated(target)
     // Pre filter pages for plugin developers convenience
     if (target.type() === 'page') {
-      const page = await target.page()
-      if (this.onPageCreated) {
-        await this.onPageCreated(page)
+      try {
+        const page = await target.page()
+        if (!page) {
+          return
+        }
+        const validPage = 'isClosed' in page && !page.isClosed()
+        if (this.onPageCreated && validPage) {
+          await this.onPageCreated(page)
+        }
+      } catch (err) {
+        console.error(err)
       }
     }
   }

packages/puppeteer-extra/src/puppeteer.ts → packages/puppeteer-extra-plugin/src/puppeteer.ts

@@ -1,10 +1,9 @@
 // A wildcard import would result in a `require("puppeteer")` statement
 // at the top of the transpiled js file, not what we want. :-/
+// "import type" is a solution here but requires TS >= v3.8 which we don't want to require yet as a minimum.
 
 export { Browser } from 'puppeteer'
 export { Page } from 'puppeteer'
+export { Target } from 'puppeteer'
 export { ConnectOptions } from 'puppeteer'
-export { ChromeArgOptions } from 'puppeteer'
 export { LaunchOptions } from 'puppeteer'
-export { FetcherOptions } from 'puppeteer'
-export { BrowserFetcher } from 'puppeteer'

packages/puppeteer-extra-plugin/tsconfig.json

@@ -17,7 +17,8 @@
     "noUnusedLocals": true,
     "noUnusedParameters": false,
     "pretty": true,
-    "stripInternal": true
+    "stripInternal": true,
+    "types": ["node"]
   },
   "include": [
     "./src/**/*.tsx",

packages/puppeteer-extra/package.json

@@ -14,15 +14,18 @@
   "scripts": {
     "clean": "rimraf dist/*",
     "prebuild": "run-s clean",
-    "build": "run-s build:tsc build:rollup",
+    "build": "run-s build:tsc build:rollup ambient-dts",
     "build:tsc": "tsc --module commonjs",
     "build:rollup": "rollup -c rollup.config.ts",
     "docs": "documentation readme --quiet --shallow --github --markdown-theme transitivebs --readme-file readme.md --section API ./src/index.ts",
     "postdocs": "npx prettier --write readme.md",
     "test:ts": "ava -v --config ava.config-ts.js",
     "test:js": "ava -v --serial --concurrency 1 --fail-fast",
     "test": "run-p test:js test:ts",
-    "test-ci": "run-s test"
+    "test-ci": "run-s test",
+    "ambient-dts": "run-s ambient-dts-copy ambient-dts-fix-path",
+    "ambient-dts-copy": "copyfiles -u 1 \"src/**/*.d.ts\" dist",
+    "ambient-dts-fix-path": "replace-in-files --string='/// <reference path=\"../src/' --replacement='/// <reference path=\"../dist/' 'dist/**/*.d.ts'"
   },
   "keywords": [
     "puppeteer",
@@ -39,11 +42,10 @@
     "node": ">=8"
   },
   "devDependencies": {
-    "@types/puppeteer": "5.4.3",
     "ava": "^2.4.0",
     "documentation-markdown-themes": "^12.1.5",
     "npm-run-all": "^4.1.5",
-    "puppeteer": "^2.0.0",
+    "puppeteer": "9",
     "puppeteer-extra-plugin": "^3.1.9",
     "puppeteer-extra-plugin-anonymize-ua": "^2.2.17",
     "puppeteer-firefox": "^0.5.0",
@@ -57,14 +59,14 @@
     "tslint": "^5.20.1",
     "tslint-config-prettier": "^1.18.0",
     "tslint-config-standard": "^9.0.0",
-    "typescript": "^3.7.4"
+    "typescript": "4.4.3"
   },
   "peerDependencies": {
     "puppeteer": "*"
   },
   "dependencies": {
     "@types/debug": "^4.1.0",
-    "@types/puppeteer": "5.4.3",
+    "@types/puppeteer": "*",
     "debug": "^4.1.1",
     "deepmerge": "^4.2.2"
   },