Merge branch 'main' into juan/sidepanel

Author: Shrinks99

.github/ISSUE_TEMPLATE/bug.yml

@@ -1,7 +1,7 @@
 {
   "name": "@webrecorder/archivewebpage",
   "productName": "ArchiveWeb.page",
-  "version": "0.14.2",
+  "version": "0.15.0",
   "main": "index.js",
   "description": "Create Web Archives directly in your browser",
   "repository": {
@@ -12,31 +12,34 @@
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.13.0",
-    "@ipld/car": "^5.3.1",
+    "@ipld/car": "^5.3.2",
     "@material/web": "^2.3.0",
-    "@webrecorder/awp-sw": "^0.5.3",
-    "@webrecorder/wabac": "^2.20.8",
+    "@ipld/unixfs": "^3.0.0",
+    "@webrecorder/wabac": "^2.22.16",
     "auto-js-ipfs": "^2.3.0",
-    "browsertrix-behaviors": "^0.6.4",
+    "browsertrix-behaviors": "^0.8.5",
     "btoa": "^1.2.1",
     "bulma": "^0.9.3",
-    "client-zip": "^2.2.2",
+    "client-zip": "^2.3.0",
+    "idb": "^7.1.1",
     "hash-wasm": "^4.9.0",
     "http-status-codes": "^2.1.4",
     "keyword-mark-element": "^0.1.2",
     "node-fetch": "2.6.7",
+    "p-queue": "^8.0.1",
     "pdfjs-dist": "2.2.228",
     "pretty-bytes": "^5.6.0",
-    "replaywebpage": "^2.2.5",
+    "replaywebpage": "^2.3.7",
     "stream-browserify": "^3.0.0",
     "tsconfig-paths-webpack-plugin": "^4.1.0",
     "unused-filename": "^4.0.1",
-    "uuid": "^8.3.2",
-    "warcio": "^2.4.2"
+    "uuid": "^9.0.0",
+    "warcio": "^2.4.4"
   },
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^6.15.0",
     "@typescript-eslint/parser": "^6.15.0",
+    "@types/uuid": "^10.0.0",
     "copy-webpack-plugin": "^9.0.1",
     "css-loader": "^6.2.0",
     "electron": "^32.2.0",
@@ -59,13 +62,13 @@
     "ts-loader": "^9.5.1",
     "ts-migrate": "^0.1.35",
     "typescript": "^5.3.3",
-    "webpack": "^5.91.0",
+    "webpack": "^5.99.7",
     "webpack-cli": "^5.1.4",
     "webpack-dev-server": "^5.0.4",
     "webpack-extension-reloader": "^1.1.4"
   },
   "resolutions": {
-    "@webrecorder/wabac": "^2.20.8"
+    "@webrecorder/wabac": "^2.22.16"
   },
   "files": [
     "src/",

.github/ISSUE_TEMPLATE/config.yml

@@ -254,8 +254,8 @@ class BrowserRecorder extends Recorder {
       // @ts-expect-error - TS2339 - Property 'behaviorState' does not exist on type 'BrowserRecorder'.
       if (this.behaviorState === BEHAVIOR_RUNNING) {
         title = "Archiving: Autopilot Running!";
-        color = "#0891b2";
-        text = "";
+        color = "#0096ff";
+        text = " ";
         // @ts-expect-error - TS2339 - Property 'numPending' does not exist on type 'BrowserRecorder'.
       } else if (this.numPending === 0) {
         title = "Archiving: No URLs pending, can continue";

.github/ISSUE_TEMPLATE/feature-change.yml

@@ -19,7 +19,7 @@ export function setLocalOption(name, value) {
 }
 
 // ===========================================================================
-export function getLocalOption(name: string) : Promise<string | null> {
+export function getLocalOption(name: string): Promise<string | null> {
   // @ts-expect-error - TS2339 - Property 'chrome' does not exist on type 'Window & typeof globalThis'. | TS2339 - Property 'chrome' does not exist on type 'Window & typeof globalThis'.
   if (self.chrome?.storage) {
     return new Promise<string>((resolve) => {

package.json

@@ -22,8 +22,7 @@ import {
 } from "./consts";
 import { getLocalOption } from "./localstorage";
 
-// @ts-expect-error - TS2554 - Expected 0 arguments, but got 1.
-const encoder = new TextEncoder("utf-8");
+const encoder = new TextEncoder();
 
 const MAX_CONCURRENT_FETCH = 6;
 
@@ -58,6 +57,9 @@ type FetchEntry = {
 class Recorder {
   archiveStorage = false;
   archiveCookies = false;
+  archiveFlash = false;
+  archiveScreenshots = false;
+  archivePDF = false;
 
   _fetchQueue: FetchEntry[] = [];
 
@@ -155,8 +157,11 @@ class Recorder {
   }
 
   async initOpts() {
-    this.archiveCookies = (await getLocalOption("archiveCookies") === "1");
-    this.archiveStorage = (await getLocalOption("archiveStorage") === "1");
+    this.archiveCookies = (await getLocalOption("archiveCookies")) === "1";
+    this.archiveStorage = (await getLocalOption("archiveStorage")) === "1";
+    this.archiveFlash = (await getLocalOption("archiveFlash")) === "1";
+    this.archiveScreenshots = (await getLocalOption("archiveScreenshots")) === "1";
+    this.archivePDF = (await getLocalOption("archivePDF")) === "1";
   }
 
   // @ts-expect-error - TS7006 - Parameter 'autorun' implicitly has an 'any' type.
@@ -190,8 +195,8 @@ class Recorder {
       this.behaviorInitStr
     });
 
-    window.addEventListener("beforeunload", () => {});` +
-      this.getFlashInjectScript()
+    window.addEventListener("beforeunload", () => {});\n` +
+      (this.archiveFlash ? this.getFlashInjectScript() : "")
     );
   }
 
@@ -550,8 +555,6 @@ class Recorder {
 
     await this.send("Page.enable");
 
-    await this.send("Runtime.enable");
-
     await this.send("DOMSnapshot.enable");
 
     await this.initPixRatio();
@@ -632,7 +635,6 @@ class Recorder {
 
   async sessionClose(sessions = []) {
     await this.send("Page.disable");
-    await this.send("Runtime.disable");
     await this.send("DOMSnapshot.disable");
 
     await this.send("Debugger.disable");
@@ -932,6 +934,98 @@ class Recorder {
     return success;
   }
 
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async savePDF(pageInfo: any) {
+    // @ts-expect-error: ignore param
+    await this.send("Emulation.setEmulatedMedia", {type: "screen"});
+
+    // @ts-expect-error: ignore param
+    const resp = await this.send("Page.printToPDF", {printBackground: true});
+
+    // @ts-expect-error: ignore param
+    await this.send("Emulation.setEmulatedMedia", {type: ""});
+
+    const payload = Buffer.from(resp.data, "base64");
+    const mime = "application/pdf";
+
+    const fullData = {
+      url: "urn:pdf:" + pageInfo.url,
+      ts: new Date().getTime(),
+      status: 200,
+      statusText: "OK",
+      pageId: pageInfo.id,
+      mime,
+      respHeaders: {"Content-Type": mime, "Content-Length": payload.length + ""},
+      reqHeaders: {},
+      payload,
+      extraOpts: {resource: true},
+    };
+
+    console.log("pdf", payload.length);
+
+    // @ts-expect-error - TS2339 - Property '_doAddResource' does not exist on type 'Recorder'.
+    await this._doAddResource(fullData);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async saveScreenshot(pageInfo: any) {
+
+    // View Screenshot
+    const width = 1920;
+    const height = 1080;
+
+    // @ts-expect-error: ignore param
+    await this.send("Emulation.setDeviceMetricsOverride", {width, height, deviceScaleFactor: 0, mobile: false});
+    // @ts-expect-error: ignore param
+    const resp = await this.send("Page.captureScreenshot", {format: "png"});
+
+    const payload = Buffer.from(resp.data, "base64");
+    const blob = new Blob([payload], {type: "image/png"});
+
+    await this.send("Emulation.clearDeviceMetricsOverride");
+    
+    const mime = "image/png";
+
+    const fullData = {
+      url: "urn:view:" + pageInfo.url,
+      ts: new Date().getTime(),
+      status: 200,
+      statusText: "OK",
+      pageId: pageInfo.id,
+      mime,
+      respHeaders: {"Content-Type": mime, "Content-Length": payload.length + ""},
+      reqHeaders: {},
+      payload,
+      extraOpts: {resource: true},
+    };
+
+    const thumbWidth = 640;
+    const thumbHeight = 360;
+
+    const bitmap = await self.createImageBitmap(blob, {resizeWidth: thumbWidth, resizeHeight: thumbHeight});
+    
+    const canvas = new OffscreenCanvas(thumbWidth, thumbWidth);
+    const context = canvas.getContext("bitmaprenderer")!;
+    context.transferFromImageBitmap(bitmap);
+
+    const resizedBlob = await canvas.convertToBlob({type: "image/png"});
+
+    const thumbPayload = new Uint8Array(await resizedBlob.arrayBuffer());
+
+    const thumbData = {...fullData,
+      url: "urn:thumbnail:" + pageInfo.url,
+      respHeaders: {"Content-Type": mime, "Content-Length": thumbPayload.length + ""},
+      payload: thumbPayload
+    };
+    
+    // @ts-expect-error - TS2339 - Property '_doAddResource' does not exist on type 'Recorder'.
+    await this._doAddResource(fullData);
+
+
+    // @ts-expect-error - TS2339 - Property '_doAddResource' does not exist on type 'Recorder'.
+    await this._doAddResource(thumbData);
+  }
+
   async getFullText(finishing = false) {
     // @ts-expect-error - TS2339 - Property 'pageInfo' does not exist on type 'Recorder'. | TS2339 - Property 'pageInfo' does not exist on type 'Recorder'.
     if (!this.pageInfo?.url) {
@@ -1191,6 +1285,14 @@ class Recorder {
     // @ts-expect-error - TS2339 - Property 'pageInfo' does not exist on type 'Recorder'.
     const pageInfo = this.pageInfo;
 
+    if (this.archiveScreenshots) {
+      await this.saveScreenshot(pageInfo);
+    }
+
+    if (this.archivePDF) {
+      await this.savePDF(pageInfo);
+    }
+
     const [domSnapshot, favIcon] = await Promise.all([
       this.getFullText(),
       // @ts-expect-error - TS2339 - Property 'getFavIcon' does not exist on type 'Recorder'.