feat(sds): persistent storage for history

.gitignore

@@ -21,3 +21,5 @@ CLAUDE.md
 .env
 postgres-data/
 packages/rln/waku-rlnv2-contract/
+/packages/**/allure-results
+/packages/**/allure-results

packages/sds/karma.conf.cjs

@@ -1,3 +1,16 @@
-const config = require("../../karma.conf.cjs");
+import path from "path";
 
-module.exports = config;
+import baseConfig from "../../karma.conf.cjs";
+
+export default function (config) {
+  baseConfig(config);
+
+  const storageDir = path.resolve(__dirname, "src/message_channel/storage");
+
+  // Swap node storage for browser storage in webpack builds
+  config.webpack.resolve.alias = {
+    ...config.webpack.resolve.alias,
+    [path.join(storageDir, "node.ts")]: path.join(storageDir, "browser.ts"),
+    [path.join(storageDir, "node.js")]: path.join(storageDir, "browser.ts")
+  };
+}

packages/sds/package.json

@@ -4,6 +4,9 @@
     "description": "Scalable Data Sync implementation for the browser. Based on https://github.com/vacp2p/rfc-index/blob/main/vac/raw/sds.md",
     "types": "./dist/index.d.ts",
     "module": "./dist/index.js",
+    "browser": {
+        "./dist/message_channel/storage/index.js": "./dist/message_channel/storage/browser.js"
+    },
     "exports": {
         ".": {
             "types": "./dist/index.d.ts",

packages/sds/src/message_channel/mem_local_history.spec.ts → packages/sds/src/message_channel/local_history.spec.ts

@@ -1,13 +1,13 @@
 import { expect } from "chai";
 
-import { MemLocalHistory } from "./mem_local_history.js";
+import { LocalHistory } from "./local_history.js";
 import { ContentMessage } from "./message.js";
 
-describe("MemLocalHistory", () => {
+describe("LocalHistory", () => {
   it("Cap max size when messages are pushed one at a time", () => {
     const maxSize = 2;
 
-    const hist = new MemLocalHistory(maxSize);
+    const hist = new LocalHistory({ maxSize });
 
     hist.push(
       new ContentMessage("1", "c", "a", [], 1n, undefined, new Uint8Array([1]))
@@ -31,7 +31,7 @@ describe("MemLocalHistory", () => {
   it("Cap max size when a pushed array is exceeding the cap", () => {
     const maxSize = 2;
 
-    const hist = new MemLocalHistory(maxSize);
+    const hist = new LocalHistory({ maxSize });
 
     hist.push(
       new ContentMessage("1", "c", "a", [], 1n, undefined, new Uint8Array([1]))

packages/sds/src/message_channel/mem_local_history.ts → packages/sds/src/message_channel/local_history.ts

@@ -1,9 +1,28 @@
+import { Logger } from "@waku/utils";
 import _ from "lodash";
 
 import { ContentMessage, isContentMessage } from "./message.js";
+import { Storage } from "./storage/index.js";
 
 export const DEFAULT_MAX_LENGTH = 10_000;
 
+/**
+ * Options for the LocalHistory constructor.
+ * @param storage - The storage to use for the local history.
+ *   - prefix - The prefix for the storage.
+ *   - customInstance - The custom storage instance to use.
+ * @param maxSize - The maximum number of messages to store.
+ */
+export type LocalHistoryOptions = {
+  storage?: {
+    prefix?: string;
+    customInstance?: Storage;
+  };
+  maxSize?: number;
+};
+
+const log = new Logger("sds:local-history");
+
 /**
  * In-Memory implementation of a local history of messages.
  *
@@ -17,15 +36,28 @@ export const DEFAULT_MAX_LENGTH = 10_000;
  * If an array of items longer than `maxLength` is pushed, dropping will happen
  * at next push.
  */
-export class MemLocalHistory {
+export class LocalHistory {
   private items: ContentMessage[] = [];
+  private readonly storage?: Storage;
+  private readonly maxSize: number;
+
+  public constructor(opts: LocalHistoryOptions = {}) {
+    const { storage, maxSize } = opts;
+    const { prefix, customInstance } = storage ?? {};
+    this.maxSize = maxSize ?? DEFAULT_MAX_LENGTH;
+    if (customInstance) {
+      this.storage = customInstance;
+      log.info("Using custom storage instance", { customInstance });
+    } else if (prefix) {
+      this.storage = new Storage(prefix);
+      log.info("Creating storage with prefix", { prefix });
+    } else {
+      this.storage = undefined;
+      log.info("Using in-memory storage");
+    }
 
-  /**
-   * Construct a new in-memory local history
-   *
-   * @param maxLength The maximum number of message to store.
-   */
-  public constructor(private maxLength: number = DEFAULT_MAX_LENGTH) {}
+    this.load();
+  }
 
   public get length(): number {
     return this.items.length;
@@ -47,11 +79,13 @@ export class MemLocalHistory {
     this.items = _.uniqBy(combinedItems, "messageId");
 
     // Let's drop older messages if max length is reached
-    if (this.length > this.maxLength) {
-      const numItemsToRemove = this.length - this.maxLength;
+    if (this.length > this.maxSize) {
+      const numItemsToRemove = this.length - this.maxSize;
       this.items.splice(0, numItemsToRemove);
     }
 
+    this.save();
+
     return this.items.length;
   }
 
@@ -99,4 +133,15 @@ export class MemLocalHistory {
       );
     }
   }
+
+  private save(): void {
+    this.storage?.save(this.items);
+  }
+
+  private load(): void {
+    const messages = this.storage?.load() ?? [];
+    if (messages.length > 0) {
+      this.items = messages;
+    }
+  }
 }