Feat(extractor): Supporting Claude.ai (#210)

* chore: allow claude.ai domain

* feat: claude.ai raw export

* feat: better content extraction

* feat: better content targeting

* feat: adding Turndown formating rules

* feat: select chat title

* feat: improving code

* feat: extract ClaudeChat artifacts content

note: needs to post-process as Turndown does not support async click()

* fix: artifacts correctly formatted

* fix: update rules and timing

* fix: make sure artifact content is extracted

* fix: optimizing

* feat: artifact name and separation in export

* fix: preserveLineBreaksInPre

* refactor&fix: separating rules for PRE vs CODE without PRE

* fix: artifact title formatting

* fix: avoid selecting code copy button as it causes claude clipboard focus error breaking execution

* fix: preserveLineBreaksInCode target only CODE tags

* fix: keep newlines in artifacts code blocks

* fix: [wrong spacing3]

* fix: refactor + spacing attempt

* docs: cleaning

note: Turndown is automatically deleting spaces in spans, need to find a workaround

* chore: attempt, not working

* feat&fix: encapsulate div>code inside a PRE node to export artifact code correctly

* chore: cleaning code

* fix: adding artifact name as an id to put artifact content at the right place

In case of bug, artifacts are ot wrongly mixed into the export

* fix: only target artifacts preview buttons for replacement

* fix: preserveLineBreaksInCode for user Questions auto code blocks without PRE

Author: Hugo-COLLIN

src/manifest.json

@@ -21,7 +21,8 @@
     "*://*.phind.com/*",
     "*://*.perplexity.ai/*",
     "*://google.com/search*",
-    "*://chatgpt.com/*"
+    "*://chatgpt.com/*",
+    "*://*.claude.ai/*"
   ],
   "action": {
     "default_icon": "files/icons/icon_disabled-500.png"
@@ -35,7 +36,8 @@
       "*://*.phind.com/*",
       "*://*.perplexity.ai/*",
       "*://*.google.com/search*",
-      "*://chatgpt.com/*"
+      "*://chatgpt.com/*",
+      "*://*.claude.ai/*"
     ],
     "js": ["tab.js"]
   }],

src/services/checker/allowedDomains.json

@@ -3,7 +3,8 @@
     "Phind": "www.phind.com",
     "Perplexity": "www.perplexity.ai",
     "MaxAIGoogle": "www.google.com",
-    "ChatGPT": "chatgpt.com"
+    "ChatGPT": "chatgpt.com",
+    "Claude": "claude.ai"
   },
   "EXPORT_DOMAINS": {
     "PhindSearch": "www.phind.com/search",
@@ -13,6 +14,7 @@
     "MaxAIGoogle": "www.google.com/search",
     "ChatGPT": "chatgpt.com/c",
     "ChatGPTShare": "chatgpt.com/share",
-    "ChatGPTBots": "chatgpt.com/g"
+    "ChatGPTBots": "chatgpt.com/g",
+    "ClaudeChat": "claude.ai/chat"
   }
 }

src/services/checker/domainChecker.ts

@@ -4,7 +4,7 @@
  * @param hostAndPath url of the page
  * @returns {null|{name: string, url: *}}
  */
-export function domainChecker(allowedDomains: { [x: string]: any; Phind?: string; Perplexity?: string; MaxAIGoogle?: string; ChatGPT?: string; }, hostAndPath: string): null | { name: string; url: any; } {
+export function domainChecker(allowedDomains: { [x: string]: any; }, hostAndPath: string): null | { name: string; url: any; } {
   for (let domainName in allowedDomains) {
     const url = allowedDomains[domainName];
     if (hostAndPath?.startsWith(url)) {

src/services/export/extractor/defineAction.ts

@@ -1,9 +1,51 @@
 import {clickElements} from "../../interact/cs/interact";
+import {sleep} from "../../../utils/jsShorteners";
 
-export function defineAction(action: { type: any; selector: string | undefined; }) {
+
+//TODO: currently for Claude artifacts, needs to be genericized
+async function clickActClose(markdown: string | undefined, format: ((html: string) => string) | undefined) {
+  // selector duplication with processMessage in ClaudeChat.js and contentSelector in ClaudeChat.json
+  const pane = document.querySelector("div.fixed.flex");
+  const btns = document.querySelectorAll('.font-claude-message button[aria-label="Preview contents"]');
+
+  for (const artifactBtn of btns) {
+    const artifactName = artifactBtn.querySelector(".break-words")?.textContent
+    // @ts-ignore
+    artifactBtn.click();
+    await sleep(100);
+    // @ts-ignore
+    document.querySelector("[data-testid=\"undefined-code\"][data-state=\"off\"]")?.click() && await sleep(100); // Click on 'Code' button if it exists and is off
+    let artifactContent;
+    let i = 0;
+    while (!artifactContent && i < 10) {
+      artifactContent = pane?.querySelector(".code-block__code, .font-claude-message");
+      await sleep(100);
+    }
+
+    let codeWithPre;
+    if (pane?.querySelector(".code-block__code")) {
+      codeWithPre = document.createElement("pre");
+      codeWithPre.innerHTML = artifactContent?.outerHTML ?? "";
+    }
+
+    markdown = markdown?.replace(`{{@CAPTURE_ARTIFACT_CONTENT:${artifactName}}}`, `---\n**${artifactName ?? "Artifact"}:**\n` + (format?.(codeWithPre?.outerHTML ?? artifactContent?.outerHTML ?? "") ?? "") + "\n---");
+  }
+
+  // @ts-ignore
+  pane?.querySelector(".justify-end > button")?.click(); // close artifact
+
+  return markdown;
+}
+
+export async function defineAction(action: {
+  type: any;
+  selector?: string
+}, markdown?: string, format?: (html: string) => string): Promise<string | void | null> {
   switch (action.type) {
     case "click":
       return clickElements(action.selector);
+    case "click_act_close":
+      return await clickActClose(markdown, format);
     // case "scroll":
     //   return scrollElements;
     // case "type":

src/services/export/extractor/domains/ClaudeChat.js

@@ -0,0 +1,13 @@
+import {capitalizeFirst} from "../../../format/formatText";
+
+export async function processMessage(content, format, metadata) {
+  const messageSelector = content.querySelector("[data-testid=\"user-message\"], [data-is-streaming] > div");
+
+  const entityName = content?.querySelector("[data-is-streaming]")
+    ? "Claude"
+    : "User";
+
+  const msgTitle = "## " + capitalizeFirst(entityName ?? "") + "\n";
+
+  return msgTitle + format(messageSelector?.innerHTML ?? "") + "\n";
+}

src/services/export/extractor/domains/ClaudeChat.json

@@ -0,0 +1,36 @@
+{
+  "domainName": "Claude Chat",
+  "pageTitle": {
+    "selector": "[data-testid=\"chat-menu-trigger\"]"
+  },
+  "contentSelector": "[data-test-render-count]",
+  "turndown": {
+    "rules": {
+      "preserveLineBreaksInPre": {
+        "filter": "filter_PreserveLineBreaksInPre_Claude",
+        "replacement": "replacement_preserveLineBreaksInPre_Claude"
+      },
+      "preserveLineBreaksInCode_userQuestion": {
+        "filter": "filter_PreserveLineBreaksInCode_Claude",
+        "replacement": "replacement_preserveLineBreaksInCode_Claude"
+      },
+      "formatTables": {
+        "filter": "filter_formatTables",
+        "replacement": "replacement_formatTables"
+      },
+      "formatKatex": {
+        "filter": "filter_formatKatex",
+        "replacement": "replacement_formatKatex"
+      },
+      "captureArtifactContent": {
+        "filter": "filter_captureArtifactContent_Claude",
+        "replacement": "replacement_captureArtifactContent_Claude"
+      }
+    }
+  },
+  "actions": {
+    "afterExtraction": {
+      "type": "click_act_close"
+    }
+  }
+}

src/services/export/extractor/extractPage.ts

@@ -31,6 +31,10 @@ export async function extractPage(domain: { name: any; url?: any; }) {
       module = require("./domains/ChatGPT");
       json = require("./domains/ChatGPT.json");
       break;
+    case "ClaudeChat":
+      module = require("./domains/ClaudeChat");
+      json = require("./domains/ClaudeChat.json");
+      break;
     default:
       module = require("./domains/ArbitraryPage");
       json = require("./domains/ArbitraryPage.json");

src/services/export/extractor/extractPageContent.ts

@@ -26,7 +26,7 @@ export async function extractPageContent(format: (html: string) => string, metad
   markdown += await extractSections(messages, metadata, format, processMessage);
 
   if (metadata?.actions?.afterExtraction)
-    await safeExecute(defineAction(metadata.actions.afterExtraction));
+    await safeExecute(async () => markdown = await defineAction(metadata.actions.afterExtraction, markdown, format) ?? markdown);
 
   return markdown;
 }

src/services/export/extractor/extractPageMetadata.ts

@@ -19,12 +19,9 @@ export function extractPageMetadata(metadataBase: {
   extractor: any;
 } {
   return {
+    ...metadataBase,
     domainName: metadataBase.domainName ?? window.location.hostname,
     pageTitle: getPageTitle(metadataBase.pageTitle?.selector, metadataBase.pageTitle?.treatment),
-    contentSelector: metadataBase.contentSelector,
-    actions: metadataBase.actions,
-    sourcesExtraction: metadataBase.sourcesExtraction,
-    extractor: metadataBase.extractor,
   };
 }
 

src/services/export/extractor/rules/rules.ts

@@ -20,6 +20,7 @@ export function getBlankReplacement_PerplexityPages(content: any, node: { nodeNa
   }
 }
 
+
 /*
  --- Format tables ---
  */
@@ -82,6 +83,10 @@ export function filter_preserveLineBreaksInPre_Phind(node: { nodeName: string; q
   return node.nodeName === 'PRE' && node.querySelector('div');
 }
 
+export function filter_PreserveLineBreaksInPre_Claude(node: { nodeName: string; querySelector: (arg0: string) => any; }) {
+  return node.nodeName === 'PRE' && node.querySelector('div');
+}
+
 export function replacement_PreserveLineBreaksInPre_Perplexity(content: any, node: { querySelector: (arg0: string) => any; }) {
   const codeBlock = node.querySelector('code');
   const codeContent = codeBlock.textContent.trim();
@@ -103,6 +108,59 @@ export function replacement_preserveLineBreaksInPre_ChatGPT(content: any, node:
   return ('\n```' + codeLang + '\n' + codeContent + '\n```');
 }
 
+export function replacement_preserveLineBreaksInPre_Claude(content: any, node: any) {
+  const codeBlock = node.querySelector('code');
+  const codeContent = codeBlock?.textContent?.trim();
+  const codeLang = codeBlock.className.split("-")[1] ?? '';
+  return ('\n```' + codeLang + '\n' + codeContent + '\n```');
+}
+
+//---
+export function filter_PreserveLineBreaksInCode_Claude(node: {
+  parentNode: any;
+  nodeName: string; querySelector: (arg0: string) => any; }) {
+  return node.nodeName === 'CODE' && node.parentNode.classList.contains('code-block__code') && !node.parentNode.parentNode.parentNode.parentNode.parentNode.querySelector('pre');
+  // return node.nodeName === 'CODE' && node.parentNode.parentNode.parentNode.parentNode.nodeType !== "PRE";
+}
+
+export function replacement_preserveLineBreaksInCode_Claude(content: any, node: any) {
+  const clonedNode = node.cloneNode(true);
+  const topLevelSpans = Array.from(clonedNode.children);
+
+  // @ts-ignore
+  topLevelSpans.forEach((span: HTMLElement, index: number) => {
+    const nestedSpans = Array.from(span.children);
+
+    if (nestedSpans.length > 0) {
+      const firstSpan = nestedSpans[0] as HTMLElement;
+      const text = firstSpan.textContent || '';
+
+      // Count spaces at the beginning of line
+      const leadingSpaces = text.match(/^[\s\t]*/)?.[0].length || 0;
+
+      // Converts spaces to tabs (4 spaces = 1 tab)
+      if (leadingSpaces > 0) {
+        const tabCount = Math.floor(leadingSpaces / 4);
+        firstSpan.textContent = '\t'.repeat(tabCount) + text.trim();
+      }
+      // else if (firstSpan?.textContent === '') {
+      //   firstSpan.textContent = '\t';
+      // }
+    }
+
+    // Adds a line break after each span except the last
+    if (index < topLevelSpans.length - 1) {
+      const newLineSpan = document.createElement('span');
+      newLineSpan.textContent = '\n';
+      span.parentNode?.insertBefore(newLineSpan, span.nextSibling);
+    }
+  });
+
+  const codeContent = clonedNode.textContent?.trim() || '';
+  const codeLang = node?.className?.split("-")[1] || '';
+
+  return `\n\`\`\`${codeLang}\n${codeContent}\n\`\`\``;
+}
 
 
 /*
@@ -179,6 +237,20 @@ export function replacement_formatKatex(content: any, node: { querySelector: (ar
   return '$' + mathml + '$';
 }
 
+/*
+  --- Claude rules ---
+ */
+export function filter_captureArtifactContent_Claude(node: Element) {
+  //target button[aria-label="Preview contents"] :
+  return node.nodeName === 'BUTTON' && node.getAttribute('aria-label') === 'Preview contents';
+}
+
+export function replacement_captureArtifactContent_Claude(content: any, node: Element) {
+  return `{{@CAPTURE_ARTIFACT_CONTENT:${node.querySelector(".break-words")?.textContent}}}`;
+}
+
+
+
 /*
   --- Phind rules ---
  */

src/services/interact/cs/interact.ts

@@ -32,19 +32,25 @@ export async function clickElements(cssSelector = '.fe-chevron-down') {
  * @param content {HTMLElement} The content of the page
  * @returns {Promise<Element | null | undefined>} The last element clicked on
  */
-export async function selectAndClick(actionsList: Array<{ selector: string; scope: string; wait: number | undefined; }>, content: HTMLElement): Promise<Element | null> {
+export async function selectAndClick(actionsList: Array<{ selector?: string; scope?: string; wait?: number; }>, content: HTMLElement): Promise<Element | Document | null> {
   let element = null;
   for (const query of actionsList) {
     switch (query.scope) {
       case 'content':
-        element = content.querySelector(query.selector);
+        element = query.selector
+          ? content.querySelector(query.selector)
+          : content;
         break;
       case 'document':
-        element = document.querySelector(query.selector);
+        element = query.selector
+          ? document.querySelector(query.selector)
+          : document;
         break;
       default:
         console.warn("Unknown scope: " + query.scope + ". Defaulting to content for query: " + query.selector + ".");
-        return content.querySelector(query.selector);
+        return query.selector
+          ? content.querySelector(query.selector)
+          : content;
     }
 
     //TODO: define custom delay for each domain separately in JSON configuration