Gemini scraping algorithm

.github/workflows/deploy.yml

@@ -25,7 +25,7 @@ jobs:
           echo "$SSH_KEY" > deploy_key
           chmod 600 deploy_key
           rsync -avz --delete \
-            --exclude 'node_modules' --exclude '.git' --exclude '.env' \
+            --exclude 'node_modules' --exclude '.git' --exclude '.env' --exclude 'deploy_key' \
             -e "ssh -i deploy_key -o StrictHostKeyChecking=no" \
             . $USER@$HOST:~/app
 

app/api/conversation/route.ts

@@ -72,9 +72,22 @@ export async function POST(req: NextRequest) {
       return NextResponse.json({ error: '`htmlDoc` must be a file field' }, { status: 400 });
     }
 
-    // Parse the conversation from HTML
+    // Parse the conversation from HTML if skipScraping is false, otherwise skip parsing
+    const skipScraping = formData.has('skipScraping');
+    let conversation;
     const html = await file.text();
-    const conversation = await parseHtmlToConversation(html, model);
+    if (!skipScraping) {
+      conversation = await parseHtmlToConversation(html, model);
+    } else {
+      // Remove CSS rule from the HTML string
+      const cleanedHtml = html.replace(/body\s*\{[^}]*\}/gm, '');
+      conversation = {
+        model: model,
+        content: cleanedHtml,
+        scrapedAt: new Date().toISOString(),
+        sourceHtmlBytes: cleanedHtml.length,
+      };
+    }
 
     // Generate a unique ID for the conversation
     const conversationId = randomUUID();

extension/manifest.json

@@ -13,7 +13,9 @@
     "https://claude.ai/*",
     "https://gemini.google.com/*",
     "https://chatgpt.com/*",
-    "https://chat.openai.com/*"
+    "https://chat.openai.com/*",
+    "https://chat.deepseek.com/*",
+    "https://copilot.microsoft.com/*"
   ],
   "content_security_policy": {
     "extension_pages": "script-src 'self'; object-src 'self'"
@@ -26,7 +28,9 @@
         "https://bard.google.com/*",
         "https://gemini.google.com/*",
         "https://claude.ai/*",
-        "https://www.meta.ai/*"
+        "https://www.meta.ai/*",
+        "https://chat.deepseek.com/*",
+        "https://copilot.microsoft.com/*"
       ],
       "js": ["config.js", "index.js"],
       "run_at": "document_end",

extension/popup.js

@@ -34,24 +34,45 @@ const scrape = () => {
 
 chrome.tabs.query({ active: true, currentWindow: true, url: 'https://bard.google.com/*' }, (tabs) => {
   if (tabs?.length) {
-    chrome.tabs.sendMessage(tabs[0].id, { action: 'model', model: 'Bard' }, function (_) {
+    chrome.tabs.sendMessage(tabs[0].id, { action: 'model', model: 'bard' }, function (_) {
       console.log('is Bard');
     });
   }
 });
+chrome.tabs.query({ active: true, currentWindow: true, url: 'https://gemini.google.com/*' }, (tabs) => {
+  if (tabs?.length) {
+    chrome.tabs.sendMessage(tabs[0].id, { action: 'model', model: 'gemini' }, function (_) {
+      console.log('is Gemini');
+    });
+  }
+});
 chrome.tabs.query({ active: true, currentWindow: true, url: 'https://www.meta.ai/*' }, (tabs) => {
   if (tabs?.length) {
-    chrome.tabs.sendMessage(tabs[0].id, { action: 'model', model: 'Meta' }, function (_) {
+    chrome.tabs.sendMessage(tabs[0].id, { action: 'model', model: 'meta' }, function (_) {
       console.log('is Meta');
     });
   }
 });
 chrome.tabs.query({ active: true, currentWindow: true, url: 'https://claude.ai/*' }, (tabs) => {
   if (tabs?.length) {
-    chrome.tabs.sendMessage(tabs[0].id, { action: 'model', model: 'Claude' }, function (_) {
+    chrome.tabs.sendMessage(tabs[0].id, { action: 'model', model: 'claude' }, function (_) {
       console.log('is Claude');
     });
   }
 });
+chrome.tabs.query({ active: true, currentWindow: true, url: 'https://chat.deepseek.com/*' }, (tabs) => {
+  if (tabs?.length) {
+    chrome.tabs.sendMessage(tabs[0].id, { action: 'model', model: 'deepseek' }, function (_) {
+      console.log('is DeepSeek');
+    });
+  }
+});
+chrome.tabs.query({ active: true, currentWindow: true, url: 'https://copilot.microsoft.com/*' }, (tabs) => {
+  if (tabs?.length) {
+    chrome.tabs.sendMessage(tabs[0].id, { action: 'model', model: 'copilot' }, function (_) {
+      console.log('is Copilot');
+    });
+  }
+});
 
 window.onload = initApp;

lib/parsers/chatgpt.ts

@@ -1,13 +1,56 @@
 import type { Conversation } from '@/types/conversation';
+import { JSDOM } from 'jsdom';
+import fs from 'fs';
+import path from 'path';
 
 /**
  * Extracts a ChatGPT share page into a structured Conversation.
  */
 export async function parseChatGPT(html: string): Promise<Conversation> {
+  const dom = new JSDOM(html);
+  const document = dom.window.document;
+
+  // Clean up elements first
+  const miscElems = document.querySelectorAll('button');
+  miscElems.forEach(button => button.remove());
+
+  // Find container
+  const chatContainer = document.querySelector('div[class*="pb-25"]');
+  if (!chatContainer) {
+    throw new Error('Conversation container not found');
+  }
+
+  // Ensure all divs are using light mode
+  const darkElements = chatContainer.querySelectorAll('.dark');
+  darkElements.forEach(element => {
+    element.classList.remove('dark');
+    element.classList.add('light');
+  });
+
+  // Read CSS separately and inject it directly into final HTML
+  const cssPath = path.resolve(process.cwd(), 'lib/parsers/assets/chatgpt.css');
+  const css = fs.readFileSync(cssPath, 'utf-8');
+
+  // Build final HTML
+  const htmlContent = `
+  <!DOCTYPE html>
+  <html>
+    <head>
+      <meta charset="utf-8">
+      <style>
+        ${css}
+      </style>
+    </head>
+    <body>
+      ${chatContainer.outerHTML}
+    </body>
+  </html>
+  `;
+
   return {
     model: 'ChatGPT',
-    content: html,
+    content: htmlContent,
     scrapedAt: new Date().toISOString(),
-    sourceHtmlBytes: html.length,
+    sourceHtmlBytes: htmlContent.length,
   };
 }

lib/parsers/claude.ts

@@ -1,13 +1,53 @@
 import type { Conversation } from '@/types/conversation';
+import { JSDOM } from 'jsdom';
+import fs from 'fs';
+import path from 'path';
 
 /**
  * Extracts a Claude share page into a structured Conversation.
  */
 export async function parseClaude(html: string): Promise<Conversation> {
+  const dom = new JSDOM(html);
+  const document = dom.window.document;
+
+  // Clean up elements first
+  const miscElems = document.querySelectorAll('button, div[data-testid="message-warning"]');
+  miscElems.forEach(button => button.remove());
+
+  // Find container
+  const chatContainer = document.querySelector('.flex-1.flex.flex-col.gap-3.px-4.max-w-3xl.mx-auto.w-full.pt-1');
+  if (!chatContainer) {
+    throw new Error('Conversation container not found');
+  }
+
+  chatContainer.querySelectorAll('pre, code').forEach(element => {
+    (element as HTMLElement).style.textShadow = '';
+  });
+
+  // Read CSS separately and inject it directly into final HTML
+  const cssPath = path.resolve(process.cwd(), 'lib/parsers/assets/claude.css');
+  const css = fs.readFileSync(cssPath, 'utf-8');
+
+  // Build final HTML
+  const htmlContent = `
+  <!DOCTYPE html>
+  <html class="h-screen antialiased [font-feature-settings:'ss01'] scroll-smooth __variable_dcab32 __variable_820c23 __variable_b4db0f __variable_669e4a __variable_e4ce97 __variable_e4195f" lang="en-US" data-theme="claude" data-mode="light" data-build-id="41c59ae0d3" data-env="" style="--font-user-message: var(--font-sans-serif); --font-claude-message: var(--font-serif);">
+    <head>
+      <meta charset="utf-8">
+      <style>
+        ${css}
+      </style>
+    </head>
+    <body class="bg-bg-100 text-text-100 font-ui min-h-screen" style="pointer-events: auto;" data-new-gr-c-s-check-loaded="14.1246.0" data-gr-ext-installed="" data-new-gr-c-s-loaded="14.1246.0">
+      ${chatContainer.outerHTML}
+    </body>
+  </html>
+  `;
+
   return {
     model: 'Claude',
-    content: html,
+    content: htmlContent,
     scrapedAt: new Date().toISOString(),
-    sourceHtmlBytes: html.length,
+    sourceHtmlBytes: htmlContent.length,
   };
 }

lib/parsers/copilot.ts

@@ -1,15 +1,57 @@
 import type { Conversation } from '@/types/conversation';
+import { JSDOM } from 'jsdom';
+import fs from 'fs';
+import path from 'path';
 
 /**
  * Extracts a Copilot share page into a structured Conversation.
  * @param html - Raw HTML content from the Copilot share page
  * @returns Promise resolving to a structured Conversation object
  */
 export async function parseCopilot(html: string): Promise<Conversation> {
+  const dom = new JSDOM(html);
+  const document = dom.window.document;
+
+  // Clean up elements first
+  const miscElems = document.querySelectorAll('button');
+  miscElems.forEach(button => button.remove());
+
+  // Find container
+  const chatContainer = document.querySelector('div[data-content="conversation"]');
+  if (!chatContainer) {
+    throw new Error('Conversation container not found');
+  }
+
+  // Make sure each conversation block is visible
+  const hiddenDivs = chatContainer.querySelectorAll('div[style*="opacity: 0"]');
+  hiddenDivs.forEach(div => {
+    (div as HTMLElement).style.opacity = '1';
+  });
+
+  // Read CSS separately and inject it directly into final HTML
+  const cssPath = path.resolve(process.cwd(), 'lib/parsers/assets/copilot.css');
+  const css = fs.readFileSync(cssPath, 'utf-8');
+
+  // Build final HTML
+  const htmlContent = `
+  <!DOCTYPE html>
+  <html data-theme="light">
+    <head>
+      <meta charset="utf-8">
+      <style>
+        ${css}
+      </style>
+    </head>
+    <body>
+      ${chatContainer.outerHTML}
+    </body>
+  </html>
+  `;
+
   return {
     model: 'Copilot',
-    content: html,
+    content: htmlContent,
     scrapedAt: new Date().toISOString(),
-    sourceHtmlBytes: html.length,
+    sourceHtmlBytes: htmlContent.length,
   };
 }

lib/parsers/deepseek.ts

@@ -1,15 +1,55 @@
 import type { Conversation } from '@/types/conversation';
+import { JSDOM } from 'jsdom';
+import fs from 'fs';
+import path from 'path';
 
 /**
  * Extracts a DeepSeek share page into a structured Conversation.
  * @param html - Raw HTML content from the DeepSeek share page
  * @returns Promise resolving to a structured Conversation object
  */
 export async function parseDeepSeek(html: string): Promise<Conversation> {
+  const dom = new JSDOM(html);
+  const document = dom.window.document;
+
+  // Clean up elements first
+  const miscElems = document.querySelectorAll('button');
+  miscElems.forEach(button => button.remove());
+
+  // Inject <style> from deepseek.css
+  const cssPath = path.resolve(process.cwd(), 'lib/parsers/assets/deepseek.css');
+  const css = fs.readFileSync(cssPath, 'utf-8');
+
+  const styleTag = document.createElement('style');
+  styleTag.textContent = css;
+  document.head.appendChild(styleTag);
+
+  // Clean up icons and extra UI elements
+  const iconSelectors = ['.ds-icon-button'];
+  iconSelectors.forEach(selector => {
+    document.querySelectorAll(selector).forEach(el => el.remove());
+  });
+
+  // Grab main conversation container
+  const chatContainer = document.querySelector('div.dad65929');
+  if (!chatContainer) {
+    throw new Error('Conversation container not found');
+  }
+
+  // Prepare the final HTML output
+  const htmlContent = `
+    <html>
+      <head>${document.head.innerHTML}</head>
+      <body class="light-theme theme-host">
+        ${chatContainer.outerHTML}
+      </body>
+    </html>
+  `;
+
   return {
     model: 'DeepSeek',
-    content: html,
+    content: htmlContent,
     scrapedAt: new Date().toISOString(),
-    sourceHtmlBytes: html.length,
+    sourceHtmlBytes: htmlContent.length,
   };
 }

lib/parsers/gemini.ts

@@ -1,13 +1,50 @@
 import type { Conversation } from '@/types/conversation';
+import { JSDOM } from 'jsdom';
 
 /**
  * Extracts a Gemini share page into a structured Conversation.
  */
 export async function parseGemini(html: string): Promise<Conversation> {
+
+  const dom = new JSDOM(html);
+  const document = dom.window.document;
+
+  // Grab the document styling
+  const styleTags = document.head.querySelectorAll('style');
+  let combinedStyles = '';
+
+  styleTags.forEach(styleTag => {
+    styleTag.textContent = "html{overflow:scroll}" + styleTag.textContent;
+    combinedStyles += styleTag.outerHTML + '\n';
+  });
+
+  const style = document.createElement("style");
+  style.textContent = combinedStyles;
+
+  // Remove all misc elements
+  const miscElems = document.querySelectorAll('sources-list, mat-icon, [class*="actions-container-v2"], [class*="source-inline-chip-container"], [class*="class-thoughts"]');
+  miscElems.forEach(element => element.remove());
+
+  // Grab the chat history container
+  const chatHistory = document.querySelector('[data-test-id="chat-history-container"]')
+
+  // Add the styling to the container
+  if (chatHistory !== null) {
+    // Add body document styling that was removed
+    const children = chatHistory.children;
+    for (const child of children) {
+      child.classList.add('theme-host', 'light-theme');
+    }
+
+    chatHistory.prepend(style);
+  }
+
+  const chat = chatHistory?.innerHTML ?? "";
+
   return {
     model: 'Gemini',
-    content: html,
+    content: chat,
     scrapedAt: new Date().toISOString(),
-    sourceHtmlBytes: html.length,
+    sourceHtmlBytes: chat.length,
   };
 }