fix(scrape): keep every page when multi-URL filenames collide

src/__tests__/commands/multi-scrape-filenames.test.ts

@@ -0,0 +1,138 @@
+/**
+ * Tests for where multi-URL scrape saves each page
+ */
+
+import * as fs from 'fs';
+import { join } from 'path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { handleMultiScrapeCommand } from '../../commands/scrape';
+import { getClient } from '../../utils/client';
+
+vi.mock('../../utils/client', () => ({
+  getClient: vi.fn(),
+  isKeylessMode: () => false,
+  keylessRequest: vi.fn(),
+}));
+vi.mock('../../utils/interact-session', () => ({
+  saveInteractSession: vi.fn(),
+  clearInteractSession: vi.fn(),
+}));
+vi.mock('fs', async () => ({
+  ...(await vi.importActual<typeof import('fs')>('fs')),
+  existsSync: vi.fn(() => true),
+  mkdirSync: vi.fn(),
+  writeFileSync: vi.fn(),
+}));
+
+describe('multi-URL scrape filenames', () => {
+  beforeEach(() => {
+    vi.mocked(getClient).mockReturnValue({
+      scrape: vi.fn(async (url: string) => ({ markdown: `page ${url}` })),
+    } as any);
+    vi.spyOn(process.stderr, 'write').mockReturnValue(true);
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+    vi.restoreAllMocks();
+  });
+
+  it('saves every URL to its own file when their names would collide', async () => {
+    const urls = [
+      'https://example.com/list?page=1',
+      'https://example.com/list?page=2',
+      'https://example.com/a/b',
+      'https://example.com/a-b',
+    ];
+
+    await handleMultiScrapeCommand(urls, { url: urls[0] });
+
+    const written = new Map(
+      vi
+        .mocked(fs.writeFileSync)
+        .mock.calls.map(([file, content]) => [String(file), String(content)])
+    );
+    expect(written).toEqual(
+      new Map([
+        [
+          join('.firecrawl', 'example.com-list.md'),
+          'page https://example.com/list?page=1',
+        ],
+        [
+          join('.firecrawl', 'example.com-list-2.md'),
+          'page https://example.com/list?page=2',
+        ],
+        [
+          join('.firecrawl', 'example.com-a-b.md'),
+          'page https://example.com/a/b',
+        ],
+        [
+          join('.firecrawl', 'example.com-a-b-2.md'),
+          'page https://example.com/a-b',
+        ],
+      ])
+    );
+  });
+
+  it('keeps the plain name for URLs that do not collide', async () => {
+    const urls = ['https://example.com/', 'https://example.com/docs/intro'];
+
+    await handleMultiScrapeCommand(urls, { url: urls[0] });
+
+    const files = vi
+      .mocked(fs.writeFileSync)
+      .mock.calls.map(([file]) => String(file))
+      .sort();
+    expect(files).toEqual([
+      join('.firecrawl', 'example.com-docs-intro.md'),
+      join('.firecrawl', 'example.com.md'),
+    ]);
+  });
+
+  it('does not give a suffixed name that another URL in the batch needs', async () => {
+    const urls = [
+      'https://example.com/a?x=1',
+      'https://example.com/a?x=2',
+      'https://example.com/a-2',
+    ];
+
+    await handleMultiScrapeCommand(urls, { url: urls[0] });
+
+    const written = new Map(
+      vi
+        .mocked(fs.writeFileSync)
+        .mock.calls.map(([file, content]) => [String(file), String(content)])
+    );
+    expect(written).toEqual(
+      new Map([
+        [
+          join('.firecrawl', 'example.com-a.md'),
+          'page https://example.com/a?x=1',
+        ],
+        [
+          join('.firecrawl', 'example.com-a-3.md'),
+          'page https://example.com/a?x=2',
+        ],
+        [
+          join('.firecrawl', 'example.com-a-2.md'),
+          'page https://example.com/a-2',
+        ],
+      ])
+    );
+  });
+
+  it('treats names that differ only in case as a collision', async () => {
+    const urls = ['https://example.com/Foo', 'https://example.com/foo'];
+
+    await handleMultiScrapeCommand(urls, { url: urls[0] });
+
+    const files = vi
+      .mocked(fs.writeFileSync)
+      .mock.calls.map(([file]) => String(file))
+      .sort();
+    expect(files).toEqual([
+      join('.firecrawl', 'example.com-Foo.md'),
+      join('.firecrawl', 'example.com-foo-2.md'),
+    ]);
+  });
+});

src/commands/scrape.ts

@@ -255,6 +255,34 @@ function urlToFilename(url: string): string {
   }
 }
 
+/**
+ * Pick one filename per URL. URLs that differ only in their query string or
+ * in `/` versus `-` in the path map to the same name, so later ones get a
+ * numeric suffix instead of overwriting an earlier page from the same batch.
+ * Every URL's plain name is reserved before any suffix is handed out, and
+ * names are compared case-insensitively because macOS and Windows file
+ * systems are.
+ */
+function uniqueFilenames(urls: string[]): string[] {
+  const bases = urls.map(urlToFilename);
+  const used = new Set(bases.map((name) => name.toLowerCase()));
+  const claimed = new Set<string>();
+  return bases.map((base) => {
+    const key = base.toLowerCase();
+    if (!claimed.has(key)) {
+      claimed.add(key);
+      return base;
+    }
+    let name = base;
+    for (let n = 2; used.has(name.toLowerCase()); n++) {
+      name = base.replace(/\.md$/, `-${n}.md`);
+    }
+    used.add(name.toLowerCase());
+    claimed.add(name.toLowerCase());
+    return name;
+  });
+}
+
 /**
  * Handle scrape for multiple URLs.
  * Each result is saved as a separate file in .firecrawl/
@@ -277,7 +305,9 @@ export async function handleMultiScrapeCommand(
 
   process.stderr.write(`Scraping ${total} URLs...\n`);
 
-  const promises = urls.map(async (url) => {
+  const filenames = uniqueFilenames(urls);
+
+  const promises = urls.map(async (url, index) => {
     const scrapeOptions: ScrapeOptions = { ...options, url };
     const result = await executeScrape(scrapeOptions);
 
@@ -291,8 +321,7 @@ export async function handleMultiScrapeCommand(
       return;
     }
 
-    const filename = urlToFilename(url);
-    const filepath = path.join(dir, filename);
+    const filepath = path.join(dir, filenames[index]);
     const content = result.data?.markdown || JSON.stringify(result.data);
     fs.writeFileSync(filepath, content, 'utf-8');