From 9809c92ec27fb96306f32fca0ab75517ec999e97 Mon Sep 17 00:00:00 2001 From: breken-ai <312387581+breken-ai@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:13:50 -0700 Subject: [PATCH 1/2] fix(scrape): keep every page when multi-URL filenames collide `firecrawl scrape` with several URLs saves each page to `.firecrawl/-.md`. The name drops the query string and turns `/` into `-`, so `list?page=1` and `list?page=2`, or `/a/b` and `/a-b`, wrote to the same file. The later page overwrote the earlier one while the run still reported every URL as saved. Filenames are now picked up front in URL order, and a name already used in the batch gets a `-2`, `-3`, ... suffix. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../commands/multi-scrape-filenames.test.ts | 91 +++++++++++++++++++ src/commands/scrape.ts | 25 ++++- 2 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 src/__tests__/commands/multi-scrape-filenames.test.ts diff --git a/src/__tests__/commands/multi-scrape-filenames.test.ts b/src/__tests__/commands/multi-scrape-filenames.test.ts new file mode 100644 index 0000000000..c768ff13e1 --- /dev/null +++ b/src/__tests__/commands/multi-scrape-filenames.test.ts @@ -0,0 +1,91 @@ +/** + * Tests for where multi-URL scrape saves each page + */ + +import * as fs from 'fs'; +import { join } from 'path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { handleMultiScrapeCommand } from '../../commands/scrape'; +import { getClient } from '../../utils/client'; + +vi.mock('../../utils/client', () => ({ + getClient: vi.fn(), + isKeylessMode: () => false, + keylessRequest: vi.fn(), +})); +vi.mock('../../utils/interact-session', () => ({ + saveInteractSession: vi.fn(), + clearInteractSession: vi.fn(), +})); +vi.mock('fs', async () => ({ + ...(await vi.importActual('fs')), + existsSync: vi.fn(() => true), + mkdirSync: vi.fn(), + writeFileSync: vi.fn(), +})); + +describe('multi-URL scrape filenames', () => { + beforeEach(() => { + vi.mocked(getClient).mockReturnValue({ + scrape: vi.fn(async (url: string) => ({ markdown: `page ${url}` })), + } as any); + vi.spyOn(process.stderr, 'write').mockReturnValue(true); + }); + + afterEach(() => { + vi.clearAllMocks(); + vi.restoreAllMocks(); + }); + + it('saves every URL to its own file when their names would collide', async () => { + const urls = [ + 'https://example.com/list?page=1', + 'https://example.com/list?page=2', + 'https://example.com/a/b', + 'https://example.com/a-b', + ]; + + await handleMultiScrapeCommand(urls, { url: urls[0] }); + + const written = new Map( + vi + .mocked(fs.writeFileSync) + .mock.calls.map(([file, content]) => [String(file), String(content)]) + ); + expect(written).toEqual( + new Map([ + [ + join('.firecrawl', 'example.com-list.md'), + 'page https://example.com/list?page=1', + ], + [ + join('.firecrawl', 'example.com-list-2.md'), + 'page https://example.com/list?page=2', + ], + [ + join('.firecrawl', 'example.com-a-b.md'), + 'page https://example.com/a/b', + ], + [ + join('.firecrawl', 'example.com-a-b-2.md'), + 'page https://example.com/a-b', + ], + ]) + ); + }); + + it('keeps the plain name for URLs that do not collide', async () => { + const urls = ['https://example.com/', 'https://example.com/docs/intro']; + + await handleMultiScrapeCommand(urls, { url: urls[0] }); + + const files = vi + .mocked(fs.writeFileSync) + .mock.calls.map(([file]) => String(file)) + .sort(); + expect(files).toEqual([ + join('.firecrawl', 'example.com-docs-intro.md'), + join('.firecrawl', 'example.com.md'), + ]); + }); +}); diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 5407275df6..0642713aab 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -255,6 +255,24 @@ function urlToFilename(url: string): string { } } +/** + * Pick one filename per URL. URLs that differ only in their query string or + * in `/` versus `-` in the path map to the same name, so later ones get a + * numeric suffix instead of overwriting an earlier page from the same batch. + */ +function uniqueFilenames(urls: string[]): string[] { + const used = new Set(); + return urls.map((url) => { + const base = urlToFilename(url); + let name = base; + for (let n = 2; used.has(name); n++) { + name = base.replace(/\.md$/, `-${n}.md`); + } + used.add(name); + return name; + }); +} + /** * Handle scrape for multiple URLs. * Each result is saved as a separate file in .firecrawl/ @@ -277,7 +295,9 @@ export async function handleMultiScrapeCommand( process.stderr.write(`Scraping ${total} URLs...\n`); - const promises = urls.map(async (url) => { + const filenames = uniqueFilenames(urls); + + const promises = urls.map(async (url, index) => { const scrapeOptions: ScrapeOptions = { ...options, url }; const result = await executeScrape(scrapeOptions); @@ -291,8 +311,7 @@ export async function handleMultiScrapeCommand( return; } - const filename = urlToFilename(url); - const filepath = path.join(dir, filename); + const filepath = path.join(dir, filenames[index]); const content = result.data?.markdown || JSON.stringify(result.data); fs.writeFileSync(filepath, content, 'utf-8'); From db4636e1422fa2bbb4db43bdd600860692bf27e2 Mon Sep 17 00:00:00 2001 From: breken-ai <312387581+breken-ai@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:23:08 -0700 Subject: [PATCH 2/2] fix(scrape): reserve plain names first and compare them case-insensitively A suffixed name no longer takes the plain name of another URL in the batch (/a?x=1, /a?x=2, /a-2), and /Foo and /foo no longer share a file on case-insensitive file systems. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../commands/multi-scrape-filenames.test.ts | 47 +++++++++++++++++++ src/commands/scrape.ts | 20 ++++++-- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/__tests__/commands/multi-scrape-filenames.test.ts b/src/__tests__/commands/multi-scrape-filenames.test.ts index c768ff13e1..27ed9a4261 100644 --- a/src/__tests__/commands/multi-scrape-filenames.test.ts +++ b/src/__tests__/commands/multi-scrape-filenames.test.ts @@ -88,4 +88,51 @@ describe('multi-URL scrape filenames', () => { join('.firecrawl', 'example.com.md'), ]); }); + + it('does not give a suffixed name that another URL in the batch needs', async () => { + const urls = [ + 'https://example.com/a?x=1', + 'https://example.com/a?x=2', + 'https://example.com/a-2', + ]; + + await handleMultiScrapeCommand(urls, { url: urls[0] }); + + const written = new Map( + vi + .mocked(fs.writeFileSync) + .mock.calls.map(([file, content]) => [String(file), String(content)]) + ); + expect(written).toEqual( + new Map([ + [ + join('.firecrawl', 'example.com-a.md'), + 'page https://example.com/a?x=1', + ], + [ + join('.firecrawl', 'example.com-a-3.md'), + 'page https://example.com/a?x=2', + ], + [ + join('.firecrawl', 'example.com-a-2.md'), + 'page https://example.com/a-2', + ], + ]) + ); + }); + + it('treats names that differ only in case as a collision', async () => { + const urls = ['https://example.com/Foo', 'https://example.com/foo']; + + await handleMultiScrapeCommand(urls, { url: urls[0] }); + + const files = vi + .mocked(fs.writeFileSync) + .mock.calls.map(([file]) => String(file)) + .sort(); + expect(files).toEqual([ + join('.firecrawl', 'example.com-Foo.md'), + join('.firecrawl', 'example.com-foo-2.md'), + ]); + }); }); diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 0642713aab..d579ffb323 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -259,16 +259,26 @@ function urlToFilename(url: string): string { * Pick one filename per URL. URLs that differ only in their query string or * in `/` versus `-` in the path map to the same name, so later ones get a * numeric suffix instead of overwriting an earlier page from the same batch. + * Every URL's plain name is reserved before any suffix is handed out, and + * names are compared case-insensitively because macOS and Windows file + * systems are. */ function uniqueFilenames(urls: string[]): string[] { - const used = new Set(); - return urls.map((url) => { - const base = urlToFilename(url); + const bases = urls.map(urlToFilename); + const used = new Set(bases.map((name) => name.toLowerCase())); + const claimed = new Set(); + return bases.map((base) => { + const key = base.toLowerCase(); + if (!claimed.has(key)) { + claimed.add(key); + return base; + } let name = base; - for (let n = 2; used.has(name); n++) { + for (let n = 2; used.has(name.toLowerCase()); n++) { name = base.replace(/\.md$/, `-${n}.md`); } - used.add(name); + used.add(name.toLowerCase()); + claimed.add(name.toLowerCase()); return name; }); }