diff --git a/src/__tests__/commands/scrape.test.ts b/src/__tests__/commands/scrape.test.ts index 13fe04157c..fdf98c746e 100644 --- a/src/__tests__/commands/scrape.test.ts +++ b/src/__tests__/commands/scrape.test.ts @@ -248,6 +248,64 @@ describe('executeScrape', () => { maxAge: 3600, }); }); + + it('should include location parameter with country and languages', async () => { + const mockResponse = { markdown: '# Test' }; + mockClient.scrape.mockResolvedValue(mockResponse); + + await executeScrape({ + url: 'https://example.com', + location: { country: 'US', languages: ['en'] }, + }); + + expect(mockClient.scrape).toHaveBeenCalledWith('https://example.com', { + formats: ['markdown'], + location: { country: 'US', languages: ['en'] }, + }); + }); + + it('should include location parameter with only country', async () => { + const mockResponse = { markdown: '# Test' }; + mockClient.scrape.mockResolvedValue(mockResponse); + + await executeScrape({ + url: 'https://example.com', + location: { country: 'DE' }, + }); + + expect(mockClient.scrape).toHaveBeenCalledWith('https://example.com', { + formats: ['markdown'], + location: { country: 'DE' }, + }); + }); + + it('should include location parameter with only languages', async () => { + const mockResponse = { markdown: '# Test' }; + mockClient.scrape.mockResolvedValue(mockResponse); + + await executeScrape({ + url: 'https://example.com', + location: { languages: ['en', 'es'] }, + }); + + expect(mockClient.scrape).toHaveBeenCalledWith('https://example.com', { + formats: ['markdown'], + location: { languages: ['en', 'es'] }, + }); + }); + + it('should not include location parameter when not provided', async () => { + const mockResponse = { markdown: '# Test' }; + mockClient.scrape.mockResolvedValue(mockResponse); + + await executeScrape({ + url: 'https://example.com', + }); + + expect(mockClient.scrape).toHaveBeenCalledWith('https://example.com', { + formats: ['markdown'], + }); + }); }); describe('Response handling', () => { diff --git a/src/__tests__/utils/options.test.ts b/src/__tests__/utils/options.test.ts index 257a2be52c..5e7f5e34ca 100644 --- a/src/__tests__/utils/options.test.ts +++ b/src/__tests__/utils/options.test.ts @@ -347,5 +347,69 @@ describe('Option Parsing Utilities', () => { timing: true, }); }); + + it('should parse location with country and languages', () => { + const options = { + url: 'https://example.com', + country: 'US', + languages: 'en,es', + }; + + const result = parseScrapeOptions(options); + + expect(result.location).toEqual({ + country: 'US', + languages: ['en', 'es'], + }); + }); + + it('should parse location with only country', () => { + const options = { + url: 'https://example.com', + country: 'DE', + }; + + const result = parseScrapeOptions(options); + + expect(result.location).toEqual({ + country: 'DE', + }); + }); + + it('should parse location with only languages', () => { + const options = { + url: 'https://example.com', + languages: 'en,fr,de', + }; + + const result = parseScrapeOptions(options); + + expect(result.location).toEqual({ + languages: ['en', 'fr', 'de'], + }); + }); + + it('should not set location when neither country nor languages provided', () => { + const options = { + url: 'https://example.com', + }; + + const result = parseScrapeOptions(options); + + expect(result.location).toBeUndefined(); + }); + + it('should trim whitespace from language codes', () => { + const options = { + url: 'https://example.com', + languages: ' en , es , fr ', + }; + + const result = parseScrapeOptions(options); + + expect(result.location).toEqual({ + languages: ['en', 'es', 'fr'], + }); + }); }); }); diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index e6192831a2..d3bd97f9e6 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -7,6 +7,7 @@ import type { ScrapeOptions, ScrapeResult, ScrapeFormat, + ScrapeLocation, } from '../types/scrape'; import { getClient } from '../utils/client'; import { handleScrapeOutput } from '../utils/output'; @@ -77,6 +78,7 @@ export async function executeScrape( includeTags?: string[]; excludeTags?: string[]; maxAge?: number; + location?: ScrapeLocation; } = { formats, }; @@ -101,6 +103,10 @@ export async function executeScrape( scrapeParams.maxAge = options.maxAge; } + if (options.location) { + scrapeParams.location = options.location; + } + // Execute scrape with timing - only wrap the scrape call in try-catch const requestStartTime = Date.now(); diff --git a/src/index.ts b/src/index.ts index c36c9dff7d..e45ec609e5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -126,6 +126,14 @@ function createScrapeCommand(): Command { 'Maximum age of cached content in milliseconds', parseInt ) + .option( + '--country ', + 'ISO country code for geo-targeted scraping (e.g., US, DE, BR)' + ) + .option( + '--languages ', + 'Comma-separated language codes for scraping (e.g., en,es)' + ) .action(async (positionalUrl, positionalFormats, options) => { // Use positional URL if provided, otherwise use --url option const url = positionalUrl || options.url; diff --git a/src/types/scrape.ts b/src/types/scrape.ts index 76411541fb..ba32b60c8c 100644 --- a/src/types/scrape.ts +++ b/src/types/scrape.ts @@ -15,6 +15,13 @@ export type ScrapeFormat = | 'attributes' | 'branding'; +export interface ScrapeLocation { + /** ISO 3166-1 alpha-2 country code (e.g., 'US', 'DE', 'BR') */ + country?: string; + /** List of language codes (e.g., ['en', 'es']) */ + languages?: string[]; +} + export interface ScrapeOptions { /** URL to scrape */ url: string; @@ -44,6 +51,8 @@ export interface ScrapeOptions { timing?: boolean; /** Maximum age of cached content in milliseconds (API-level caching) */ maxAge?: number; + /** Location settings for geo-targeted scraping */ + location?: ScrapeLocation; } export interface ScrapeResult { diff --git a/src/utils/options.ts b/src/utils/options.ts index efed091e27..2d57ec4767 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -2,7 +2,11 @@ * Option parsing utilities */ -import type { ScrapeOptions, ScrapeFormat } from '../types/scrape'; +import type { + ScrapeOptions, + ScrapeFormat, + ScrapeLocation, +} from '../types/scrape'; /** * Valid scrape format values @@ -72,6 +76,20 @@ export function parseScrapeOptions(options: any): ScrapeOptions { formats = parseFormats(options.format); } + // Build location object if country or languages are provided + let location: ScrapeLocation | undefined; + if (options.country || options.languages) { + location = {}; + if (options.country) { + location.country = options.country; + } + if (options.languages) { + location.languages = options.languages + .split(',') + .map((l: string) => l.trim()); + } + } + return { url: options.url, formats, @@ -91,5 +109,6 @@ export function parseScrapeOptions(options: any): ScrapeOptions { json: options.json, timing: options.timing, maxAge: options.maxAge, + location, }; }