From e5fc1636a93ac41b9174a14d88742ee1cfb6957c Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Thu, 3 Sep 2026 14:49:58 +0100 Subject: [PATCH 1/4] fix: X blocks bundled headless Chromium with 403 Playwright's bundled headless Chromium gets a 403 from x.com on every request, regardless of User-Agent. Fix by: - setting a normal desktop Chrome UA on the context (removes the HeadlessChrome tell) - preferring a locally installed real Chrome (channel: chrome) over bundled Chromium when no explicit --browser-channel/--browser-executable-path is given, since X fingerprints the bundled binary itself and blocks it outright even with a spoofed UA Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014i3FdrzgdTStsSytFiyzyV --- src/extractor.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/extractor.ts b/src/extractor.ts index 3896dfb..f48d9e9 100644 --- a/src/extractor.ts +++ b/src/extractor.ts @@ -12,6 +12,9 @@ import { parseTweetUrl, } from './utils.ts'; +const DESKTOP_CHROME_UA = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'; + type ExtractCandidate = { url: string; format: VideoUrl['format']; @@ -145,10 +148,18 @@ export class VideoExtractor { launchOptions.executablePath = this.browserExecutablePath; } else if (this.browserChannel) { launchOptions.channel = this.browserChannel; + } else { + // X blocks Playwright's bundled Chromium outright (403 regardless of UA); + // prefer a real installed Chrome, which it doesn't fingerprint the same way. + const { findChromePath } = await import('./private.ts'); + if (findChromePath()) { + launchOptions.channel = 'chrome'; + } } const browser = await chromium.launch(launchOptions); - const context = await browser.newContext(); + // Headless Chromium's default UA includes "HeadlessChrome", which X blocks with a 403. + const context = await browser.newContext({ userAgent: DESKTOP_CHROME_UA }); const page = await context.newPage(); return { browser, context, page }; } From fb0ea8f5ffd8e81eb3d7642a84da134dbab93455 Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Thu, 3 Sep 2026 14:58:09 +0100 Subject: [PATCH 2/4] fix: improve Chrome detection and UA string realism - Use static import of findChromePath instead of dynamic await import - Use executablePath when Chrome found, not channel: 'chrome' (more robust) - Update hardcoded UA to macOS 14 (Sonoma) + Chrome 132 (realistic pairing) - Add comment explaining UA needs periodic updates - Add Windows support to findChromePath with common installation paths - Simplify comments to explain 'why' not 'what' Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014i3FdrzgdTStsSytFiyzyV --- bun.lock | 6 +++--- package.json | 2 +- src/extractor.ts | 35 ++++++++++++++++++----------------- src/private.ts | 17 ++++++++++++++++- 4 files changed, 38 insertions(+), 22 deletions(-) diff --git a/bun.lock b/bun.lock index 74a0c24..47fb0a2 100644 --- a/bun.lock +++ b/bun.lock @@ -5,15 +5,15 @@ "": { "name": "x-dl", "dependencies": { - "playwright": "^1.57.0", + "playwright": "^1.60.0", }, }, }, "packages": { "fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], - "playwright": ["playwright@1.57.0", "", { "dependencies": { "playwright-core": "1.57.0" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-ilYQj1s8sr2ppEJ2YVadYBN0Mb3mdo9J0wQ+UuDhzYqURwSoW4n1Xs5vs7ORwgDGmyEh33tRMeS8KhdkMoLXQw=="], + "playwright": ["playwright@1.62.1", "", { "dependencies": { "playwright-core": "1.62.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg=="], - "playwright-core": ["playwright-core@1.57.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ=="], + "playwright-core": ["playwright-core@1.62.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw=="], } } diff --git a/package.json b/package.json index eaf9295..775bcef 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "author": "", "license": "MIT", "dependencies": { - "playwright": "^1.57.0" + "playwright": "^1.60.0" }, "devDependencies": {} } diff --git a/src/extractor.ts b/src/extractor.ts index f48d9e9..787160b 100644 --- a/src/extractor.ts +++ b/src/extractor.ts @@ -11,9 +11,12 @@ import { isValidTwitterUrl, parseTweetUrl, } from './utils.ts'; +import { findChromePath } from './private.ts'; +// Looks like desktop Chrome, not headless. X blocks the HeadlessChrome UA string. +// Version needs periodic updates to remain realistic (currently ~2 years old). const DESKTOP_CHROME_UA = - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'; + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36'; type ExtractCandidate = { url: string; @@ -40,7 +43,7 @@ export class VideoExtractor { } async extract(url: string, authenticatedPage?: Page): Promise { - console.log(`\ud83c\udfac Extracting video from: ${url}`); + console.log(`🎬 Extracting video from: ${url}`); if (!isValidTwitterUrl(url)) { return { @@ -59,7 +62,7 @@ export class VideoExtractor { }; } - console.log(`\ud83d\udcdd Tweet: @${tweetInfo.author} (ID: ${tweetInfo.id})`); + console.log(`📝 Tweet: @${tweetInfo.author} (ID: ${tweetInfo.id})`); const { chromium } = await import('playwright'); @@ -82,7 +85,7 @@ export class VideoExtractor { candidates.add(u); }); - console.log('\ud83c\udf10 Opening tweet in browser...'); + console.log('🌐 Opening tweet in browser...'); await page.goto(url, { waitUntil: 'domcontentloaded', timeout: this.timeout }); // Give X a moment to hydrate @@ -92,7 +95,7 @@ export class VideoExtractor { const loginWall = hasLoginWall(pageHtml); if (loginWall) { - console.log('\u26a0\ufe0f Login wall detected; trying to extract anyway...'); + console.log('⚠️ Login wall detected; trying to extract anyway...'); } // Try to trigger media loading. @@ -119,8 +122,8 @@ export class VideoExtractor { } const filename = generateFilename(tweetInfo); - console.log(`\u2705 Video extracted: ${videoUrl.url}`); - console.log(`\ud83d\udccb Suggested filename: ${filename}`); + console.log(`✅ Video extracted: ${videoUrl.url}`); + console.log(`📋 Suggested filename: ${filename}`); return { videoUrl }; } catch (error) { @@ -149,16 +152,14 @@ export class VideoExtractor { } else if (this.browserChannel) { launchOptions.channel = this.browserChannel; } else { - // X blocks Playwright's bundled Chromium outright (403 regardless of UA); - // prefer a real installed Chrome, which it doesn't fingerprint the same way. - const { findChromePath } = await import('./private.ts'); - if (findChromePath()) { - launchOptions.channel = 'chrome'; + const chromePath = findChromePath(); + if (chromePath) { + // X fingerprints bundled Chromium itself; prefer a real Chrome installation. + launchOptions.executablePath = chromePath; } } const browser = await chromium.launch(launchOptions); - // Headless Chromium's default UA includes "HeadlessChrome", which X blocks with a 403. const context = await browser.newContext({ userAgent: DESKTOP_CHROME_UA }); const page = await context.newPage(); return { browser, context, page }; @@ -202,7 +203,7 @@ export class VideoExtractor { const htmlPath = path.join(this.debugArtifactsDir, `${prefix}.html`); fs.writeFileSync(htmlPath, pageHtml, 'utf-8'); debugInfo.htmlPath = htmlPath; - console.log(`\ud83d\udcc3 HTML saved to: ${htmlPath}`); + console.log(`📄 HTML saved to: ${htmlPath}`); } // Save screenshot @@ -210,7 +211,7 @@ export class VideoExtractor { const screenshotPath = path.join(this.debugArtifactsDir, `${prefix}.png`); await page.screenshot({ path: screenshotPath, fullPage: true }); debugInfo.screenshotPath = screenshotPath; - console.log(`\ud83d\udcf7 Screenshot saved to: ${screenshotPath}`); + console.log(`📷 Screenshot saved to: ${screenshotPath}`); } catch { // Screenshot failed, continue without it } @@ -218,7 +219,7 @@ export class VideoExtractor { return debugInfo; } catch (error) { console.warn( - `\u26a0\ufe0f Failed to save debug artifacts: ${error instanceof Error ? error.message : 'Unknown error'}` + `⚠️ Failed to save debug artifacts: ${error instanceof Error ? error.message : 'Unknown error'}` ); return undefined; } @@ -280,7 +281,7 @@ export class VideoExtractor { page: Page; networkCandidates: string[]; }): Promise { - console.log('\ud83d\udd0d Looking for video...'); + console.log('🔍 Looking for video...'); const perfCandidates = await this.getPerformanceCandidates(page); const domCandidates = await this.getDomCandidates(page); diff --git a/src/private.ts b/src/private.ts index 3c2105d..0ee40ac 100644 --- a/src/private.ts +++ b/src/private.ts @@ -17,9 +17,24 @@ const CHROME_PATHS_LINUX = [ '/snap/bin/chromium', ]; +const CHROME_PATHS_WINDOWS = [ + path.join(os.homedir(), 'AppData/Local/Google/Chrome/Application/chrome.exe'), + 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe', + 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe', +]; + export function findChromePath(): string | null { const platform = os.platform(); - const candidates = platform === 'darwin' ? CHROME_PATHS_MACOS : CHROME_PATHS_LINUX; + let candidates: string[] = []; + + if (platform === 'darwin') { + candidates = CHROME_PATHS_MACOS; + } else if (platform === 'linux') { + candidates = CHROME_PATHS_LINUX; + } else if (platform === 'win32') { + candidates = CHROME_PATHS_WINDOWS; + } + for (const p of candidates) { if (fs.existsSync(p)) return p; } From 43575fbc8b707a9bd92ca0f258ccd46563e120fe Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Thu, 3 Sep 2026 16:24:05 +0100 Subject: [PATCH 3/4] fix: apply desktop Chrome UA to private/CDP login browser too Devin Review correctly flagged that the CDP and login-retry flows (x-dl cdp, and the automatic login-wall retry) launch their own headless Chrome via launchPrivateBrowser, bypassing createContextAndPage entirely. That context never got the desktop UA override, so it kept the default HeadlessChrome UA and hit the same 403 the original fix addressed, just on a different path. Move DESKTOP_CHROME_UA into private.ts and apply it in launchPrivateBrowser whenever running headless (a headed, user-visible login flow doesn't need the spoof). extractor.ts now imports the constant instead of duplicating it. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014i3FdrzgdTStsSytFiyzyV --- src/extractor.ts | 7 +------ src/private.ts | 9 ++++++++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/extractor.ts b/src/extractor.ts index 787160b..19caaf9 100644 --- a/src/extractor.ts +++ b/src/extractor.ts @@ -11,12 +11,7 @@ import { isValidTwitterUrl, parseTweetUrl, } from './utils.ts'; -import { findChromePath } from './private.ts'; - -// Looks like desktop Chrome, not headless. X blocks the HeadlessChrome UA string. -// Version needs periodic updates to remain realistic (currently ~2 years old). -const DESKTOP_CHROME_UA = - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36'; +import { findChromePath, DESKTOP_CHROME_UA } from './private.ts'; type ExtractCandidate = { url: string; diff --git a/src/private.ts b/src/private.ts index 0ee40ac..061d9ad 100644 --- a/src/private.ts +++ b/src/private.ts @@ -5,6 +5,10 @@ import type { BrowserContext, Page } from 'playwright'; const DEFAULT_PROFILE_DIR = path.join(os.homedir(), '.x-dl-chrome-profile'); +// Headless Chrome's default UA includes "HeadlessChrome", which X blocks with a 403. +export const DESKTOP_CHROME_UA = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36'; + const CHROME_PATHS_MACOS = [ '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', ]; @@ -56,9 +60,12 @@ export async function launchPrivateBrowser(options?: { }): Promise { const { chromium } = await import('playwright'); + const headless = !(options?.headed); + const context = await chromium.launchPersistentContext(DEFAULT_PROFILE_DIR, { channel: 'chrome', - headless: !(options?.headed), + headless, + userAgent: headless ? DESKTOP_CHROME_UA : undefined, args: [ '--disable-blink-features=AutomationControlled', ], From def636c7bf0b30a6d8917a8313c5d1bfb45439ed Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Thu, 3 Sep 2026 17:03:38 +0100 Subject: [PATCH 4/4] fix: make UA spoof policy consistent (headless-only in both paths) createContextAndPage was always applying DESKTOP_CHROME_UA, including in headed mode, while launchPrivateBrowser only applies it when headless. This was inconsistent: headed browsers (used for login or user testing) don't need and shouldn't need UA spoofing. Both now apply the override only in headless mode. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014i3FdrzgdTStsSytFiyzyV --- src/extractor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extractor.ts b/src/extractor.ts index 19caaf9..7fd7f09 100644 --- a/src/extractor.ts +++ b/src/extractor.ts @@ -155,7 +155,7 @@ export class VideoExtractor { } const browser = await chromium.launch(launchOptions); - const context = await browser.newContext({ userAgent: DESKTOP_CHROME_UA }); + const context = await browser.newContext({ userAgent: !this.headed ? DESKTOP_CHROME_UA : undefined }); const page = await context.newPage(); return { browser, context, page }; }