Skip to content

fix: X blocks bundled headless Chromium with 403 #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
RichardBray merged 4 commits into from
Sep 3, 2026
Merged

fix: X blocks bundled headless Chromium with 403 #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e5fc163
fix: X blocks bundled headless Chromium with 403
Sep 3, 2026
fb0ea8f
fix: improve Chrome detection and UA string realism
Sep 3, 2026
43575fb
fix: apply desktop Chrome UA to private/CDP login browser too
Sep 3, 2026
def636c
fix: make UA spoof policy consistent (headless-only in both paths)
Sep 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions bun.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
"author": "",
"license": "MIT",
"dependencies": {
"playwright": "^1.57.0"
"playwright": "^1.60.0"
},
"devDependencies": {}
}
29 changes: 18 additions & 11 deletions src/extractor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
isValidTwitterUrl,
parseTweetUrl,
} from './utils.ts';
import { findChromePath, DESKTOP_CHROME_UA } from './private.ts';

type ExtractCandidate = {
url: string;
Expand All @@ -37,7 +38,7 @@ export class VideoExtractor {
}

async extract(url: string, authenticatedPage?: Page): Promise<ExtractResult> {
console.log(`\ud83c\udfac Extracting video from: ${url}`);
console.log(`🎬 Extracting video from: ${url}`);

if (!isValidTwitterUrl(url)) {
return {
Expand All @@ -56,7 +57,7 @@ export class VideoExtractor {
};
}

console.log(`\ud83d\udcdd Tweet: @${tweetInfo.author} (ID: ${tweetInfo.id})`);
console.log(`📝 Tweet: @${tweetInfo.author} (ID: ${tweetInfo.id})`);

const { chromium } = await import('playwright');

Expand All @@ -79,7 +80,7 @@ export class VideoExtractor {
candidates.add(u);
});

console.log('\ud83c\udf10 Opening tweet in browser...');
console.log('🌐 Opening tweet in browser...');
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: this.timeout });

// Give X a moment to hydrate
Expand All @@ -89,7 +90,7 @@ export class VideoExtractor {

const loginWall = hasLoginWall(pageHtml);
if (loginWall) {
console.log('\u26a0\ufe0f Login wall detected; trying to extract anyway...');
console.log('⚠️ Login wall detected; trying to extract anyway...');
}

// Try to trigger media loading.
Expand All @@ -116,8 +117,8 @@ export class VideoExtractor {
}

const filename = generateFilename(tweetInfo);
console.log(`\u2705 Video extracted: ${videoUrl.url}`);
console.log(`\ud83d\udccb Suggested filename: ${filename}`);
console.log(`✅ Video extracted: ${videoUrl.url}`);
console.log(`📋 Suggested filename: ${filename}`);

return { videoUrl };
} catch (error) {
Expand Down Expand Up @@ -145,10 +146,16 @@ export class VideoExtractor {
launchOptions.executablePath = this.browserExecutablePath;
} else if (this.browserChannel) {
launchOptions.channel = this.browserChannel;
} else {
const chromePath = findChromePath();
if (chromePath) {
// X fingerprints bundled Chromium itself; prefer a real Chrome installation.
launchOptions.executablePath = chromePath;
}
}

const browser = await chromium.launch(launchOptions);
const context = await browser.newContext();
const context = await browser.newContext({ userAgent: !this.headed ? DESKTOP_CHROME_UA : undefined });
const page = await context.newPage();
return { browser, context, page };
}
Expand Down Expand Up @@ -191,23 +198,23 @@ export class VideoExtractor {
const htmlPath = path.join(this.debugArtifactsDir, `${prefix}.html`);
fs.writeFileSync(htmlPath, pageHtml, 'utf-8');
debugInfo.htmlPath = htmlPath;
console.log(`\ud83d\udcc3 HTML saved to: ${htmlPath}`);
console.log(`📄 HTML saved to: ${htmlPath}`);
}

// Save screenshot
try {
const screenshotPath = path.join(this.debugArtifactsDir, `${prefix}.png`);
await page.screenshot({ path: screenshotPath, fullPage: true });
debugInfo.screenshotPath = screenshotPath;
console.log(`\ud83d\udcf7 Screenshot saved to: ${screenshotPath}`);
console.log(`📷 Screenshot saved to: ${screenshotPath}`);
} catch {
// Screenshot failed, continue without it
}

return debugInfo;
} catch (error) {
console.warn(
`\u26a0\ufe0f Failed to save debug artifacts: ${error instanceof Error ? error.message : 'Unknown error'}`
`⚠️ Failed to save debug artifacts: ${error instanceof Error ? error.message : 'Unknown error'}`
);
return undefined;
}
Expand Down Expand Up @@ -269,7 +276,7 @@ export class VideoExtractor {
page: Page;
networkCandidates: string[];
}): Promise<VideoUrl | null> {
console.log('\ud83d\udd0d Looking for video...');
console.log('🔍 Looking for video...');

const perfCandidates = await this.getPerformanceCandidates(page);
const domCandidates = await this.getDomCandidates(page);
Expand Down
26 changes: 24 additions & 2 deletions src/private.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ import type { BrowserContext, Page } from 'playwright';

const DEFAULT_PROFILE_DIR = path.join(os.homedir(), '.x-dl-chrome-profile');

// Headless Chrome's default UA includes "HeadlessChrome", which X blocks with a 403.
export const DESKTOP_CHROME_UA =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36';

const CHROME_PATHS_MACOS = [
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
];
Expand All @@ -17,9 +21,24 @@ const CHROME_PATHS_LINUX = [
'/snap/bin/chromium',
];

const CHROME_PATHS_WINDOWS = [
path.join(os.homedir(), 'AppData/Local/Google/Chrome/Application/chrome.exe'),
'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
];

export function findChromePath(): string | null {
const platform = os.platform();
const candidates = platform === 'darwin' ? CHROME_PATHS_MACOS : CHROME_PATHS_LINUX;
let candidates: string[] = [];

if (platform === 'darwin') {
candidates = CHROME_PATHS_MACOS;
} else if (platform === 'linux') {
candidates = CHROME_PATHS_LINUX;
} else if (platform === 'win32') {
candidates = CHROME_PATHS_WINDOWS;
}

for (const p of candidates) {
if (fs.existsSync(p)) return p;
}
Expand All @@ -41,9 +60,12 @@ export async function launchPrivateBrowser(options?: {
}): Promise<PrivateConnection> {
const { chromium } = await import('playwright');

const headless = !(options?.headed);

const context = await chromium.launchPersistentContext(DEFAULT_PROFILE_DIR, {
channel: 'chrome',
headless: !(options?.headed),
headless,
userAgent: headless ? DESKTOP_CHROME_UA : undefined,
args: [
'--disable-blink-features=AutomationControlled',
],
Expand Down
Loading