diff --git a/.gitignore b/.gitignore index 1b739a1..d8814f3 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ test-results/ # Docs docs/ +.worktrees/ diff --git a/README.md b/README.md index 869d746..5ea01ad 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ Extract videos from X (formerly Twitter) tweets. - ✅ Extract videos from public X/Twitter tweets - ✅ Supports multiple formats (mp4, webm, gif, etc.) - ✅ Automatic format selection (highest quality) -- ✅ Download videos directly or just get of URL +- ✅ Download videos directly or just get the URL +- ✅ Clip videos to a specific time range (`--from` and `--to`) - ⚠️ Downloading videos from private tweets (experimental alpha features) - ❌ Windows support @@ -59,6 +60,11 @@ x-dl https://x.com/user/status/123456 - mp4/webm/gif files: direct download - HLS (m3u8) playlists: downloads via ffmpeg to produce mp4 - If direct download fails with 401/403 auth errors and `--profile` is used, automatically retries using authenticated Playwright requests +- **Clipping:** + - `--from` and `--to` (MM:SS format) trim videos to a specific time range + - HLS streams are clipped during download with ffmpeg re-encoding + - MP4 streams download full video, then clip locally + - Clipped files get a `_clip` suffix in the filename - **Auth:** with `--profile`, Playwright reuses cookies/session from a persistent profile directory - **ffmpeg:** checked at runtime and auto-installed when possible @@ -158,6 +164,8 @@ x-dl install --with-deps | `--headed` | Show browser window for debugging | | `--profile [dir]` | Use a persistent browser profile for authenticated extraction (default: `~/.x-dl-profile`) | | `--login` | Open X in a persistent profile and wait for you to log in | +| `--from ` | Clip start time in minutes and seconds (e.g. `00:30`) | +| `--to ` | Clip end time in minutes and seconds (e.g. `01:30`) | | `--help, -h` | Show help message | **Note:** The `-o` option accepts any file extension. If you specify a path with an extension (e.g., `video.mp4`, `video.webm`), that format will be used. Otherwise, the format is auto-detected from the extracted video. @@ -198,6 +206,17 @@ x-dl --profile ~/.x-dl-profile https://x.com/user/status/123456 x-dl --timeout 60 https://x.com/user/status/123456 ``` +**Clip a video to a specific time range:** +```bash +# Download only the 30s–90s portion of a video +x-dl --from 00:30 --to 01:30 https://x.com/user/status/123456 + +# Download from 1 minute to the end +x-dl --from 01:00 https://x.com/user/status/123456 +``` + +Clipped files are saved with a `_clip` suffix, e.g. `username_123456_clip.mp4`. Both `--from` and `--to` are optional — omitting `--from` starts from the beginning, omitting `--to` runs to the end. + ## Output When extracting a video, the tool will: @@ -239,8 +258,36 @@ When extracting a video, the tool will: ✅ Video saved to: ~/Downloads/Remotion_2013626968386765291.mp4 ``` +### Example Output with Clipping + +``` +🎬 x-dl - X/Twitter Video Extractor + +🔍 Checking for Playwright (Chromium)... +✅ Playwright Chromium is ready +🔍 Checking for ffmpeg... +✅ ffmpeg is ready + +🎬 Extracting video from: https://x.com/Remotion/status/2013626968386765291 +📝 Tweet: @Remotion (ID: 2013626968386765291) +🌐 Opening tweet in browser... +✅ Page loaded +🔍 Looking for video... +✅ Video extracted: https://video.twimg.com/ext_tw_video/... +📋 Suggested filename: Remotion_2013626968386765291_clip.mp4 +📥 Downloading HLS video via ffmpeg... +⠋ Downloading HLS... +✅ HLS download completed + +✅ Video saved to: ~/Downloads/Remotion_2013626968386765291_clip.mp4 +``` + ## Limitations +- **Public tweets only**: Private or protected tweets cannot be extracted +- **Clipping requires ffmpeg**: `--from` and `--to` require ffmpeg for processing +- **Clipping time format**: Times must be in MM:SS format (e.g., `00:30`, not `0:30` or `30`) + - **Public tweets only**: Private or protected tweets cannot be extracted - **Time-limited URLs**: Video URLs may expire after some time - **Rate limiting**: X may rate-limit excessive requests diff --git a/package.json b/package.json index 0607f3d..5d45635 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "x-dl", - "version": "0.4.3", + "version": "0.4.4", "description": "Extract videos from X/Twitter tweets", "main": "./src/index.ts", "type": "module", @@ -24,7 +24,12 @@ "release:major": "bun version major && git push && git push --tags", "version:tags": "git tag -l --sort=-version:refname" }, - "keywords": ["twitter", "x", "video", "extractor"], + "keywords": [ + "twitter", + "x", + "video", + "extractor" + ], "author": "", "license": "MIT", "dependencies": { diff --git a/src/ffmpeg.ts b/src/ffmpeg.ts index bdd332d..9b0d6ff 100644 --- a/src/ffmpeg.ts +++ b/src/ffmpeg.ts @@ -4,6 +4,8 @@ export interface DownloadHlsOptions { playlistUrl: string; outputPath: string; timeout?: number; + clipFromSecs?: number; + clipDurationSecs?: number; } export interface FfmpegCapabilities { @@ -19,7 +21,7 @@ export interface FfmpegCapabilityCheckResult extends FfmpegCapabilities { } export async function downloadHlsWithFfmpeg(options: DownloadHlsOptions): Promise { - const { playlistUrl, outputPath, timeout } = options; + const { playlistUrl, outputPath, timeout, clipFromSecs, clipDurationSecs } = options; console.log('📥 Downloading HLS video via ffmpeg...'); @@ -31,19 +33,24 @@ export async function downloadHlsWithFfmpeg(options: DownloadHlsOptions): Promis } const timeoutMs = timeout ?? 120000; - const noProgressTimeoutMs = 30000; + const noProgressTimeoutMs = 60000; let lastFileSize = 0; let lastProgressTime = Date.now(); let rejected = false; return new Promise((resolve, reject) => { + const isClipping = clipFromSecs !== undefined || clipDurationSecs !== undefined; const args = [ '-y', '-hide_banner', '-loglevel', 'error', '-i', playlistUrl, - '-c', 'copy', - '-bsf:a', 'aac_adtstoasc', + // Use -ss after -i (slow/accurate seek) — fast seek on HLS produces empty output + ...(clipFromSecs !== undefined ? ['-ss', String(clipFromSecs)] : []), + ...(clipDurationSecs !== undefined ? ['-t', String(clipDurationSecs)] : []), + ...(isClipping + ? ['-c:v', 'libx264', '-c:a', 'aac', '-movflags', '+faststart'] + : ['-c', 'copy', '-bsf:a', 'aac_adtstoasc']), outputPath, ]; @@ -136,6 +143,69 @@ export async function downloadHlsWithFfmpeg(options: DownloadHlsOptions): Promis }); } +export interface ClipLocalFileOptions { + inputPath: string; + outputPath: string; + clipFrom?: string; + clipTo?: string; +} + +export function mmssToSeconds(mmss: string): number { + const [mm, ss] = mmss.split(':').map(Number); + return mm * 60 + ss; +} + +export async function clipLocalFile(options: ClipLocalFileOptions): Promise { + const { inputPath, outputPath, clipFrom, clipTo } = options; + + const startSecs = clipFrom ? mmssToSeconds(clipFrom) : 0; + const duration = clipTo ? mmssToSeconds(clipTo) - startSecs : undefined; + + return new Promise((resolve, reject) => { + const args = [ + '-y', + '-hide_banner', + '-loglevel', 'error', + ...(clipFrom ? ['-ss', String(startSecs)] : []), + '-i', inputPath, + ...(duration !== undefined ? ['-t', String(duration)] : []), + '-c:v', 'libx264', '-c:a', 'aac', '-movflags', '+faststart', + outputPath, + ]; + + const ffmpeg = spawn('ffmpeg', args); + let stderr = ''; + + if (ffmpeg.stderr) { + ffmpeg.stderr.on('data', (data: Buffer) => { stderr += data.toString(); }); + } + + const spinnerChars = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; + let spinnerIndex = 0; + const spinnerInterval = setInterval(() => { + process.stdout.write(`\r${spinnerChars[spinnerIndex]} Clipping...`); + spinnerIndex = (spinnerIndex + 1) % spinnerChars.length; + }, 80); + + ffmpeg.on('close', (code: number | null) => { + clearInterval(spinnerInterval); + if (code === 0) { + process.stdout.write('\r✅ Clip complete\n'); + resolve(); + } else { + process.stdout.write('\r\x1b[K'); + reject(new Error(stderr.trim() || `ffmpeg exited with code ${code ?? 'null (signal)'}`)); + } + }); + + ffmpeg.on('error', (err: Error) => { + clearInterval(spinnerInterval); + process.stdout.write('\r\x1b[K'); + reject(new Error(`Failed to start ffmpeg: ${err.message}`)); + }); + }); +} + export async function checkFfmpegCapabilities(): Promise { const results: FfmpegCapabilityCheckResult = { available: false, @@ -199,6 +269,132 @@ export async function checkFfmpegCapabilities(): Promise { + const { videoUrl, outputPath, clipFrom, clipTo, timeout } = options; + + console.log('📥 Downloading video via ffmpeg...'); + + const fs = await import('node:fs'); + + if (fs.existsSync(outputPath)) { + console.log(`⚠️ File already exists, removing: ${outputPath}`); + fs.unlinkSync(outputPath); + } + + const timeoutMs = timeout ?? 120000; + const noProgressTimeoutMs = 60000; + let lastFileSize = 0; + let lastProgressTime = Date.now(); + let rejected = false; + + const fromSecs = clipFrom ? mmssToSeconds(clipFrom) : 0; + const toSecs = clipTo ? mmssToSeconds(clipTo) : undefined; + const duration = toSecs !== undefined ? toSecs - fromSecs : undefined; + + return new Promise((resolve, reject) => { + const args = [ + '-y', + '-hide_banner', + '-loglevel', 'error', + ...(clipFrom ? ['-ss', String(fromSecs)] : []), + '-i', videoUrl, + ...(duration !== undefined ? ['-t', String(duration)] : []), + '-c:v', 'libx264', '-c:a', 'aac', '-movflags', '+faststart', + outputPath, + ]; + + const ffmpeg = spawn('ffmpeg', args); + + let stderr = ''; + + if (ffmpeg.stderr) { + ffmpeg.stderr.on('data', (data: Buffer) => { + stderr += data.toString(); + }); + } + + const spinnerChars = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; + let spinnerIndex = 0; + + const spinnerInterval = setInterval(() => { + process.stdout.write(`\r${spinnerChars[spinnerIndex]} Downloading...`); + spinnerIndex = (spinnerIndex + 1) % spinnerChars.length; + }, 80); + + const pollInterval = setInterval(() => { + try { + const now = Date.now(); + if (fs.existsSync(outputPath)) { + const currentFileSize = fs.statSync(outputPath).size; + if (currentFileSize > lastFileSize) { + lastFileSize = currentFileSize; + lastProgressTime = now; + } else if (now - lastProgressTime > noProgressTimeoutMs) { + clearInterval(spinnerInterval); + clearInterval(pollInterval); + clearTimeout(timeoutHandle); + rejected = true; + ffmpeg.kill(); + process.stdout.write('\r\x1b[K'); + reject(new Error(`FFMPEG stuck: no progress for ${noProgressTimeoutMs / 1000} seconds`)); + } + } else if (now - lastProgressTime > noProgressTimeoutMs) { + clearInterval(spinnerInterval); + clearInterval(pollInterval); + clearTimeout(timeoutHandle); + rejected = true; + ffmpeg.kill(); + process.stdout.write('\r\x1b[K'); + reject(new Error(`FFMPEG stuck: no progress for ${noProgressTimeoutMs / 1000} seconds`)); + } + } catch (_err) {} + }, 2000); + + const timeoutHandle = setTimeout(() => { + clearInterval(spinnerInterval); + clearInterval(pollInterval); + rejected = true; + ffmpeg.kill(); + process.stdout.write('\r\x1b[K'); + reject(new Error(`FFMPEG download timed out after ${timeoutMs / 1000} seconds`)); + }, timeoutMs); + + ffmpeg.on('close', (code: number | null) => { + clearInterval(spinnerInterval); + clearInterval(pollInterval); + clearTimeout(timeoutHandle); + + if (code === 0 && !rejected) { + process.stdout.write('\r✅ Download completed\n'); + resolve(outputPath); + } else if (!rejected) { + process.stdout.write('\r\x1b[K'); + const error = stderr.trim() || `ffmpeg exited with code ${code ?? 'null (signal)'}`; + reject(new Error(`Failed to download: ${error}`)); + } + }); + + ffmpeg.on('error', (err: Error) => { + clearInterval(spinnerInterval); + clearInterval(pollInterval); + clearTimeout(timeoutHandle); + if (!rejected) { + rejected = true; + process.stdout.write('\r\x1b[K'); + reject(new Error(`Failed to start ffmpeg: ${err.message}`)); + } + }); + }); +} + export function hasRequiredFfmpegCapabilities(capabilities: FfmpegCapabilities): { has: boolean; missing: string[] } { const required = { protocol: 'https', diff --git a/src/index.ts b/src/index.ts index 89642bc..0ca7c86 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ import { VideoExtractor } from './extractor.ts'; import { downloadVideo } from './downloader.ts'; import { ensurePlaywrightReady, runInstall } from './installer.ts'; import { generateFilename, isValidTwitterUrl, parseTweetUrl, formatBytes } from './utils.ts'; -import { downloadHlsWithFfmpeg } from './ffmpeg.ts'; +import { downloadHlsWithFfmpeg, clipLocalFile, mmssToSeconds } from './ffmpeg.ts'; interface CliOptions { url?: string; @@ -21,6 +21,8 @@ interface CliOptions { verifyAuth?: boolean; browserChannel?: 'chrome' | 'chromium' | 'msedge'; browserExecutablePath?: string; + clipFrom?: string; + clipTo?: string; } interface InstallCliOptions { @@ -122,6 +124,30 @@ function parseArgs(args: string[]): CliOptions { i++; } break; + case '--from': + if (!nextArg || nextArg.startsWith('-')) { + console.error('❌ Error: --from requires a time value (e.g., --from 00:30)'); + process.exit(1); + } + if (!/^\d{2}:\d{2}$/.test(nextArg)) { + console.error(`❌ Error: --from must be in MM:SS format (got: ${nextArg})`); + process.exit(1); + } + options.clipFrom = nextArg; + i++; + break; + case '--to': + if (!nextArg || nextArg.startsWith('-')) { + console.error('❌ Error: --to requires a time value (e.g., --to 01:30)'); + process.exit(1); + } + if (!/^\d{2}:\d{2}$/.test(nextArg)) { + console.error(`❌ Error: --to must be in MM:SS format (got: ${nextArg})`); + process.exit(1); + } + options.clipTo = nextArg; + i++; + break; case '--version': case '-v': showVersion(); @@ -168,6 +194,8 @@ OPTIONS: --browser-channel Browser channel: chrome, chromium, or msedge (default: chromium) --browser-executable-path Path to browser executable (optional, overrides channel) --verify-auth Check authentication status (EXPERIMENTAL ALPHA) + --from Clip start time (e.g., 00:30) + --to Clip end time (e.g., 01:30) --version, -v Show version information --help, -h Show this help message @@ -191,6 +219,13 @@ BROWSER EXAMPLES: # Use custom browser executable ${commandName} --browser-executable-path /path/to/browser https://x.com/user/status/123 + +CLIP EXAMPLES: + # Download only 30s–90s of a video + ${commandName} --from 00:30 --to 01:30 https://x.com/user/status/123 + + # Download from 1 minute to end + ${commandName} --from 01:00 https://x.com/user/status/123 `); } @@ -225,7 +260,8 @@ EXAMPLES: } function showVersion(): void { - console.log('0.4.3'); + const pkg = require('../package.json'); + console.log(pkg.version); process.exit(0); } @@ -425,7 +461,21 @@ async function main(): Promise { defaultExtension = result.videoUrl.format; } - const outputPath = getOutputPath(args.url, args, defaultExtension); + const basePath = getOutputPath(args.url, args, defaultExtension); + const isClipping = args.clipFrom || args.clipTo; + + if (args.clipFrom && args.clipTo) { + const fromSecs = mmssToSeconds(args.clipFrom); + const toSecs = mmssToSeconds(args.clipTo); + if (toSecs <= fromSecs) { + console.error('❌ Error: --to must be after --from'); + process.exit(1); + } + } + + const outputPath = isClipping + ? path.join(path.dirname(basePath), `${path.basename(basePath, path.extname(basePath))}_clip${path.extname(basePath)}`) + : basePath; if (result.videoUrl.format === 'm3u8') { const { ensureFfmpegReady } = await import('./installer.ts'); @@ -442,9 +492,15 @@ async function main(): Promise { } try { + const fromSecs = args.clipFrom ? mmssToSeconds(args.clipFrom) : undefined; + const toSecs = args.clipTo ? mmssToSeconds(args.clipTo) : undefined; + const durationSecs = toSecs !== undefined ? toSecs - (fromSecs ?? 0) : undefined; + await downloadHlsWithFfmpeg({ playlistUrl: result.videoUrl.url, outputPath, + clipFromSecs: fromSecs, + clipDurationSecs: durationSecs, }); console.log(`\n✅ Video saved to: ${outputPath}\n`); } catch (error) { @@ -456,6 +512,55 @@ async function main(): Promise { return; } + if (isClipping) { + const { ensureFfmpegReady: ensureFfmpegReadyForClip } = await import('./installer.ts'); + const ffmpegReady = await ensureFfmpegReadyForClip(); + if (!ffmpegReady) { + console.error('\n❌ ffmpeg is required to clip videos.'); + console.error('Please install ffmpeg:'); + console.error(' macOS: brew install ffmpeg'); + console.error(' Linux: sudo apt-get install ffmpeg'); + process.exit(1); + } + + // Download the full video first, then clip locally. + // ffmpeg can't access X/Twitter direct MP4 URLs (auth headers required). + const os = await import('node:os'); + const fs = await import('node:fs'); + const tmpPath = path.join(os.tmpdir(), `x-dl-tmp-${Date.now()}.mp4`); + + try { + await downloadVideo({ + url: result.videoUrl.url, + outputPath: tmpPath, + onProgress: (progress, downloaded, total) => { + process.stdout.write( + `\r⏳ Downloading: ${progress.toFixed(1)}% (${formatBytes(downloaded)}/${formatBytes(total)})` + ); + }, + }); + process.stdout.write('\n'); + + await clipLocalFile({ + inputPath: tmpPath, + outputPath, + clipFrom: args.clipFrom, + clipTo: args.clipTo, + }); + + console.log(`\n✅ Video saved to: ${outputPath}\n`); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stdout.write('\r\x1b[K'); + console.error(`❌ Failed: ${message}\n`); + if (fs.existsSync(tmpPath)) fs.unlinkSync(tmpPath); + process.exit(1); + } finally { + if (fs.existsSync(tmpPath)) fs.unlinkSync(tmpPath); + } + return; + } + try { await downloadVideo({ url: result.videoUrl.url, diff --git a/test/unit/clip-args.test.ts b/test/unit/clip-args.test.ts new file mode 100644 index 0000000..2be36d6 --- /dev/null +++ b/test/unit/clip-args.test.ts @@ -0,0 +1,18 @@ +import { describe, it, expect } from 'bun:test'; + +describe('clip time format validation', () => { + const valid = /^\d{2}:\d{2}$/; + + it('accepts valid MM:SS', () => { + expect(valid.test('00:30')).toBe(true); + expect(valid.test('01:30')).toBe(true); + expect(valid.test('59:59')).toBe(true); + }); + + it('rejects invalid formats', () => { + expect(valid.test('1:30')).toBe(false); + expect(valid.test('90')).toBe(false); + expect(valid.test('00:00:30')).toBe(false); + expect(valid.test('1:3')).toBe(false); + }); +});