From ee437d3f9846c70836a03fccbd998b92b0da9368 Mon Sep 17 00:00:00 2001 From: Naved Khan <94339342+nxved@users.noreply.github.com> Date: Wed, 25 Feb 2026 23:04:36 +0530 Subject: [PATCH 1/6] feat(x): add Playwright connector for profile and recent posts --- README.md | 10 +- registry.json | 17 +- schemas/x.posts.json | 41 +++++ schemas/x.profile.json | 26 +++ x/x-playwright.js | 359 +++++++++++++++++++++++++++++++++++++++++ x/x-playwright.json | 26 +++ 6 files changed, 475 insertions(+), 4 deletions(-) create mode 100644 schemas/x.posts.json create mode 100644 schemas/x.profile.json create mode 100644 x/x-playwright.js create mode 100644 x/x-playwright.json diff --git a/README.md b/README.md index b0f90ef..f151c93 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Playwright-based data connectors for [DataConnect](https://github.com/vana-com/d | Instagram | Meta | playwright | instagram.profile, instagram.posts | | LinkedIn | LinkedIn | playwright | linkedin.profile, .experience, .education, .skills, .languages | | Spotify | Spotify | playwright | spotify.profile, spotify.savedTracks, spotify.playlists | +| X (Twitter) | X | playwright | x.profile, x.posts | ## Repository structure @@ -30,9 +31,12 @@ connectors/ ├── meta/ │ ├── instagram-playwright.js │ └── instagram-playwright.json -└── spotify/ - ├── spotify-playwright.js - └── spotify-playwright.json +├── spotify/ +│ ├── spotify-playwright.js +│ └── spotify-playwright.json +└── x/ + ├── x-playwright.js + └── x-playwright.json ``` Each connector consists of two files inside a `/` directory: diff --git a/registry.json b/registry.json index 005ff70..f4bdd87 100644 --- a/registry.json +++ b/registry.json @@ -1,6 +1,6 @@ { "version": "2.0.0", - "lastUpdated": "2026-02-21T00:00:00Z", + "lastUpdated": "2026-02-25T00:00:00Z", "baseUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main", "connectors": [ { @@ -62,6 +62,21 @@ "script": "sha256:26f5aa145e5ef6d7ce6abd791287d6e0ad2e2d68aa2769a938ff47d40b7e952c", "metadata": "sha256:e68b0367d1b5527338b65f4930ac433cc6caa342a31880dd587074b2ee5a6b3e" } + }, + { + "id": "x-playwright", + "company": "x", + "version": "1.0.0", + "name": "X", + "description": "Exports your X profile and recent posts using Playwright browser automation.", + "files": { + "script": "x/x-playwright.js", + "metadata": "x/x-playwright.json" + }, + "checksums": { + "script": "sha256:aa800f68b14d291607f9aeecac55776e767100b156a303fa3b0225269c4fc7ed", + "metadata": "sha256:26ed3bdf3afae66a8607d65a1238e9fdd612f25816e84fda4b16552b7fb049be" + } } ] } diff --git a/schemas/x.posts.json b/schemas/x.posts.json new file mode 100644 index 0000000..ad378d7 --- /dev/null +++ b/schemas/x.posts.json @@ -0,0 +1,41 @@ +{ + "name": "X Posts", + "version": "1.0.0", + "scope": "x.posts", + "dialect": "json", + "description": "Recent posts from the user's X profile timeline", + "schema": { + "type": "object", + "properties": { + "posts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": "string" }, + "url": { "type": "string" }, + "authorUsername": { "type": "string" }, + "text": { "type": "string" }, + "createdAt": { "type": ["string", "null"] }, + "replyCount": { "type": "number" }, + "repostCount": { "type": "number" }, + "likeCount": { "type": "number" }, + "bookmarkCount": { "type": "number" }, + "viewCount": { "type": "number" }, + "isPinned": { "type": "boolean" }, + "isReply": { "type": "boolean" }, + "lang": { "type": "string" }, + "mediaUrls": { + "type": "array", + "items": { "type": "string" } + } + }, + "required": ["id", "url", "text"], + "additionalProperties": false + } + } + }, + "required": ["posts"], + "additionalProperties": false + } +} diff --git a/schemas/x.profile.json b/schemas/x.profile.json new file mode 100644 index 0000000..203f9f6 --- /dev/null +++ b/schemas/x.profile.json @@ -0,0 +1,26 @@ +{ + "name": "X Profile", + "version": "1.0.0", + "scope": "x.profile", + "dialect": "json", + "description": "X user profile information", + "schema": { + "type": "object", + "properties": { + "username": { "type": "string" }, + "displayName": { "type": "string" }, + "bio": { "type": "string" }, + "location": { "type": "string" }, + "website": { "type": "string" }, + "joinedDate": { "type": "string" }, + "avatarUrl": { "type": "string" }, + "following": { "type": "number" }, + "followers": { "type": "number" }, + "likes": { "type": "number" }, + "isVerified": { "type": "boolean" }, + "profileUrl": { "type": "string" } + }, + "required": ["username", "profileUrl"], + "additionalProperties": false + } +} diff --git a/x/x-playwright.js b/x/x-playwright.js new file mode 100644 index 0000000..1659ea3 --- /dev/null +++ b/x/x-playwright.js @@ -0,0 +1,359 @@ +/** + * X Connector (Playwright) + * + * Exports: + * - Profile + * - Recent posts from the profile timeline + */ + +const state = { + username: null, + profile: null, + posts: [], + isComplete: false, +}; + +const X_USERNAME_REGEX = /^[A-Za-z0-9_]{1,15}$/; + +const isValidXUsername = (value) => + typeof value === 'string' && X_USERNAME_REGEX.test(value); + +const TO_INT_HELPER = ` +const toInt = (raw) => { + const text = String(raw || '').toLowerCase().replace(/,/g, '').replace(/\s+/g, '').trim(); + if (!text) return 0; + + const match = text.match(/^([0-9]+(?:[.][0-9]+)?)([kmb])?$/); + if (match) { + let value = parseFloat(match[1]); + if (Number.isNaN(value)) return 0; + if (match[2] === 'k') value *= 1_000; + if (match[2] === 'm') value *= 1_000_000; + if (match[2] === 'b') value *= 1_000_000_000; + return Math.round(value); + } + + const digits = text.replace(/[^0-9]/g, ''); + if (!digits) return 0; + return parseInt(digits, 10); +}; +`; + +const isLoggedIn = async () => { + try { + return await page.evaluate(` + (() => { + const hasProfileTab = !!document.querySelector('[data-testid="AppTabBar_Profile_Link"]'); + const hasAccountSwitcher = !!document.querySelector('[data-testid="SideNav_AccountSwitcher_Button"]'); + const hasHome = !!document.querySelector('a[href="/home"]'); + const hasLoginInputs = !!document.querySelector('input[autocomplete="username"], input[name="text"]'); + return (hasProfileTab || hasAccountSwitcher || hasHome) && !hasLoginInputs; + })() + `); + } catch { + return false; + } +}; + +const readLoggedInUsername = async () => { + try { + const username = await page.evaluate(` + (() => { + const parseUsername = (href) => { + if (!href) return null; + try { + const url = new URL(href, window.location.origin); + const first = (url.pathname || '/').split('/').filter(Boolean)[0] || ''; + if (!first) return null; + const blocked = new Set(['home', 'explore', 'notifications', 'messages', 'i', 'settings', 'search']); + if (blocked.has(first.toLowerCase())) return null; + return first.replace(/^@/, ''); + } catch { + return null; + } + }; + + const metaUsername = document.querySelector('meta[name="session-user-screen_name"]')?.getAttribute('content') || ''; + if (metaUsername) return metaUsername; + + const profileLink = document.querySelector('[data-testid="AppTabBar_Profile_Link"][href]'); + const fromProfileTab = parseUsername(profileLink?.getAttribute('href')); + if (fromProfileTab) return fromProfileTab; + + const statusLink = document.querySelector('a[href*="/status/"]'); + const fromStatus = parseUsername(statusLink?.getAttribute('href')); + if (fromStatus) return fromStatus; + + return null; + })() + `); + return isValidXUsername(username) ? username : null; + } catch { + return null; + } +}; + +const extractProfile = async (username) => { + if (!isValidXUsername(username)) return null; + + await page.goto(`https://x.com/${username}`); + await page.sleep(3000); + + try { + return await page.evaluate(` + (() => { + ${TO_INT_HELPER} + + const primary = document.querySelector('main [data-testid="primaryColumn"]') || document; + const userNameNode = primary.querySelector('[data-testid="UserName"]'); + const userNameSpans = Array.from(userNameNode?.querySelectorAll('span') || []); + const usernameText = userNameSpans.find((span) => (span.textContent || '').trim().startsWith('@'))?.textContent?.trim() || ''; + const displayName = userNameSpans.find((span) => { + const text = (span.textContent || '').trim(); + return text && !text.startsWith('@'); + })?.textContent?.trim() || ''; + + const statLinks = Array.from(primary.querySelectorAll('a[href]')); + const findStatText = (needle) => { + const link = statLinks.find((a) => (a.getAttribute('href') || '').includes(needle)); + return (link?.textContent || '').trim(); + }; + + const bio = (primary.querySelector('[data-testid="UserDescription"]')?.textContent || '').trim(); + const location = (primary.querySelector('[data-testid="UserLocation"]')?.textContent || '').trim(); + const joinedDate = (primary.querySelector('[data-testid="UserJoinDate"]')?.textContent || '').trim(); + const website = primary.querySelector('[data-testid="UserUrl"] a[href]')?.getAttribute('href') || ''; + const avatarUrl = primary.querySelector('img[src*="profile_images"]')?.getAttribute('src') || ''; + const isVerified = !!primary.querySelector('[data-testid="icon-verified"]'); + + return { + username: usernameText.replace(/^@/, '') || ${JSON.stringify(username)}, + displayName, + bio, + location, + website, + joinedDate, + avatarUrl, + following: toInt(findStatText('/following')), + followers: toInt(findStatText('/followers')), + likes: toInt(findStatText('/likes')), + isVerified, + profileUrl: window.location.href.split('?')[0] + }; + })() + `); + } catch { + return null; + } +}; + +const extractVisiblePosts = async () => { + try { + const result = await page.evaluate(` + (() => { + ${TO_INT_HELPER} + + const parseStatusLink = (href) => { + if (!href) return null; + try { + const url = new URL(href, window.location.origin); + const match = url.pathname.match(/^\/([^/]+)\/status\/(\d+)/); + if (!match) return null; + return { username: match[1], id: match[2] }; + } catch { + return null; + } + }; + + const metricValue = (article, testId) => { + const node = article.querySelector('[data-testid="' + testId + '"]'); + return toInt((node?.textContent || '').trim()); + }; + + const articles = Array.from(document.querySelectorAll('article[data-testid="tweet"]')); + const posts = []; + + for (const article of articles) { + const statusAnchors = Array.from(article.querySelectorAll('a[href*="/status/"]')); + let parsed = null; + let statusUrl = null; + for (const anchor of statusAnchors) { + parsed = parseStatusLink(anchor.getAttribute('href') || ''); + if (parsed) { + statusUrl = 'https://x.com/' + parsed.username + '/status/' + parsed.id; + break; + } + } + if (!parsed || !statusUrl) continue; + + const textNode = article.querySelector('[data-testid="tweetText"]'); + const text = (textNode?.textContent || '').trim(); + const createdAt = article.querySelector('time')?.getAttribute('datetime') || null; + const lang = textNode?.getAttribute('lang') || ''; + const socialContext = (article.querySelector('[data-testid="socialContext"]')?.textContent || '').trim(); + const isPinned = /pinned/i.test(socialContext) || /pinned/i.test(article.textContent || ''); + const isReply = /replying to/i.test(article.textContent || ''); + + const mediaUrls = Array.from(article.querySelectorAll('img[src]')) + .map((img) => img.getAttribute('src') || '') + .filter((src) => src.includes('pbs.twimg.com/media/')) + .filter((src, idx, arr) => src && arr.indexOf(src) === idx); + + posts.push({ + id: parsed.id, + url: statusUrl, + authorUsername: parsed.username, + text, + createdAt, + replyCount: metricValue(article, 'reply'), + repostCount: metricValue(article, 'retweet'), + likeCount: metricValue(article, 'like'), + bookmarkCount: metricValue(article, 'bookmark'), + viewCount: metricValue(article, 'viewCount'), + isPinned, + isReply, + lang, + mediaUrls, + }); + } + + return posts; + })() + `); + + return Array.isArray(result) ? result : []; + } catch { + return []; + } +}; + +const extractPosts = async (username) => { + if (!isValidXUsername(username)) return []; + + await page.goto(`https://x.com/${username}`); + await page.sleep(3000); + + const all = []; + const seen = new Set(); + const maxPosts = 120; + const maxScrolls = 12; + let stagnantScrolls = 0; + + for (let scrollIndex = 0; scrollIndex < maxScrolls; scrollIndex++) { + const visible = await extractVisiblePosts(); + let added = 0; + + for (const post of visible) { + if (!post?.id || seen.has(post.id)) continue; + seen.add(post.id); + all.push(post); + added += 1; + if (all.length >= maxPosts) break; + } + + await page.setProgress({ + phase: { step: 2, total: 2, label: 'Posts' }, + message: `Captured ${all.length} post${all.length === 1 ? '' : 's'}${all.length < maxPosts ? '...' : ''}`, + count: all.length, + }); + + if (all.length >= maxPosts) break; + + if (added === 0) stagnantScrolls += 1; + else stagnantScrolls = 0; + + if (stagnantScrolls >= 3) break; + + await page.evaluate(`window.scrollBy(0, Math.round(window.innerHeight * 0.9))`); + await page.sleep(1800); + } + + return all; +}; + +(async () => { + await page.setData('status', 'Checking X login status...'); + await page.goto('https://x.com/home'); + await page.sleep(2000); + + let loggedIn = await isLoggedIn(); + if (!loggedIn) { + await page.showBrowser('https://x.com/i/flow/login'); + await page.sleep(2500); + await page.setData('status', 'Please log in to X...'); + + await page.promptUser( + 'Please log in to X. Click "Done" when your home timeline loads.', + async () => await isLoggedIn(), + 2000 + ); + + loggedIn = await isLoggedIn(); + } + + if (!loggedIn) { + await page.setData('error', 'X login could not be confirmed.'); + return; + } + + await page.setData('status', 'Login confirmed. Collecting data in background...'); + await page.goHeadless(); + + await page.setProgress({ + phase: { step: 1, total: 2, label: 'Profile' }, + message: 'Resolving account...', + }); + + const username = await readLoggedInUsername(); + if (!isValidXUsername(username)) { + await page.setData('error', 'Could not resolve a valid X username after login.'); + return; + } + state.username = username; + + await page.setProgress({ + phase: { step: 1, total: 2, label: 'Profile' }, + message: `Fetching @${username} profile...`, + }); + state.profile = await extractProfile(username); + + await page.setProgress({ + phase: { step: 2, total: 2, label: 'Posts' }, + message: 'Fetching recent posts...', + }); + state.posts = await extractPosts(username); + + const result = { + 'x.profile': state.profile || { + username, + displayName: '', + bio: '', + location: '', + website: '', + joinedDate: '', + avatarUrl: '', + following: 0, + followers: 0, + likes: 0, + isVerified: false, + profileUrl: `https://x.com/${username}`, + }, + 'x.posts': { + posts: state.posts, + }, + exportSummary: { + count: state.posts.length, + label: state.posts.length === 1 ? 'post' : 'posts', + details: `${state.posts.length} recent posts`, + }, + timestamp: new Date().toISOString(), + version: '1.0.0-playwright', + platform: 'x', + }; + + state.isComplete = true; + await page.setData('result', result); + await page.setData('status', `Complete! Exported ${state.posts.length} posts for @${username}`); + + return { success: true, data: result }; +})(); diff --git a/x/x-playwright.json b/x/x-playwright.json new file mode 100644 index 0000000..9c830af --- /dev/null +++ b/x/x-playwright.json @@ -0,0 +1,26 @@ +{ + "id": "x-playwright", + "version": "1.0.0", + "name": "X", + "company": "X", + "description": "Exports your X profile and recent posts using Playwright browser automation.", + "connectURL": "https://x.com/i/flow/login", + "connectSelector": "[data-testid='AppTabBar_Profile_Link'], [data-testid='SideNav_AccountSwitcher_Button']", + "exportFrequency": "daily", + "runtime": "playwright", + "scopes": [ + { + "scope": "x.profile", + "label": "Your X profile", + "description": "Profile information including bio and follower/following counts" + }, + { + "scope": "x.posts", + "label": "Your recent posts", + "description": "Recent posts visible on your profile timeline with text, metrics, and media URLs" + } + ], + "vectorize_config": { + "documents": "text" + } +} From fcc2dbe8cd6d567be7905b0884b06463c2377f1c Mon Sep 17 00:00:00 2001 From: Naved Khan <94339342+nxved@users.noreply.github.com> Date: Wed, 25 Feb 2026 23:21:18 +0530 Subject: [PATCH 2/6] fix(x): harden login detection and username resolution --- registry.json | 6 +++--- x/x-playwright.js | 27 +++++++++++++++++++++------ x/x-playwright.json | 2 +- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/registry.json b/registry.json index f4bdd87..081969c 100644 --- a/registry.json +++ b/registry.json @@ -66,7 +66,7 @@ { "id": "x-playwright", "company": "x", - "version": "1.0.0", + "version": "1.0.1", "name": "X", "description": "Exports your X profile and recent posts using Playwright browser automation.", "files": { @@ -74,8 +74,8 @@ "metadata": "x/x-playwright.json" }, "checksums": { - "script": "sha256:aa800f68b14d291607f9aeecac55776e767100b156a303fa3b0225269c4fc7ed", - "metadata": "sha256:26ed3bdf3afae66a8607d65a1238e9fdd612f25816e84fda4b16552b7fb049be" + "script": "sha256:872c419608f6a77a3568e1a498c22deea7bc4f6b6e2f1bf9db4907ef73df2efe", + "metadata": "sha256:b5c807a93f139fc5784149bfa2a01323ec4db433836cb43e411d365983b80fb0" } } ] diff --git a/x/x-playwright.js b/x/x-playwright.js index 1659ea3..72c9e2c 100644 --- a/x/x-playwright.js +++ b/x/x-playwright.js @@ -43,11 +43,13 @@ const isLoggedIn = async () => { try { return await page.evaluate(` (() => { + const hasLoggedInNav = !!document.querySelector( + '[data-testid="AppTabBar_Home_Link"], [data-testid="AppTabBar_Profile_Link"], [data-testid="SideNav_NewTweet_Button"]' + ); const hasProfileTab = !!document.querySelector('[data-testid="AppTabBar_Profile_Link"]'); const hasAccountSwitcher = !!document.querySelector('[data-testid="SideNav_AccountSwitcher_Button"]'); const hasHome = !!document.querySelector('a[href="/home"]'); - const hasLoginInputs = !!document.querySelector('input[autocomplete="username"], input[name="text"]'); - return (hasProfileTab || hasAccountSwitcher || hasHome) && !hasLoginInputs; + return hasLoggedInNav || hasProfileTab || hasAccountSwitcher || hasHome; })() `); } catch { @@ -80,6 +82,11 @@ const readLoggedInUsername = async () => { const fromProfileTab = parseUsername(profileLink?.getAttribute('href')); if (fromProfileTab) return fromProfileTab; + const accountSwitcherText = + document.querySelector('[data-testid="SideNav_AccountSwitcher_Button"]')?.textContent || ''; + const fromAccountSwitcher = (accountSwitcherText.match(/@([A-Za-z0-9_]{1,15})/) || [])[1] || ''; + if (fromAccountSwitcher) return fromAccountSwitcher; + const statusLink = document.querySelector('a[href*="/status/"]'); const fromStatus = parseUsername(statusLink?.getAttribute('href')); if (fromStatus) return fromStatus; @@ -288,6 +295,7 @@ const extractPosts = async (username) => { 2000 ); + await page.sleep(2000); loggedIn = await isLoggedIn(); } @@ -296,21 +304,28 @@ const extractPosts = async (username) => { return; } - await page.setData('status', 'Login confirmed. Collecting data in background...'); - await page.goHeadless(); + await page.setData('status', 'Login confirmed. Resolving account...'); + + let username = await readLoggedInUsername(); + if (!isValidXUsername(username)) { + await page.goto('https://x.com/home'); + await page.sleep(2500); + username = await readLoggedInUsername(); + } await page.setProgress({ phase: { step: 1, total: 2, label: 'Profile' }, message: 'Resolving account...', }); - const username = await readLoggedInUsername(); if (!isValidXUsername(username)) { await page.setData('error', 'Could not resolve a valid X username after login.'); return; } state.username = username; + await page.goHeadless(); + await page.setProgress({ phase: { step: 1, total: 2, label: 'Profile' }, message: `Fetching @${username} profile...`, @@ -347,7 +362,7 @@ const extractPosts = async (username) => { details: `${state.posts.length} recent posts`, }, timestamp: new Date().toISOString(), - version: '1.0.0-playwright', + version: '1.0.1-playwright', platform: 'x', }; diff --git a/x/x-playwright.json b/x/x-playwright.json index 9c830af..0b2e299 100644 --- a/x/x-playwright.json +++ b/x/x-playwright.json @@ -1,6 +1,6 @@ { "id": "x-playwright", - "version": "1.0.0", + "version": "1.0.1", "name": "X", "company": "X", "description": "Exports your X profile and recent posts using Playwright browser automation.", From 51ffbc6f155d94295e7768d54166448af282ee97 Mon Sep 17 00:00:00 2001 From: Naved Khan <94339342+nxved@users.noreply.github.com> Date: Wed, 25 Feb 2026 23:30:57 +0530 Subject: [PATCH 3/6] fix(x): improve recent post detection on profile timeline --- registry.json | 6 ++-- x/x-playwright.js | 69 +++++++++++++++++++++++++++++++++------------ x/x-playwright.json | 2 +- 3 files changed, 55 insertions(+), 22 deletions(-) diff --git a/registry.json b/registry.json index 081969c..8784d7e 100644 --- a/registry.json +++ b/registry.json @@ -66,7 +66,7 @@ { "id": "x-playwright", "company": "x", - "version": "1.0.1", + "version": "1.0.2", "name": "X", "description": "Exports your X profile and recent posts using Playwright browser automation.", "files": { @@ -74,8 +74,8 @@ "metadata": "x/x-playwright.json" }, "checksums": { - "script": "sha256:872c419608f6a77a3568e1a498c22deea7bc4f6b6e2f1bf9db4907ef73df2efe", - "metadata": "sha256:b5c807a93f139fc5784149bfa2a01323ec4db433836cb43e411d365983b80fb0" + "script": "sha256:1275453662190b51ad78a825ecc573917d131fdc8efa4aa45b1845a2be56096a", + "metadata": "sha256:eeb4e317c07133649748cf0d87d66e24e78a752f52a604723989c143cfe04723" } } ] diff --git a/x/x-playwright.js b/x/x-playwright.js index 72c9e2c..2c155f1 100644 --- a/x/x-playwright.js +++ b/x/x-playwright.js @@ -164,20 +164,36 @@ const extractVisiblePosts = async () => { if (!href) return null; try { const url = new URL(href, window.location.origin); - const match = url.pathname.match(/^\/([^/]+)\/status\/(\d+)/); - if (!match) return null; - return { username: match[1], id: match[2] }; + const pathname = url.pathname || ''; + let match = pathname.match(/^\/([^/]+)\/status\/(\d+)/); + if (match) return { username: match[1], id: match[2] }; + + match = pathname.match(/^\/i\/web\/status\/(\d+)/); + if (match) return { username: null, id: match[1] }; + + return null; } catch { return null; } }; - const metricValue = (article, testId) => { - const node = article.querySelector('[data-testid="' + testId + '"]'); - return toInt((node?.textContent || '').trim()); + const metricValue = (article, testIds) => { + for (const testId of testIds) { + const node = article.querySelector('[data-testid="' + testId + '"]'); + const value = toInt((node?.textContent || '').trim()); + if (value > 0) return value; + } + return 0; }; - const articles = Array.from(document.querySelectorAll('article[data-testid="tweet"]')); + const articleCandidates = Array.from( + document.querySelectorAll( + 'article[data-testid="tweet"], main article, [data-testid="primaryColumn"] article' + ) + ); + const articles = articleCandidates.filter((article, index, list) => { + return article && list.indexOf(article) === index; + }); const posts = []; for (const article of articles) { @@ -185,11 +201,27 @@ const extractVisiblePosts = async () => { let parsed = null; let statusUrl = null; for (const anchor of statusAnchors) { - parsed = parseStatusLink(anchor.getAttribute('href') || ''); - if (parsed) { + const nextParsed = parseStatusLink(anchor.getAttribute('href') || ''); + if (!nextParsed) continue; + + const fallbackAuthor = (Array.from(article.querySelectorAll('a[href^="/"]')) + .map((a) => (a.getAttribute('href') || '').match(/^\/([A-Za-z0-9_]{1,15})$/)) + .find(Boolean) || [])[1] || null; + + parsed = { + id: nextParsed.id, + username: nextParsed.username || fallbackAuthor, + }; + + if (parsed.id && parsed.username) { statusUrl = 'https://x.com/' + parsed.username + '/status/' + parsed.id; break; } + + if (parsed.id) { + statusUrl = 'https://x.com/i/web/status/' + parsed.id; + break; + } } if (!parsed || !statusUrl) continue; @@ -209,14 +241,14 @@ const extractVisiblePosts = async () => { posts.push({ id: parsed.id, url: statusUrl, - authorUsername: parsed.username, + authorUsername: parsed.username || '', text, createdAt, - replyCount: metricValue(article, 'reply'), - repostCount: metricValue(article, 'retweet'), - likeCount: metricValue(article, 'like'), - bookmarkCount: metricValue(article, 'bookmark'), - viewCount: metricValue(article, 'viewCount'), + replyCount: metricValue(article, ['reply']), + repostCount: metricValue(article, ['retweet', 'unretweet']), + likeCount: metricValue(article, ['like', 'unlike']), + bookmarkCount: metricValue(article, ['bookmark', 'removeBookmark']), + viewCount: metricValue(article, ['viewCount']), isPinned, isReply, lang, @@ -238,7 +270,8 @@ const extractPosts = async (username) => { if (!isValidXUsername(username)) return []; await page.goto(`https://x.com/${username}`); - await page.sleep(3000); + await page.sleep(4500); + await page.evaluate(`window.scrollTo(0, 0)`); const all = []; const seen = new Set(); @@ -272,7 +305,7 @@ const extractPosts = async (username) => { if (stagnantScrolls >= 3) break; await page.evaluate(`window.scrollBy(0, Math.round(window.innerHeight * 0.9))`); - await page.sleep(1800); + await page.sleep(2000); } return all; @@ -362,7 +395,7 @@ const extractPosts = async (username) => { details: `${state.posts.length} recent posts`, }, timestamp: new Date().toISOString(), - version: '1.0.1-playwright', + version: '1.0.2-playwright', platform: 'x', }; diff --git a/x/x-playwright.json b/x/x-playwright.json index 0b2e299..1370408 100644 --- a/x/x-playwright.json +++ b/x/x-playwright.json @@ -1,6 +1,6 @@ { "id": "x-playwright", - "version": "1.0.1", + "version": "1.0.2", "name": "X", "company": "X", "description": "Exports your X profile and recent posts using Playwright browser automation.", From 41e4b3aa01f51321fd782e468ae8b96b88bc4d48 Mon Sep 17 00:00:00 2001 From: Naved Khan <94339342+nxved@users.noreply.github.com> Date: Mon, 2 Mar 2026 15:28:18 +0530 Subject: [PATCH 4/6] fix(x): harden post extraction and add headless fallback retry --- registry.json | 6 +- x/x-playwright.js | 274 ++++++++++++++++++++++++++++---------------- x/x-playwright.json | 2 +- 3 files changed, 179 insertions(+), 103 deletions(-) diff --git a/registry.json b/registry.json index 8784d7e..dd1f29f 100644 --- a/registry.json +++ b/registry.json @@ -66,7 +66,7 @@ { "id": "x-playwright", "company": "x", - "version": "1.0.2", + "version": "1.0.6", "name": "X", "description": "Exports your X profile and recent posts using Playwright browser automation.", "files": { @@ -74,8 +74,8 @@ "metadata": "x/x-playwright.json" }, "checksums": { - "script": "sha256:1275453662190b51ad78a825ecc573917d131fdc8efa4aa45b1845a2be56096a", - "metadata": "sha256:eeb4e317c07133649748cf0d87d66e24e78a752f52a604723989c143cfe04723" + "script": "sha256:1035039436b8260d28b3ead28b0c72068877359d30a61a3b804110aa0a959678", + "metadata": "sha256:8507aa1be1071e8664d5368b0cbc664762533e0da1de714ddb93915ef3ac4c0a" } } ] diff --git a/x/x-playwright.js b/x/x-playwright.js index 2c155f1..048a218 100644 --- a/x/x-playwright.js +++ b/x/x-playwright.js @@ -158,111 +158,163 @@ const extractVisiblePosts = async () => { try { const result = await page.evaluate(` (() => { - ${TO_INT_HELPER} - - const parseStatusLink = (href) => { - if (!href) return null; - try { - const url = new URL(href, window.location.origin); - const pathname = url.pathname || ''; - let match = pathname.match(/^\/([^/]+)\/status\/(\d+)/); - if (match) return { username: match[1], id: match[2] }; - - match = pathname.match(/^\/i\/web\/status\/(\d+)/); - if (match) return { username: null, id: match[1] }; - - return null; - } catch { - return null; - } - }; + try { + ${TO_INT_HELPER} + + const parseStatusLink = (href) => { + if (!href) return null; + try { + const url = new URL(href, window.location.origin); + const pathname = url.pathname || ''; + let match = pathname.match(/^\\/([^/]+)\\/status\\/(\\d+)/); + if (match) return { username: match[1], id: match[2] }; + + match = pathname.match(/^\\/i\\/web\\/status\\/(\\d+)/); + if (match) return { username: null, id: match[1] }; + + return null; + } catch { + return null; + } + }; - const metricValue = (article, testIds) => { - for (const testId of testIds) { - const node = article.querySelector('[data-testid="' + testId + '"]'); - const value = toInt((node?.textContent || '').trim()); - if (value > 0) return value; + const metricValue = (article, testIds) => { + for (const testId of testIds) { + const node = article.querySelector('[data-testid="' + testId + '"]'); + const value = toInt((node?.textContent || '').trim()); + if (value > 0) return value; + } + return 0; + }; + + const containerSet = new Set(); + const pushContainer = (node) => { + if (!node || !(node instanceof Element)) return; + containerSet.add(node); + }; + + Array.from( + document.querySelectorAll( + 'article[data-testid="tweet"], main article, [data-testid="primaryColumn"] article, [data-testid="cellInnerDiv"]' + ) + ).forEach(pushContainer); + + const statusLinks = Array.from(document.querySelectorAll('a[href*="/status/"]')); + for (const link of statusLinks) { + let node = link; + let depth = 0; + while (node && depth < 10) { + if ( + node.matches?.('article, [data-testid="cellInnerDiv"], [data-testid="tweet"]') || + node.querySelector?.('time') + ) { + pushContainer(node); + break; + } + node = node.parentElement; + depth += 1; + } } - return 0; - }; - const articleCandidates = Array.from( - document.querySelectorAll( - 'article[data-testid="tweet"], main article, [data-testid="primaryColumn"] article' - ) - ); - const articles = articleCandidates.filter((article, index, list) => { - return article && list.indexOf(article) === index; - }); - const posts = []; - - for (const article of articles) { - const statusAnchors = Array.from(article.querySelectorAll('a[href*="/status/"]')); - let parsed = null; - let statusUrl = null; - for (const anchor of statusAnchors) { - const nextParsed = parseStatusLink(anchor.getAttribute('href') || ''); - if (!nextParsed) continue; - - const fallbackAuthor = (Array.from(article.querySelectorAll('a[href^="/"]')) - .map((a) => (a.getAttribute('href') || '').match(/^\/([A-Za-z0-9_]{1,15})$/)) - .find(Boolean) || [])[1] || null; - - parsed = { - id: nextParsed.id, - username: nextParsed.username || fallbackAuthor, - }; - - if (parsed.id && parsed.username) { - statusUrl = 'https://x.com/' + parsed.username + '/status/' + parsed.id; - break; + const articles = Array.from(containerSet).filter((container) => { + if (!container || !(container instanceof Element)) return false; + const text = (container.textContent || '').toLowerCase(); + const hasStatus = !!container.querySelector('a[href*="/status/"]'); + const hasTweetText = !!container.querySelector('[data-testid="tweetText"]'); + const hasTime = !!container.querySelector('time'); + if (!(hasStatus || hasTweetText || hasTime)) return false; + if (/who to follow|relevant people|trending/.test(text) && !hasTime) return false; + return true; + }); + const posts = []; + + for (const article of articles) { + const statusAnchors = []; + const timeAnchor = article.querySelector('time')?.closest('a[href]'); + if (timeAnchor) statusAnchors.push(timeAnchor); + for (const anchor of Array.from(article.querySelectorAll('a[href*="/status/"]'))) { + if (!statusAnchors.includes(anchor)) statusAnchors.push(anchor); } - - if (parsed.id) { - statusUrl = 'https://x.com/i/web/status/' + parsed.id; - break; + let parsed = null; + let statusUrl = null; + for (const anchor of statusAnchors) { + const nextParsed = parseStatusLink(anchor.getAttribute('href') || ''); + if (!nextParsed) continue; + + const fallbackAuthor = (Array.from(article.querySelectorAll('a[href^="/"]')) + .map((a) => (a.getAttribute('href') || '').match(/^\\/([A-Za-z0-9_]{1,15})$/)) + .find(Boolean) || [])[1] || null; + + parsed = { + id: nextParsed.id, + username: nextParsed.username || fallbackAuthor, + }; + + if (parsed.id && parsed.username) { + statusUrl = 'https://x.com/' + parsed.username + '/status/' + parsed.id; + break; + } + + if (parsed.id) { + statusUrl = 'https://x.com/i/web/status/' + parsed.id; + break; + } + } + if (!parsed || !statusUrl) { + continue; } + + const textNode = article.querySelector('[data-testid="tweetText"]'); + const text = (textNode?.textContent || '').trim(); + const createdAt = article.querySelector('time')?.getAttribute('datetime') || null; + const lang = textNode?.getAttribute('lang') || ''; + const socialContext = (article.querySelector('[data-testid="socialContext"]')?.textContent || '').trim(); + const isPinned = /pinned/i.test(socialContext) || /pinned/i.test(article.textContent || ''); + const isReply = /replying to/i.test(article.textContent || ''); + + const mediaUrls = Array.from(article.querySelectorAll('img[src]')) + .map((img) => img.getAttribute('src') || '') + .filter((src) => src.includes('pbs.twimg.com/media/')) + .filter((src, idx, arr) => src && arr.indexOf(src) === idx); + + posts.push({ + id: parsed.id, + url: statusUrl, + authorUsername: parsed.username || '', + text, + createdAt, + replyCount: metricValue(article, ['reply']), + repostCount: metricValue(article, ['retweet', 'unretweet']), + likeCount: metricValue(article, ['like', 'unlike']), + bookmarkCount: metricValue(article, ['bookmark', 'removeBookmark']), + viewCount: metricValue(article, ['viewCount']), + isPinned, + isReply, + lang, + mediaUrls, + }); } - if (!parsed || !statusUrl) continue; - - const textNode = article.querySelector('[data-testid="tweetText"]'); - const text = (textNode?.textContent || '').trim(); - const createdAt = article.querySelector('time')?.getAttribute('datetime') || null; - const lang = textNode?.getAttribute('lang') || ''; - const socialContext = (article.querySelector('[data-testid="socialContext"]')?.textContent || '').trim(); - const isPinned = /pinned/i.test(socialContext) || /pinned/i.test(article.textContent || ''); - const isReply = /replying to/i.test(article.textContent || ''); - - const mediaUrls = Array.from(article.querySelectorAll('img[src]')) - .map((img) => img.getAttribute('src') || '') - .filter((src) => src.includes('pbs.twimg.com/media/')) - .filter((src, idx, arr) => src && arr.indexOf(src) === idx); - - posts.push({ - id: parsed.id, - url: statusUrl, - authorUsername: parsed.username || '', - text, - createdAt, - replyCount: metricValue(article, ['reply']), - repostCount: metricValue(article, ['retweet', 'unretweet']), - likeCount: metricValue(article, ['like', 'unlike']), - bookmarkCount: metricValue(article, ['bookmark', 'removeBookmark']), - viewCount: metricValue(article, ['viewCount']), - isPinned, - isReply, - lang, - mediaUrls, - }); - } - return posts; + return { posts, error: null }; + } catch (err) { + return { + posts: [], + error: String(err && err.message ? err.message : err), + }; + } })() `); - return Array.isArray(result) ? result : []; + if (result && typeof result === 'object') { + return { + posts: Array.isArray(result.posts) ? result.posts : [], + error: result.error || null, + }; + } + + return { posts: [], error: 'Unexpected evaluate result' }; } catch { - return []; + return { posts: [], error: 'extractVisiblePosts wrapper failed' }; } }; @@ -270,17 +322,30 @@ const extractPosts = async (username) => { if (!isValidXUsername(username)) return []; await page.goto(`https://x.com/${username}`); - await page.sleep(4500); + await page.sleep(3000); await page.evaluate(`window.scrollTo(0, 0)`); + // X sometimes delays timeline hydration in headless/responsive layouts. + for (let i = 0; i < 3; i++) { + const visibleStatusLinks = await page.evaluate(` + (() => document.querySelectorAll('main a[href*="/status/"]').length)() + `); + if (visibleStatusLinks > 0) break; + await page.sleep(1500); + await page.evaluate(`window.scrollBy(0, Math.round(window.innerHeight * 0.3))`); + } + const all = []; const seen = new Set(); + let lastVisibleError = null; const maxPosts = 120; const maxScrolls = 12; let stagnantScrolls = 0; for (let scrollIndex = 0; scrollIndex < maxScrolls; scrollIndex++) { - const visible = await extractVisiblePosts(); + const visibleResult = await extractVisiblePosts(); + const visible = visibleResult.posts || []; + lastVisibleError = visibleResult.error || lastVisibleError; let added = 0; for (const post of visible) { @@ -308,6 +373,10 @@ const extractPosts = async (username) => { await page.sleep(2000); } + if (lastVisibleError) { + await page.setData('status', `X post extraction note: ${lastVisibleError}`); + } + return all; }; @@ -319,7 +388,7 @@ const extractPosts = async (username) => { let loggedIn = await isLoggedIn(); if (!loggedIn) { await page.showBrowser('https://x.com/i/flow/login'); - await page.sleep(2500); + await page.sleep(3000); await page.setData('status', 'Please log in to X...'); await page.promptUser( @@ -342,7 +411,7 @@ const extractPosts = async (username) => { let username = await readLoggedInUsername(); if (!isValidXUsername(username)) { await page.goto('https://x.com/home'); - await page.sleep(2500); + await page.sleep(2000); username = await readLoggedInUsername(); } @@ -371,6 +440,13 @@ const extractPosts = async (username) => { }); state.posts = await extractPosts(username); + if (state.posts.length === 0) { + await page.setData('status', 'No posts found in headless mode. Retrying with visible browser...'); + await page.showBrowser(`https://x.com/${username}`); + await page.sleep(3000); + state.posts = await extractPosts(username); + } + const result = { 'x.profile': state.profile || { username, @@ -395,7 +471,7 @@ const extractPosts = async (username) => { details: `${state.posts.length} recent posts`, }, timestamp: new Date().toISOString(), - version: '1.0.2-playwright', + version: '1.0.6-playwright', platform: 'x', }; diff --git a/x/x-playwright.json b/x/x-playwright.json index 1370408..c3d50e9 100644 --- a/x/x-playwright.json +++ b/x/x-playwright.json @@ -1,6 +1,6 @@ { "id": "x-playwright", - "version": "1.0.2", + "version": "1.0.6", "name": "X", "company": "X", "description": "Exports your X profile and recent posts using Playwright browser automation.", From f23cccf60ec45d219aac0cb7f8e18d78914238c7 Mon Sep 17 00:00:00 2001 From: Naved Khan <94339342+nxved@users.noreply.github.com> Date: Mon, 2 Mar 2026 19:32:37 +0530 Subject: [PATCH 5/6] docs: Remove Shopify connector documentation from README. --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index b41e2f1..02a66de 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ Playwright-based data connectors for [DataConnect](https://github.com/vana-com/d | GitHub | GitHub | playwright | github.profile, github.repositories, github.starred | | Instagram | Meta | playwright | instagram.profile, instagram.posts | | LinkedIn | LinkedIn | playwright | linkedin.profile, .experience, .education, .skills, .languages | -| Shop | Shopify | playwright | shop.orders | | Spotify | Spotify | playwright | spotify.profile, spotify.savedTracks, spotify.playlists | | X (Twitter) | X | playwright | x.profile, x.posts | | YouTube | Google | playwright | youtube.profile, youtube.subscriptions, youtube.playlists, youtube.playlistItems, youtube.likes, youtube.watchLater, youtube.history (top 50 recent items) | @@ -44,9 +43,6 @@ connectors/ ├── google/ │ ├── youtube-playwright.js # Connector script │ └── youtube-playwright.json # Metadata -├── shopify/ -│ ├── shop-playwright.js -│ └── shop-playwright.json └── x/ ├── x-playwright.js └── x-playwright.json From 769b4f76f828064e48923bd0c179eb90994aeedd Mon Sep 17 00:00:00 2001 From: Naved Khan <94339342+nxved@users.noreply.github.com> Date: Mon, 2 Mar 2026 19:33:38 +0530 Subject: [PATCH 6/6] docs: Reorder X (Twitter) entry in README table. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 02a66de..da91b99 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,9 @@ Playwright-based data connectors for [DataConnect](https://github.com/vana-com/d | Instagram | Meta | playwright | instagram.profile, instagram.posts | | LinkedIn | LinkedIn | playwright | linkedin.profile, .experience, .education, .skills, .languages | | Spotify | Spotify | playwright | spotify.profile, spotify.savedTracks, spotify.playlists | -| X (Twitter) | X | playwright | x.profile, x.posts | | YouTube | Google | playwright | youtube.profile, youtube.subscriptions, youtube.playlists, youtube.playlistItems, youtube.likes, youtube.watchLater, youtube.history (top 50 recent items) | +| X (Twitter) | X | playwright | x.profile, x.posts | + ## Repository structure