From 31be1910b6df076b1e3959da4e26b57f1de430a2 Mon Sep 17 00:00:00 2001
From: Milan Kubicek <5452309+mkubicek@users.noreply.github.com>
Date: Tue, 26 May 2026 22:43:52 +0200
Subject: [PATCH 1/2] fix: skip incomplete article stubs

---
 AGENTS.md                                     |  4 +-
 README.md                                     |  4 +-
 lib/dedup.js                                  |  2 +-
 lib/tweet-parser.js                           | 30 +++++++++---
 .../sanitized/timeline-basic/README.md        |  4 +-
 .../sanitized/timeline-basic/expected.jsonl   |  1 -
 .../sanitized/timeline-basic/manifest.json    |  2 +-
 .../sanitized/user-articles/README.md         | 14 ++----
 .../sanitized/user-articles/expected.jsonl    | 12 -----
 .../sanitized/user-articles/manifest.json     |  4 +-
 .../sanitized/user-highlights/README.md       |  2 +-
 tests/parser-golden.test.mjs                  |  4 +-
 tests/tweet-parser.test.mjs                   | 48 +++++++++++++++++++
 13 files changed, 90 insertions(+), 41 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index 4004ea8..0b103a3 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -119,7 +119,7 @@ The tweet parser (`lib/tweet-parser.js`) has known instruction paths for:
 
 `HomeTimeline`, `HomeLatestTimeline`, `UserTweets`, `UserTweetsAndReplies`, `UserMedia`, `UserLikes`, `TweetDetail`, `SearchTimeline`, `ListLatestTweetsTimeline`, `Bookmarks`, `Likes`, `CommunityTweetsTimeline`, `BookmarkFolderTimeline`
 
-`TweetResultByRestId` is also handled — it returns a single tweet (not a timeline) and is only processed when the tweet contains article data (long-form posts). This avoids duplicating tweets already captured from timeline endpoints.
+`TweetResultByRestId` is also handled — it returns a single tweet (not a timeline). Full article payloads from this endpoint are captured when `content_state.blocks` is present; article stubs without the body are skipped so they do not poison dedup.
 
 Unknown endpoints fall back to a recursive search for `instructions[]` arrays (max depth 5). Non-tweet endpoints are filtered in `background.js` via `IGNORED_ENDPOINTS`.
 
@@ -174,7 +174,7 @@ Each JSONL line contains:
 }
 ```
 
-Notes: `media[].duration_ms` only present for videos. `views` may be `null`. For retweets, `text` contains the full original tweet text (not the truncated `RT @user:` form). For articles, `is_article` and `article` are present — `article.text` is a markdown rendering with inline `![](media/<id>/file)` image refs, `article.blocks` preserves the raw Draft.js structure, and `article.media[]` lists images with CDN URLs and local paths. Article tweets bypass dedup so the enriched version (from `TweetResultByRestId`) replaces the stub captured from timeline endpoints. Top-level `media[]` entries deliberately do NOT carry `local_path` — when image download is enabled, photos land at `media/<tweet_id>/<basename(url)>` by convention. Consumers reconstruct the path; the daemon never round-trips it through the JSONL.
+Notes: `media[].duration_ms` only present for videos. `views` may be `null`. For retweets, `text` contains the full original tweet text (not the truncated `RT @user:` form). For articles, `is_article` and `article` are present — `article.text` is a markdown rendering with inline `![](media/<id>/file)` image refs, `article.blocks` preserves the raw Draft.js structure, and `article.media[]` lists images with CDN URLs and local paths. Article stubs without `content_state.blocks` are skipped rather than saved as incomplete rows, so a later full article payload can still be captured. Top-level `media[]` entries deliberately do NOT carry `local_path` — when image download is enabled, photos land at `media/<tweet_id>/<basename(url)>` by convention. Consumers reconstruct the path; the daemon never round-trips it through the JSONL.
 
 ## Known Issues
 
diff --git a/README.md b/README.md
index 1b4c8b7..21a65db 100644
--- a/README.md
+++ b/README.md
@@ -32,7 +32,7 @@ xTap is a browser extension (Chrome + Firefox) that silently intercepts the Grap
 
 - **Zero footprint** — no additional network requests; captures what your browser already receives
 - **Structured output** — each tweet saved as a clean JSON object with author, metrics, media, and more
-- **Article support** — long-form X articles are captured with full text, inline image references, and Draft.js block structure
+- **Article support** — long-form X articles are captured when X returns the full body, with inline image references and Draft.js block structure
 - **Video download** — download videos from tweets using yt-dlp (or direct MP4 fallback) via the extension popup. Requires the HTTP daemon. **Note:** unlike passive capture, video downloads make additional network requests to X and are not stealth.
 - **Image download** — opt-in toggle in the popup ("Download images automatically") fetches photos from `pbs.twimg.com` to `<output_dir>/media/<tweet_id>/<filename>` as you browse. Daemon-side; rate-limited; logs to `media-manifest.jsonl`. **Note:** also not stealth — adds requests to the Twitter CDN.
 - **Pause / resume** — click the extension icon to toggle capture on the fly
@@ -331,7 +331,7 @@ Output is written to daily files (`tweets-YYYY-MM-DD.jsonl`). Each line is a sel
 }
 ```
 
-For regular tweets, `is_article` and `article` are absent. For articles, `text` contains a markdown-style rendering of the article with inline image references pointing to `media/<tweet_id>/`.
+For regular tweets, `is_article` and `article` are absent. For articles, `text` contains a markdown-style rendering of the article with inline image references pointing to `media/<tweet_id>/`. Article stubs without `content_state.blocks` are skipped rather than saved as incomplete rows.
 
 ### Media file convention
 
diff --git a/lib/dedup.js b/lib/dedup.js
index d0e39a7..63b3152 100644
--- a/lib/dedup.js
+++ b/lib/dedup.js
@@ -11,7 +11,7 @@
  * Returns false → tweet is a duplicate, should be skipped.
  *
  * - Tweets without an id are always new (and not tracked).
- * - Article tweets bypass dedup — they enrich a previously captured stub.
+ * - Article tweets bypass dedup — they may enrich a previously captured row.
  * - When `options.imageBackfill` is set and the tweet has photo media,
  *   a duplicate is also let through once per session (tracked in
  *   `options.imageCheckedIds`) so the daemon can download images that
diff --git a/lib/tweet-parser.js b/lib/tweet-parser.js
index 6fe62ed..44d6e08 100644
--- a/lib/tweet-parser.js
+++ b/lib/tweet-parser.js
@@ -254,6 +254,12 @@ export function normalizeTweet(raw) {
     return null;
   }
 
+  const tweetId = legacy.id_str || raw.rest_id;
+  if (isArticleStub(raw)) {
+    console.log(`[xTap:parser] Skipping article tweet ${tweetId || '(no id)'} without content_state.blocks`);
+    return null;
+  }
+
   const userResult = raw.core?.user_results?.result;
   const userCore = userResult?.core;      // X nests name/screen_name here
   const userLegacy = userResult?.legacy;  // follower_count, verified, etc.
@@ -269,7 +275,6 @@ export function normalizeTweet(raw) {
   const media = extractMedia(legacy);
   const urls = extractUrls(legacy);
 
-  const tweetId = legacy.id_str || raw.rest_id;
   const username = userCore?.screen_name || userLegacy?.screen_name;
 
   const tweet = {
@@ -339,18 +344,17 @@ export function normalizeTweet(raw) {
  * Articles use Draft.js block format in content_state.
  */
 export function extractArticle(raw) {
-  const articleResult = raw.article?.article_results?.result;
+  const articleResult = getArticleResult(raw);
   if (!articleResult) return null;
 
   // Timeline endpoints return article stubs with only title/preview_text.
   // Require content_state to be present — full data comes from TweetResultByRestId.
-  const contentState = articleResult.content_state;
-  if (!contentState?.blocks?.length) return null;
+  if (!hasArticleContent(articleResult)) return null;
 
   const tweetId = raw.legacy?.id_str || raw.rest_id;
   const title = articleResult.title || null;
-  const blocks = contentState.blocks;
-  const entityMap = contentState.entityMap || [];
+  const blocks = articleResult.content_state.blocks;
+  const entityMap = articleResult.content_state.entityMap || [];
 
   // Build mediaId → {url, filename} from media_entities
   const mediaEntities = articleResult.media_entities || [];
@@ -404,6 +408,20 @@ export function extractArticle(raw) {
   return { title, text, blocks, media };
 }
 
+function getArticleResult(raw) {
+  return raw.article?.article_results?.result || null;
+}
+
+function hasArticleContent(articleResult) {
+  return Array.isArray(articleResult?.content_state?.blocks) &&
+    articleResult.content_state.blocks.length > 0;
+}
+
+function isArticleStub(raw) {
+  const articleResult = getArticleResult(raw);
+  return !!articleResult && !hasArticleContent(articleResult);
+}
+
 /**
  * Extract full text, preferring note_tweet (long-form) over legacy.full_text.
  */
diff --git a/tests/fixtures/sanitized/timeline-basic/README.md b/tests/fixtures/sanitized/timeline-basic/README.md
index fdd28a3..2b1dd40 100644
--- a/tests/fixtures/sanitized/timeline-basic/README.md
+++ b/tests/fixtures/sanitized/timeline-basic/README.md
@@ -13,7 +13,7 @@ xTap tweet parsing pipeline.
 - Tweet normalization (author, text, metrics, media, URLs, mentions)
 - Reply chains, quote tweets, retweets
 - Note tweets (long-form posts)
-- Article stubs
+- Article stubs (skipped when full content is unavailable)
 - Card embeds
 - Cursor entries (skipped by parser)
 
@@ -46,7 +46,7 @@ and author references remain internally consistent.
 ## Invariants
 
 - `fixture.json` fed to `extractTweets("HomeTimeline", data)` must produce
-  exactly 39 tweets matching `expected.jsonl`
+  exactly 38 tweets matching `expected.jsonl`
 - All `in_reply_to`, `quoted_tweet_id`, and `conversation_id` values that
   reference tweets in the fixture use the same remapped IDs
 - Author IDs and handles are consistent across all tweets by the same user
diff --git a/tests/fixtures/sanitized/timeline-basic/expected.jsonl b/tests/fixtures/sanitized/timeline-basic/expected.jsonl
index d3de0c8..191e6e7 100644
--- a/tests/fixtures/sanitized/timeline-basic/expected.jsonl
+++ b/tests/fixtures/sanitized/timeline-basic/expected.jsonl
@@ -7,7 +7,6 @@
 {"id":"7843560721270987274","url":"https://x.com/user_177791b9/status/7843560721270987274","created_at":"2026-04-18T03:18:09.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":57993},"text":"Jade jade echo gamma apex ratio apex terra. kite orbit zonal logic bloom pulse jade drift. bloom sigma apex pulse ionic haze mesa lunar. sigma glyph kite trace flux ratio grid haze.","lang":"en","metrics":{"likes":42,"retweets":0,"replies":9,"views":4132,"bookmarks":7,"quotes":0},"media":[],"urls":[],"hashtags":[],"mentions":[{"id":"317476795064","username":"user_b924e844"}],"in_reply_to":"4082130165462299434","quoted_tweet_id":null,"conversation_id":"4082130165462299434","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":false,"source_endpoint":"HomeTimeline","captured_at":"2025-01-01T00:00:00.000Z"}
 {"id":"7883792353538994169","url":"https://x.com/user_dcc6212f/status/7883792353538994169","created_at":"2026-04-17T17:54:16.000Z","author":{"id":"189535729767","username":"user_dcc6212f","display_name":"User 8f90c2","verified":false,"is_blue_verified":true,"follower_count":47071},"text":"Echo kraft glyph helix yield echo mesa kraft. pulse micro micro delta unity micro core flux. orbit delta beta ionic glyph bloom ultra bloom. node quartz logic optic prism valve foxtrot optic.","lang":"en","metrics":{"likes":4197,"retweets":178,"replies":89,"views":953458,"bookmarks":1196,"quotes":13},"media":[{"type":"video","url":"https://video.twimg.com/sanitized/e27f62c92aa1.mp4","alt_text":null,"duration_ms":153878}],"urls":[],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"7883792353538994169","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":false,"source_endpoint":"HomeTimeline","captured_at":"2025-01-01T00:00:00.000Z"}
 {"id":"4525395474206381929","url":"https://x.com/user_4631fa03/status/4525395474206381929","created_at":"2026-04-18T05:20:56.000Z","author":{"id":"370991728694","username":"user_4631fa03","display_name":"User fe8a05","verified":false,"is_blue_verified":true,"follower_count":754198},"text":"Gamma lunar valve kite trace vortex echo quartz. glyph nexus sigma ultra node ionic mesa micro.","lang":"en","metrics":{"likes":154,"retweets":1,"replies":36,"views":20266,"bookmarks":20,"quotes":2},"media":[],"urls":[],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":"2365697391495111868","conversation_id":"4525395474206381929","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":false,"source_endpoint":"HomeTimeline","captured_at":"2025-01-01T00:00:00.000Z"}
-{"id":"5377202632873564524","url":"https://x.com/user_f90ddcb0/status/5377202632873564524","created_at":"2026-04-17T13:45:35.000Z","author":{"id":"137793696974","username":"user_f90ddcb0","display_name":"User cfa689","verified":false,"is_blue_verified":true,"follower_count":32579},"text":"Zonal query alpha unity.","lang":"zxx","metrics":{"likes":131,"retweets":16,"replies":4,"views":35164,"bookmarks":353,"quotes":3},"media":[],"urls":[{"display":"example.com/san_2748c965a0","expanded":"https://example.com/sanitized/3214dad364","shortened":"https://t.co/SAN_a11851e8"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"5377202632873564524","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":false,"source_endpoint":"HomeTimeline","captured_at":"2025-01-01T00:00:00.000Z"}
 {"id":"1048209185645476420","url":"https://x.com/user_87e0a70d/status/1048209185645476420","created_at":"2026-04-18T03:52:15.000Z","author":{"id":"103858680551","username":"user_87e0a70d","display_name":"User e83e9a","verified":false,"is_blue_verified":true,"follower_count":22173},"text":"Vortex optic quartz terra glyph sigma mesa joule. kraft lunar gamma ultra mesa unity widen wave. beta relay echo grid kraft haze echo ultra. sigma vortex foxtrot yield micro delta quartz delta.","lang":"en","metrics":{"likes":264,"retweets":34,"replies":43,"views":14976,"bookmarks":107,"quotes":13},"media":[{"type":"photo","url":"https://pbs.twimg.com/sanitized/e3c68b576ec5.jpg:orig","alt_text":null}],"urls":[],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"1048209185645476420","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":false,"source_endpoint":"HomeTimeline","captured_at":"2025-01-01T00:00:00.000Z"}
 {"id":"1767626127106045159","url":"https://x.com/user_e57091aa/status/1767626127106045159","created_at":"2026-04-16T21:18:05.000Z","author":{"id":"100508716875","username":"user_e57091aa","display_name":"User c7b576","verified":false,"is_blue_verified":true,"follower_count":485174},"text":"Kite pulse sigma xenon terra ionic delta unity. valve quartz vortex trace kraft helix joule beta. query zonal.","lang":"en","metrics":{"likes":373,"retweets":19,"replies":23,"views":111146,"bookmarks":59,"quotes":7},"media":[],"urls":[],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":"2236680614779141805","conversation_id":"1767626127106045159","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":false,"source_endpoint":"HomeTimeline","captured_at":"2025-01-01T00:00:00.000Z"}
 {"id":"9651080193327180310","url":"https://x.com/user_85536508/status/9651080193327180310","created_at":"2026-04-16T19:57:55.000Z","author":{"id":"209260777194","username":"user_85536508","display_name":"User c624c9","verified":false,"is_blue_verified":true,"follower_count":594545},"text":"Ionic flux jade haze vortex lunar delta index. yield alpha valve core index core echo trace. zonal core bloom zonal yield wave index kite. xenon relay logic echo ratio logic kraft xenon.","lang":"en","metrics":{"likes":0,"retweets":671,"replies":0,"views":10,"bookmarks":0,"quotes":0},"media":[],"urls":[],"hashtags":[],"mentions":[{"id":"797042816569","username":"user_ebe970cb"}],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"9651080193327180310","is_retweet":true,"retweeted_tweet_id":"2409614990081141699","is_subscriber_only":false,"source_endpoint":"HomeTimeline","captured_at":"2025-01-01T00:00:00.000Z"}
diff --git a/tests/fixtures/sanitized/timeline-basic/manifest.json b/tests/fixtures/sanitized/timeline-basic/manifest.json
index c68ac80..1394c1b 100644
--- a/tests/fixtures/sanitized/timeline-basic/manifest.json
+++ b/tests/fixtures/sanitized/timeline-basic/manifest.json
@@ -6,7 +6,7 @@
     "fixture": "fixture.json",
     "expected": "expected.jsonl"
   },
-  "tweet_count": 39,
+  "tweet_count": 38,
   "remap_stats": {
     "tweet_ids": 166,
     "user_ids": 77,
diff --git a/tests/fixtures/sanitized/user-articles/README.md b/tests/fixtures/sanitized/user-articles/README.md
index 05ebf0e..0d485e3 100644
--- a/tests/fixtures/sanitized/user-articles/README.md
+++ b/tests/fixtures/sanitized/user-articles/README.md
@@ -2,20 +2,16 @@
 
 ## Scenario
 
-Basic timeline capture from a `UserArticlesTweets` response exercising the core
-xTap tweet parsing pipeline.
+Timeline capture from a `UserArticlesTweets` response where every tweet is an
+article stub without full body content.
 
 ## What it covers
 
 - Timeline instruction parsing (TimelineAddEntries)
 - Single tweet items (TimelineTimelineItem)
 - Conversation modules (TimelineTimelineModule)
-- Tweet normalization (author, text, metrics, media, URLs, mentions)
-- Reply chains, quote tweets, retweets
-- Note tweets (long-form posts)
-- Article stubs
-- Card embeds
-- Cursor entries (skipped by parser)
+- Article stubs skipped when full content is unavailable
+- Empty parser output when no complete tweets remain
 
 ## Raw input
 
@@ -46,7 +42,7 @@ and author references remain internally consistent.
 ## Invariants
 
 - `fixture.json` fed to `extractTweets("UserArticlesTweets", data)` must produce
-  exactly 12 tweets matching `expected.jsonl`
+  exactly 0 tweets matching `expected.jsonl`
 - All `in_reply_to`, `quoted_tweet_id`, and `conversation_id` values that
   reference tweets in the fixture use the same remapped IDs
 - Author IDs and handles are consistent across all tweets by the same user
diff --git a/tests/fixtures/sanitized/user-articles/expected.jsonl b/tests/fixtures/sanitized/user-articles/expected.jsonl
index 37c70c9..e69de29 100644
--- a/tests/fixtures/sanitized/user-articles/expected.jsonl
+++ b/tests/fixtures/sanitized/user-articles/expected.jsonl
@@ -1,12 +0,0 @@
-{"id":"905108854956","url":"https://x.com/user_177791b9/status/905108854956","created_at":"2026-04-05T06:51:33.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":58086},"text":"Haze helix ratio terra.","lang":"zxx","metrics":{"likes":171,"retweets":2,"replies":17,"views":16416,"bookmarks":210,"quotes":0},"media":[],"urls":[{"display":"example.com/san_966d002e1d","expanded":"https://example.com/sanitized/f8e6ed4de5","shortened":"https://t.co/SAN_e7e0cb51"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"8522401703929171399","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":true,"source_endpoint":"UserArticlesTweets","captured_at":"2025-01-01T00:00:00.000Z"}
-{"id":"122792578054","url":"https://x.com/user_177791b9/status/122792578054","created_at":"2026-03-29T07:02:28.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":58086},"text":"Kraft ionic sigma micro.","lang":"zxx","metrics":{"likes":187,"retweets":2,"replies":20,"views":14427,"bookmarks":220,"quotes":1},"media":[],"urls":[{"display":"example.com/san_68b171c933","expanded":"https://example.com/sanitized/a947e586e2","shortened":"https://t.co/SAN_499f8445"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"8566492360084009186","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":true,"source_endpoint":"UserArticlesTweets","captured_at":"2025-01-01T00:00:00.000Z"}
-{"id":"592753624451","url":"https://x.com/user_177791b9/status/592753624451","created_at":"2026-03-21T19:53:01.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":58086},"text":"Widen core query ionic.","lang":"zxx","metrics":{"likes":161,"retweets":1,"replies":12,"views":13794,"bookmarks":222,"quotes":0},"media":[],"urls":[{"display":"example.com/san_5f2f6e3798","expanded":"https://example.com/sanitized/85b08c6abe","shortened":"https://t.co/SAN_418621c8"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"5123188778018280265","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":true,"source_endpoint":"UserArticlesTweets","captured_at":"2025-01-01T00:00:00.000Z"}
-{"id":"293047059165","url":"https://x.com/user_177791b9/status/293047059165","created_at":"2026-03-14T16:35:23.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":58086},"text":"Alpha node apex apex echo.","lang":"zxx","metrics":{"likes":187,"retweets":3,"replies":14,"views":12333,"bookmarks":232,"quotes":0},"media":[],"urls":[{"display":"example.com/san_4515d8ecdc","expanded":"https://example.com/sanitized/034dd0440e","shortened":"https://t.co/SAN_b973fd03"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"1112622143004065384","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":true,"source_endpoint":"UserArticlesTweets","captured_at":"2025-01-01T00:00:00.000Z"}
-{"id":"970346563490","url":"https://x.com/user_177791b9/status/970346563490","created_at":"2026-03-07T06:44:59.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":58086},"text":"Glyph drift widen vortex.","lang":"zxx","metrics":{"likes":182,"retweets":4,"replies":17,"views":45958,"bookmarks":263,"quotes":1},"media":[],"urls":[{"display":"example.com/san_c813c70b34","expanded":"https://example.com/sanitized/dc63cf839c","shortened":"https://t.co/SAN_5fff11d5"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"8511881747913975421","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":true,"source_endpoint":"UserArticlesTweets","captured_at":"2025-01-01T00:00:00.000Z"}
-{"id":"900849105948","url":"https://x.com/user_177791b9/status/900849105948","created_at":"2026-03-01T18:54:57.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":58086},"text":"Widen optic pulse xenon.","lang":"zxx","metrics":{"likes":174,"retweets":2,"replies":14,"views":34447,"bookmarks":246,"quotes":3},"media":[],"urls":[{"display":"example.com/san_048b3ecfef","expanded":"https://example.com/sanitized/f861486f28","shortened":"https://t.co/SAN_a2d22b97"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"3577207859051471915","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":true,"source_endpoint":"UserArticlesTweets","captured_at":"2025-01-01T00:00:00.000Z"}
-{"id":"225666192008","url":"https://x.com/user_177791b9/status/225666192008","created_at":"2026-02-24T05:53:13.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":58086},"text":"Helix mesa echo nexus pulse.","lang":"zxx","metrics":{"likes":104,"retweets":0,"replies":4,"views":8702,"bookmarks":140,"quotes":0},"media":[],"urls":[{"display":"example.com/san_906f61679e","expanded":"https://example.com/sanitized/c489c786b9","shortened":"https://t.co/SAN_ed46afa3"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"8443619391608446109","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":true,"source_endpoint":"UserArticlesTweets","captured_at":"2025-01-01T00:00:00.000Z"}
-{"id":"576729378577","url":"https://x.com/user_177791b9/status/576729378577","created_at":"2026-02-23T08:11:27.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":58086},"text":"Pulse glyph trace ionic.","lang":"zxx","metrics":{"likes":150,"retweets":1,"replies":12,"views":14598,"bookmarks":127,"quotes":1},"media":[],"urls":[{"display":"example.com/san_ae3c7ada6a","expanded":"https://example.com/sanitized/ab94f3fb79","shortened":"https://t.co/SAN_32be0c69"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"1050573815646234460","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":true,"source_endpoint":"UserArticlesTweets","captured_at":"2025-01-01T00:00:00.000Z"}
-{"id":"434953211468","url":"https://x.com/user_177791b9/status/434953211468","created_at":"2026-02-21T20:54:34.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":58086},"text":"Kraft glyph unity apex.","lang":"zxx","metrics":{"likes":169,"retweets":4,"replies":8,"views":32299,"bookmarks":197,"quotes":2},"media":[],"urls":[{"display":"example.com/san_ae3c7ada6a","expanded":"https://example.com/sanitized/d72395721b","shortened":"https://t.co/SAN_c899ab98"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"5834632303453035368","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":true,"source_endpoint":"UserArticlesTweets","captured_at":"2025-01-01T00:00:00.000Z"}
-{"id":"110900722835","url":"https://x.com/user_177791b9/status/110900722835","created_at":"2026-02-15T15:07:19.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":58086},"text":"Glyph kite unity query.","lang":"zxx","metrics":{"likes":191,"retweets":3,"replies":24,"views":21669,"bookmarks":211,"quotes":1},"media":[],"urls":[{"display":"example.com/san_712cd6922e","expanded":"https://example.com/sanitized/3673e46bd7","shortened":"https://t.co/SAN_f1ff69b1"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"1011523868042915170","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":true,"source_endpoint":"UserArticlesTweets","captured_at":"2025-01-01T00:00:00.000Z"}
-{"id":"488131094028","url":"https://x.com/user_177791b9/status/488131094028","created_at":"2026-02-08T19:12:37.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":58086},"text":"Query grid kite micro sigma.","lang":"zxx","metrics":{"likes":205,"retweets":3,"replies":29,"views":20324,"bookmarks":270,"quotes":1},"media":[],"urls":[{"display":"example.com/san_2d8337a9cc","expanded":"https://example.com/sanitized/29673ba864","shortened":"https://t.co/SAN_957f0ba1"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"8297556799354132528","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":true,"source_endpoint":"UserArticlesTweets","captured_at":"2025-01-01T00:00:00.000Z"}
-{"id":"262868366999","url":"https://x.com/user_177791b9/status/262868366999","created_at":"2026-02-01T13:59:04.000Z","author":{"id":"568793795593","username":"user_177791b9","display_name":"User 610e3e","verified":false,"is_blue_verified":true,"follower_count":58086},"text":"Beta delta logic yield.","lang":"zxx","metrics":{"likes":204,"retweets":2,"replies":17,"views":29998,"bookmarks":205,"quotes":1},"media":[],"urls":[{"display":"example.com/san_315d846f7a","expanded":"https://example.com/sanitized/d1cf49d0eb","shortened":"https://t.co/SAN_01396dc3"}],"hashtags":[],"mentions":[],"in_reply_to":null,"quoted_tweet_id":null,"conversation_id":"1822623311959686325","is_retweet":false,"retweeted_tweet_id":null,"is_subscriber_only":true,"source_endpoint":"UserArticlesTweets","captured_at":"2025-01-01T00:00:00.000Z"}
diff --git a/tests/fixtures/sanitized/user-articles/manifest.json b/tests/fixtures/sanitized/user-articles/manifest.json
index a4f2e4a..98a4250 100644
--- a/tests/fixtures/sanitized/user-articles/manifest.json
+++ b/tests/fixtures/sanitized/user-articles/manifest.json
@@ -1,12 +1,12 @@
 {
   "scenario": "user-articles",
   "endpoint": "UserArticlesTweets",
-  "description": "Basic UserArticlesTweets timeline capture with standard tweet types",
+  "description": "UserArticlesTweets capture with article stubs that must be skipped",
   "files": {
     "fixture": "fixture.json",
     "expected": "expected.jsonl"
   },
-  "tweet_count": 12,
+  "tweet_count": 0,
   "remap_stats": {
     "tweet_ids": 29,
     "user_ids": 26,
diff --git a/tests/fixtures/sanitized/user-highlights/README.md b/tests/fixtures/sanitized/user-highlights/README.md
index 424cfc3..204d79c 100644
--- a/tests/fixtures/sanitized/user-highlights/README.md
+++ b/tests/fixtures/sanitized/user-highlights/README.md
@@ -13,7 +13,7 @@ xTap tweet parsing pipeline.
 - Tweet normalization (author, text, metrics, media, URLs, mentions)
 - Reply chains, quote tweets, retweets
 - Note tweets (long-form posts)
-- Article stubs
+- Article stubs (skipped when full content is unavailable)
 - Card embeds
 - Cursor entries (skipped by parser)
 
diff --git a/tests/parser-golden.test.mjs b/tests/parser-golden.test.mjs
index 670d624..61a35d8 100644
--- a/tests/parser-golden.test.mjs
+++ b/tests/parser-golden.test.mjs
@@ -61,8 +61,8 @@ describe('parser golden scenarios', () => {
   for (const { name, dir, manifest } of scenarios) {
     it(name, () => {
       const fixture = JSON.parse(readFileSync(join(dir, manifest.files.fixture), 'utf8'));
-      const expectedLines = readFileSync(join(dir, manifest.files.expected), 'utf8').trim().split('\n');
-      const expected = expectedLines.map(l => JSON.parse(l));
+      const expectedText = readFileSync(join(dir, manifest.files.expected), 'utf8').trim();
+      const expected = expectedText ? expectedText.split('\n').map(l => JSON.parse(l)) : [];
 
       const actual = extractTweets(manifest.endpoint, fixture);
       // source_endpoint is set by the caller (service worker), not by extractTweets
diff --git a/tests/tweet-parser.test.mjs b/tests/tweet-parser.test.mjs
index 4274c31..199fae2 100644
--- a/tests/tweet-parser.test.mjs
+++ b/tests/tweet-parser.test.mjs
@@ -221,6 +221,20 @@ describe('normalizeTweet', () => {
     assert.equal(t.article.title, 'My Article');
   });
 
+  it('skips article stubs without full content', () => {
+    const raw = makeRawTweet({
+      article: {
+        article_results: {
+          result: {
+            title: 'Stub only',
+            preview_text: 'Preview only',
+          },
+        },
+      },
+    });
+    assert.equal(normalizeTweet(raw), null);
+  });
+
   it('converts created_at to ISO format', () => {
     const raw = makeRawTweet();
     const t = normalizeTweet(raw);
@@ -571,6 +585,40 @@ describe('extractTweets', () => {
     assert.equal(tweets[0].is_article, true);
   });
 
+  it('TweetResultByRestId with article stub returns no tweet', () => {
+    const raw = makeRawTweet({
+      article: {
+        article_results: {
+          result: {
+            title: 'Stub only',
+            preview_text: 'Preview only',
+          },
+        },
+      },
+    });
+    const data = {
+      data: { tweetResult: { result: raw } },
+    };
+    const tweets = extractTweets('TweetResultByRestId', data);
+    assert.deepStrictEqual(tweets, []);
+  });
+
+  it('timeline article stub returns no tweet', () => {
+    const raw = makeRawTweet({
+      article: {
+        article_results: {
+          result: {
+            title: 'Stub only',
+            preview_text: 'Preview only',
+          },
+        },
+      },
+    });
+    const data = makeTimelineResponse(raw);
+    const tweets = extractTweets('HomeTimeline', data);
+    assert.deepStrictEqual(tweets, []);
+  });
+
   it('skips cursor entries', () => {
     const data = {
       data: {

From b54e447b5254bc2b5c87fa1d4ff333c9f680e3e4 Mon Sep 17 00:00:00 2001
From: Milan Kubicek <5452309+mkubicek@users.noreply.github.com>
Date: Tue, 26 May 2026 23:10:29 +0200
Subject: [PATCH 2/2] test: support empty e2e golden output

---
 tests/e2e/golden-jsonl-test.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/e2e/golden-jsonl-test.js b/tests/e2e/golden-jsonl-test.js
index ac95cd4..d76b4c5 100644
--- a/tests/e2e/golden-jsonl-test.js
+++ b/tests/e2e/golden-jsonl-test.js
@@ -334,8 +334,8 @@ async function run() {
       // Compare only this scenario's tweets (slice off previous scenarios)
       const scenarioActual = actual.slice(cumulativeTweets);
       const expectedJsonl = join(scenario.dir, scenario.manifest.files.expected);
-      const expectedLines = readFileSync(expectedJsonl, 'utf8').trim().split('\n');
-      const expected = expectedLines.map(l => JSON.parse(l));
+      const expectedText = readFileSync(expectedJsonl, 'utf8').trim();
+      const expected = expectedText ? expectedText.split('\n').map(l => JSON.parse(l)) : [];
 
       const result = compareJsonl(scenarioActual, expected);
       if (!result.pass) {