From 1b09f051f0b630fde4c5a29b2c9dd3983a40a93d Mon Sep 17 00:00:00 2001 From: ssavutu Date: Wed, 12 Aug 2026 19:50:16 -0400 Subject: [PATCH] Keep WordPress drafts out of the published archive The extractor never collected wp:status, so there was nothing downstream to filter on and every of post_type "post" became a live article. On Delta that published 18 posts WordPress was holding back -- 4 of its 5 private posts, its only pending one, and 13 drafts -- each reachable at its own URL and marked "index, follow". A private 2016 sponsored post and a story about a business being cited for violations were among them. Status now rides along from Extractor through the translator, and the formatter withholds pub_date for anything that is not "publish". That is exactly how the CMS models a draft (PublishDueArticles keys on a null pub_date), so a withheld post lands as an editable draft rather than a live page. creation_date is resolved before that point, so the piece keeps its date and still sorts. A missing status defaults to "publish": an export taken before the extractor collected the field must still import its archive as published rather than blanking ten thousand articles. The 18 already-published posts were archived directly in prod; this stops a reseed from putting them back. Co-Authored-By: Claude Opus 5 --- Extractor.py | 5 +++ Formatter/ArticleFormatter.py | 13 +++++- Translator/ArticleTranslator.py | 14 ++++++ tests/test_article_status.py | 77 +++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 tests/test_article_status.py diff --git a/Extractor.py b/Extractor.py index fb9d916..5e011a4 100644 --- a/Extractor.py +++ b/Extractor.py @@ -13,6 +13,10 @@ # The byline for every pre-Co-Authors-Plus post; without it they are authorless. "dc:creator", "wp:comment_status", + # Publication state. Without it every becomes a live article: 16 posts + # WordPress was holding as draft, pending or private went public on + # thetriangle.org, four of them from its five private posts. + "wp:status", "description", "wp:post_id", "wp:post_name", @@ -52,6 +56,7 @@ _CONTENT_ENCODED_TAG: "content:encoded", _DC_CREATOR_TAG: "dc:creator", f"{{{_WP_NS}}}comment_status": "wp:comment_status", + f"{{{_WP_NS}}}status": "wp:status", f"{{{_WP_NS}}}post_id": "wp:post_id", f"{{{_WP_NS}}}post_name": "wp:post_name", f"{{{_WP_NS}}}post_modified_gmt": "wp:post_modified_gmt", diff --git a/Formatter/ArticleFormatter.py b/Formatter/ArticleFormatter.py index b4a33dc..362cb69 100644 --- a/Formatter/ArticleFormatter.py +++ b/Formatter/ArticleFormatter.py @@ -126,6 +126,17 @@ def _to_cms_row(self, obj): metadata = obj.get("metadata") + # The CMS reads "published" as a non-null pub_date, so withholding the + # date is what makes a WordPress draft arrive as a CMS draft: editable, + # dated, and absent from every public path. creation_date above is taken + # from the same value before this point, so the timeline survives. + # + # Absent status means an export from before Extractor collected it; + # those import as published, matching the old behaviour. + pub_date = self._normalize_datetime(obj.get("pubDate")) + if str(obj.get("status", "publish")).strip().lower() != "publish": + pub_date = None + return { **self._seo_columns(metadata), "id": obj.get("id"), @@ -140,7 +151,7 @@ def _to_cms_row(self, obj): "priority": obj.get("priority"), "mod_date": self._normalize_datetime(obj.get("modDate")), "photo_url": photo_url, - "pub_date": self._normalize_datetime(obj.get("pubDate")), + "pub_date": pub_date, "tags": obj.get("tags"), "categories": obj.get("categories"), "metadata": metadata, diff --git a/Translator/ArticleTranslator.py b/Translator/ArticleTranslator.py index c5de79e..ec81fe5 100644 --- a/Translator/ArticleTranslator.py +++ b/Translator/ArticleTranslator.py @@ -35,6 +35,7 @@ def _getArticleData(self, data): # once the attachments have been indexed (resolveFeaturedImages). "photoURL": self._checkForImg(text), "pubDate": data.get('wp:post_date_gmt', DEFAULT_VALUE), + "status": self._normalizeStatus(data.get('wp:status')), "tags": data.get('category'), "categories": [], "metadata": data.get('wp:postmeta'), @@ -68,6 +69,19 @@ def _dataSanityCheck(self, obj, debugMode=False): # subsection that looked like a broken section page rather than a filter. return isTextNotNull and isTitleNotUnderscore + def _normalizeStatus(self, value): + # WordPress publication state. Anything that is not exactly "publish" -- + # draft, pending, private, future, trash -- is content the newsroom has not + # released, and the formatter withholds a pub_date for it. + # + # Defaults to "publish" when the key is absent so an export that predates + # this field still imports its archive as published rather than blanking + # ten thousand articles. + if value is None: + return "publish" + normalized = str(value).strip().lower() + return normalized or "publish" + def _normalizeCommentStatus(self, value): # WordPress exports carry inconsistent casing/whitespace for comment_status # ("open", "Open", " closed"). Force a uniform enum here so downstream diff --git a/tests/test_article_status.py b/tests/test_article_status.py new file mode 100644 index 0000000..7939aa4 --- /dev/null +++ b/tests/test_article_status.py @@ -0,0 +1,77 @@ +"""Tests that WordPress publication state survives the pipeline. Run from the +repo root: + + .venv/bin/python -m unittest tests.test_article_status + +Regression: the extractor never collected wp:status, so every became a +live article. 16 posts WordPress was holding as draft, pending or private were +published on thetriangle.org, including four of its five private posts. +""" +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from Extractor import _POST_ITEM_KEYS +from Formatter.ArticleFormatter import ArticleFormatter +from Translator.ArticleTranslator import ArticleTranslator + + +class StatusIsExtracted(unittest.TestCase): + def test_status_is_collected_from_the_export(self): + # The root cause: the key was simply absent from the extracted set, so + # there was no status downstream to filter on. + self.assertIn("wp:status", _POST_ITEM_KEYS) + + +class StatusNormalization(unittest.TestCase): + def _status(self, value): + return ArticleTranslator([])._normalizeStatus(value) + + def test_publish_variants_normalize(self): + for value in ("publish", "Publish", " publish "): + with self.subTest(value=value): + self.assertEqual(self._status(value), "publish") + + def test_withheld_states_keep_their_name(self): + for value in ("draft", "pending", "private", "future", "trash"): + with self.subTest(value=value): + self.assertEqual(self._status(value), value) + + def test_absent_status_defaults_to_publish(self): + # An export taken before the extractor collected the field must still + # import its archive as published, not blank ten thousand articles. + for value in (None, "", " "): + with self.subTest(value=value): + self.assertEqual(self._status(value), "publish") + + +class PubDateReflectsStatus(unittest.TestCase): + def _row(self, status): + obj = {"pubDate": "2016-11-06 01:14:09"} + if status is not None: + obj["status"] = status + return ArticleFormatter([])._to_cms_row(obj) + + def test_published_posts_keep_their_date(self): + self.assertEqual(self._row("publish")["pub_date"], "2016-11-06 01:14:09") + + def test_withheld_posts_arrive_as_drafts(self): + # The CMS reads a null pub_date as "draft": present for editors, absent + # from every public path. + for status in ("draft", "pending", "private", "future", "trash"): + with self.subTest(status=status): + self.assertIsNone(self._row(status)["pub_date"]) + + def test_withheld_posts_keep_their_timeline(self): + # creation_date is what preserves when the piece was written, so a + # recovered draft still sorts correctly for an editor. + self.assertEqual(self._row("private")["creation_date"], "2016-11-06 01:14:09") + + def test_missing_status_still_publishes(self): + self.assertEqual(self._row(None)["pub_date"], "2016-11-06 01:14:09") + + +if __name__ == "__main__": + unittest.main()