Keep WordPress drafts out of the published archive

Extractor.py

@@ -13,6 +13,10 @@
   # The byline for every pre-Co-Authors-Plus post; without it they are authorless.
   "dc:creator",
   "wp:comment_status",
+  # Publication state. Without it every <item> becomes a live article: 16 posts
+  # WordPress was holding as draft, pending or private went public on
+  # thetriangle.org, four of them from its five private posts.
+  "wp:status",
   "description",
   "wp:post_id",
   "wp:post_name",
@@ -52,6 +56,7 @@
   _CONTENT_ENCODED_TAG: "content:encoded",
   _DC_CREATOR_TAG: "dc:creator",
   f"{{{_WP_NS}}}comment_status": "wp:comment_status",
+  f"{{{_WP_NS}}}status": "wp:status",
   f"{{{_WP_NS}}}post_id": "wp:post_id",
   f"{{{_WP_NS}}}post_name": "wp:post_name",
   f"{{{_WP_NS}}}post_modified_gmt": "wp:post_modified_gmt",

Formatter/ArticleFormatter.py

@@ -126,6 +126,17 @@ def _to_cms_row(self, obj):
 
         metadata = obj.get("metadata")
 
+        # The CMS reads "published" as a non-null pub_date, so withholding the
+        # date is what makes a WordPress draft arrive as a CMS draft: editable,
+        # dated, and absent from every public path. creation_date above is taken
+        # from the same value before this point, so the timeline survives.
+        #
+        # Absent status means an export from before Extractor collected it;
+        # those import as published, matching the old behaviour.
+        pub_date = self._normalize_datetime(obj.get("pubDate"))
+        if str(obj.get("status", "publish")).strip().lower() != "publish":
+            pub_date = None
+
         return {
             **self._seo_columns(metadata),
             "id": obj.get("id"),
@@ -140,7 +151,7 @@ def _to_cms_row(self, obj):
             "priority": obj.get("priority"),
             "mod_date": self._normalize_datetime(obj.get("modDate")),
             "photo_url": photo_url,
-            "pub_date": self._normalize_datetime(obj.get("pubDate")),
+            "pub_date": pub_date,
             "tags": obj.get("tags"),
             "categories": obj.get("categories"),
             "metadata": metadata,

Translator/ArticleTranslator.py

@@ -35,6 +35,7 @@ def _getArticleData(self, data):
       # once the attachments have been indexed (resolveFeaturedImages).
       "photoURL": self._checkForImg(text),
       "pubDate": data.get('wp:post_date_gmt', DEFAULT_VALUE),
+      "status": self._normalizeStatus(data.get('wp:status')),
       "tags": data.get('category'),
       "categories": [],
       "metadata": data.get('wp:postmeta'),
@@ -68,6 +69,19 @@ def _dataSanityCheck(self, obj, debugMode=False):
     # subsection that looked like a broken section page rather than a filter.
     return isTextNotNull and isTitleNotUnderscore
 
+  def _normalizeStatus(self, value):
+    # WordPress publication state. Anything that is not exactly "publish" --
+    # draft, pending, private, future, trash -- is content the newsroom has not
+    # released, and the formatter withholds a pub_date for it.
+    #
+    # Defaults to "publish" when the key is absent so an export that predates
+    # this field still imports its archive as published rather than blanking
+    # ten thousand articles.
+    if value is None:
+      return "publish"
+    normalized = str(value).strip().lower()
+    return normalized or "publish"
+
   def _normalizeCommentStatus(self, value):
     # WordPress exports carry inconsistent casing/whitespace for comment_status
     # ("open", "Open", " closed"). Force a uniform enum here so downstream

tests/test_article_status.py

@@ -0,0 +1,77 @@
+"""Tests that WordPress publication state survives the pipeline. Run from the
+repo root:
+
+    .venv/bin/python -m unittest tests.test_article_status
+
+Regression: the extractor never collected wp:status, so every <item> became a
+live article. 16 posts WordPress was holding as draft, pending or private were
+published on thetriangle.org, including four of its five private posts.
+"""
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from Extractor import _POST_ITEM_KEYS
+from Formatter.ArticleFormatter import ArticleFormatter
+from Translator.ArticleTranslator import ArticleTranslator
+
+
+class StatusIsExtracted(unittest.TestCase):
+    def test_status_is_collected_from_the_export(self):
+        # The root cause: the key was simply absent from the extracted set, so
+        # there was no status downstream to filter on.
+        self.assertIn("wp:status", _POST_ITEM_KEYS)
+
+
+class StatusNormalization(unittest.TestCase):
+    def _status(self, value):
+        return ArticleTranslator([])._normalizeStatus(value)
+
+    def test_publish_variants_normalize(self):
+        for value in ("publish", "Publish", "  publish  "):
+            with self.subTest(value=value):
+                self.assertEqual(self._status(value), "publish")
+
+    def test_withheld_states_keep_their_name(self):
+        for value in ("draft", "pending", "private", "future", "trash"):
+            with self.subTest(value=value):
+                self.assertEqual(self._status(value), value)
+
+    def test_absent_status_defaults_to_publish(self):
+        # An export taken before the extractor collected the field must still
+        # import its archive as published, not blank ten thousand articles.
+        for value in (None, "", "   "):
+            with self.subTest(value=value):
+                self.assertEqual(self._status(value), "publish")
+
+
+class PubDateReflectsStatus(unittest.TestCase):
+    def _row(self, status):
+        obj = {"pubDate": "2016-11-06 01:14:09"}
+        if status is not None:
+            obj["status"] = status
+        return ArticleFormatter([])._to_cms_row(obj)
+
+    def test_published_posts_keep_their_date(self):
+        self.assertEqual(self._row("publish")["pub_date"], "2016-11-06 01:14:09")
+
+    def test_withheld_posts_arrive_as_drafts(self):
+        # The CMS reads a null pub_date as "draft": present for editors, absent
+        # from every public path.
+        for status in ("draft", "pending", "private", "future", "trash"):
+            with self.subTest(status=status):
+                self.assertIsNone(self._row(status)["pub_date"])
+
+    def test_withheld_posts_keep_their_timeline(self):
+        # creation_date is what preserves when the piece was written, so a
+        # recovered draft still sorts correctly for an editor.
+        self.assertEqual(self._row("private")["creation_date"], "2016-11-06 01:14:09")
+
+    def test_missing_status_still_publishes(self):
+        self.assertEqual(self._row(None)["pub_date"], "2016-11-06 01:14:09")
+
+
+if __name__ == "__main__":
+    unittest.main()