Request action and full_text_xml_url; rank proposals by length

docketlab/fedreg.py

@@ -109,7 +109,7 @@ def find_documents(docket_id: str) -> list[dict]:
             "fields[]": [
                 "document_number", "title", "type", "publication_date",
                 "start_page", "end_page", "raw_text_url", "comments_close_on",
-                "effective_on", "html_url",
+                "effective_on", "html_url", "action", "full_text_xml_url",
             ],
         },
     )
@@ -157,10 +157,14 @@ def _pick(docs: list[dict], want: str) -> tuple[dict | None, list[str]]:
         pool = plain or eligible
         chosen = max(pool, key=lambda d: (_pages(d), d.get("publication_date") or ""))
     else:
-        # A supplemental proposal supersedes the original it amends.
+        # A supplemental proposal supersedes the original it amends. Absent one,
+        # length decides: a docket carries ancillary documents typed "Proposed
+        # Rule" that no action keyword predicts - a notice of posting an
+        # informational video, of data availability, of a public hearing. Ranking
+        # by date first handed the NPRM slot to whichever of those came last.
         supp = [d for d in eligible if _SUPPLEMENTAL.search(d.get("action") or "")]
         pool = supp or eligible
-        chosen = max(pool, key=lambda d: (d.get("publication_date") or "", _pages(d)))
+        chosen = max(pool, key=lambda d: (_pages(d), d.get("publication_date") or ""))
 
     notes.append(
         f"chose {chosen['document_number']} ({_pages(chosen)}pp, "

stress.py

@@ -305,6 +305,42 @@ def seed_diff(mags):
       _n["document_number"] != "2021-27312")
 check("the selection explains itself", any("chose" in x for x in _fw), str(_fw[:2]))
 
+# The checks above hand _pick documents that carry an "action". find_documents
+# does not request that field, so in the live pipeline every action test above
+# is vacuous and selection falls through to the ordering rules. Both are checked
+# here: that the field is actually requested, and that the ordering is right
+# even when it is absent.
+_asked = {}
+def _capture(path, params=None):
+    _asked.update(params or {})
+    return {"results": []}
+_real_get, _fr._get = _fr._get, _capture
+try:
+    _fr.find_documents("X-0001")
+finally:
+    _fr._get = _real_get
+_fields = set(_asked.get("fields[]") or [])
+check("find_documents requests the action field", "action" in _fields, str(sorted(_fields)))
+check("find_documents requests the XML url, without which structural parsing "
+      "never runs", "full_text_xml_url" in _fields)
+
+# A docket carries ancillary documents typed "Proposed Rule" that no action
+# keyword predicts - a notice of posting a video, of data availability, of a
+# hearing. Ranking proposals by date first gave the NPRM slot to whichever came
+# last. This is the real CMMC docket with the action text stripped, as
+# find_documents currently returns it.
+_NOACTION = [
+ {"document_number":"2023-27280","type":"Proposed Rule","publication_date":"2023-12-26",
+  "start_page":1,"end_page":81},
+ {"document_number":"2024-03460","type":"Proposed Rule","publication_date":"2024-02-21",
+  "start_page":1,"end_page":1},
+ {"document_number":"2024-22905","type":"Rule","publication_date":"2024-10-15",
+  "start_page":1,"end_page":146},
+]
+_n2, _ = _fr._pick(_NOACTION, "Proposed Rule")
+check("a later one-page notice does not displace the NPRM",
+      _n2["document_number"] == "2023-27280", f"chose {_n2['document_number']}")
+
 # #2 - prose ending "...we received comments." opened a pair.
 _PROSE = ("The agency reviewed the record. In response to the notice we received comments. "
           "Commenters raised concerns about methodology and the underlying data at length.\n\n"