diff --git a/docketlab/fedreg.py b/docketlab/fedreg.py index 1554b96..c82d88b 100644 --- a/docketlab/fedreg.py +++ b/docketlab/fedreg.py @@ -109,7 +109,7 @@ def find_documents(docket_id: str) -> list[dict]: "fields[]": [ "document_number", "title", "type", "publication_date", "start_page", "end_page", "raw_text_url", "comments_close_on", - "effective_on", "html_url", + "effective_on", "html_url", "action", "full_text_xml_url", ], }, ) @@ -157,10 +157,14 @@ def _pick(docs: list[dict], want: str) -> tuple[dict | None, list[str]]: pool = plain or eligible chosen = max(pool, key=lambda d: (_pages(d), d.get("publication_date") or "")) else: - # A supplemental proposal supersedes the original it amends. + # A supplemental proposal supersedes the original it amends. Absent one, + # length decides: a docket carries ancillary documents typed "Proposed + # Rule" that no action keyword predicts - a notice of posting an + # informational video, of data availability, of a public hearing. Ranking + # by date first handed the NPRM slot to whichever of those came last. supp = [d for d in eligible if _SUPPLEMENTAL.search(d.get("action") or "")] pool = supp or eligible - chosen = max(pool, key=lambda d: (d.get("publication_date") or "", _pages(d))) + chosen = max(pool, key=lambda d: (_pages(d), d.get("publication_date") or "")) notes.append( f"chose {chosen['document_number']} ({_pages(chosen)}pp, " diff --git a/stress.py b/stress.py index a7b0884..45e944a 100644 --- a/stress.py +++ b/stress.py @@ -305,6 +305,42 @@ def seed_diff(mags): _n["document_number"] != "2021-27312") check("the selection explains itself", any("chose" in x for x in _fw), str(_fw[:2])) +# The checks above hand _pick documents that carry an "action". find_documents +# does not request that field, so in the live pipeline every action test above +# is vacuous and selection falls through to the ordering rules. Both are checked +# here: that the field is actually requested, and that the ordering is right +# even when it is absent. +_asked = {} +def _capture(path, params=None): + _asked.update(params or {}) + return {"results": []} +_real_get, _fr._get = _fr._get, _capture +try: + _fr.find_documents("X-0001") +finally: + _fr._get = _real_get +_fields = set(_asked.get("fields[]") or []) +check("find_documents requests the action field", "action" in _fields, str(sorted(_fields))) +check("find_documents requests the XML url, without which structural parsing " + "never runs", "full_text_xml_url" in _fields) + +# A docket carries ancillary documents typed "Proposed Rule" that no action +# keyword predicts - a notice of posting a video, of data availability, of a +# hearing. Ranking proposals by date first gave the NPRM slot to whichever came +# last. This is the real CMMC docket with the action text stripped, as +# find_documents currently returns it. +_NOACTION = [ + {"document_number":"2023-27280","type":"Proposed Rule","publication_date":"2023-12-26", + "start_page":1,"end_page":81}, + {"document_number":"2024-03460","type":"Proposed Rule","publication_date":"2024-02-21", + "start_page":1,"end_page":1}, + {"document_number":"2024-22905","type":"Rule","publication_date":"2024-10-15", + "start_page":1,"end_page":146}, +] +_n2, _ = _fr._pick(_NOACTION, "Proposed Rule") +check("a later one-page notice does not displace the NPRM", + _n2["document_number"] == "2023-27280", f"chose {_n2['document_number']}") + # #2 - prose ending "...we received comments." opened a pair. _PROSE = ("The agency reviewed the record. In response to the notice we received comments. " "Commenters raised concerns about methodology and the underlying data at length.\n\n"