fix(government_us): read USDA ERS renamed zip members and preliminary years

openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_farm_household_income_and_characteristics.py

@@ -170,7 +170,7 @@ def parse_year(column: str) -> int:
     int
         The four-digit year as an integer.
     """
-    return int(column.rstrip("F"))
+    return int(column.rstrip("FP"))
 
 
 def year_sort_key(column: str) -> tuple[int, bool]:
@@ -187,7 +187,7 @@ def year_sort_key(column: str) -> tuple[int, bool]:
         The integer year and whether the header is a forecast, so a forecast
         year sorts after a non-forecast year of the same value.
     """
-    return (parse_year(column), column.endswith("F"))
+    return (parse_year(column), column.endswith(("F", "P")))
 
 
 def parse_table(text: str, table: str) -> list[dict]:

openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_livestock_and_meat_domestic_data.py

@@ -15,7 +15,7 @@
 LIVESTOCK_AND_MEAT_FILES: dict[str, dict] = {
     "supply_and_disappearance": {
         "label": "Meat supply and disappearance",
-        "member": "MeatSD.csv",
+        "member": ("Meat-Supply-Disappearance.csv", "MeatSD.csv"),
         "table_name": None,
         "fold_dims": (("commodity_desc", "commodity"),),
         "col_dims": ("attribute_desc",),
@@ -25,7 +25,7 @@
     },
     "slaughter": {
         "label": "Livestock and poultry slaughter",
-        "member": "MeatStats.csv",
+        "member": ("Meat-Statistics.csv", "MeatStats.csv"),
         "table_name": "Livestock and poultry slaughter",
         "fold_dims": (("commodity_desc", "commodity"),),
         "col_dims": ("attribute_desc",),
@@ -35,7 +35,7 @@
     },
     "production": {
         "label": "Red meat and poultry production",
-        "member": "MeatStats.csv",
+        "member": ("Meat-Statistics.csv", "MeatStats.csv"),
         "table_name": "Red meat and poultry production",
         "fold_dims": (("commodity_desc", "commodity"),),
         "col_dims": ("attribute_desc",),
@@ -45,7 +45,7 @@
     },
     "live_and_dressed_weights": {
         "label": "Livestock and poultry live and dressed weights",
-        "member": "MeatStats.csv",
+        "member": ("Meat-Statistics.csv", "MeatStats.csv"),
         "table_name": "Livestock and poultry live and dressed weights",
         "fold_dims": (("commodity_desc", "commodity"),),
         "col_dims": ("attribute_desc",),
@@ -55,7 +55,7 @@
     },
     "cold_storage": {
         "label": "Red meat and poultry cold storage stocks",
-        "member": "MeatStats.csv",
+        "member": ("Meat-Statistics.csv", "MeatStats.csv"),
         "table_name": "Red meat and poultry beginning cold storage stocks",
         "fold_dims": (("commodity_desc", "commodity"),),
         "col_dims": ("attribute_desc",),
@@ -286,6 +286,14 @@ async def afetch_table(table: str, **kwargs) -> list[dict]:
 
     config = LIVESTOCK_AND_MEAT_FILES[table]
     content = await afetch_ers_file(ZIP_MEDIA, product=PRODUCT_PAGE)
+    members = config["member"]
+    if isinstance(members, str):
+        members = (members,)
     with zipfile.ZipFile(BytesIO(content)) as archive:
-        raw = archive.read(f"{ZIP_DIR}/{config['member']}")
+        names = {name.lower(): name for name in archive.namelist()}
+        member = next(
+            (names[key] for m in members if (key := f"{ZIP_DIR}/{m}".lower()) in names),
+            f"{ZIP_DIR}/{members[0]}",
+        )
+        raw = archive.read(member)
     return parse_table(raw.decode("utf-8-sig", errors="replace"), table)

openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_vegetables_and_pulses.py

@@ -9,7 +9,7 @@
     "data-products/vegetables-and-pulses-data/vegetables-and-pulses-yearbook-tables"
 )
 DATA_ZIP = "/media/5018/vegetables-and-pulses-machine-readable-data.zip"
-CSV_MEMBER = "Vegetables_Pulses.csv"
+CSV_MEMBERS = ("vegetables-pulses-data.csv", "Vegetables_Pulses.csv")
 
 DEFAULT_TABLE = "Table33_Potatoes"
 
@@ -239,5 +239,10 @@ async def afetch_table(table: str, **kwargs) -> list[dict]:
 
     content = await afetch_ers_file(DATA_ZIP, product=PRODUCT_PAGE)
     with zipfile.ZipFile(BytesIO(content)) as archive:
-        raw = archive.read(CSV_MEMBER)
+        names = {name.lower(): name for name in archive.namelist()}
+        member = next(
+            (names[m.lower()] for m in CSV_MEMBERS if m.lower() in names),
+            CSV_MEMBERS[0],
+        )
+        raw = archive.read(member)
     return parse_rows(raw.decode("utf-8-sig", errors="replace"), table)

openbb_platform/providers/government_us/tests/test_usda_farm_household_income_and_characteristics.py

@@ -194,6 +194,17 @@ def test_parse_year_and_sort_key(self):
             "2026F",
         ]
 
+    def test_parse_year_preliminary_suffix(self):
+        """A preliminary year such as 2025P parses and sorts like a forecast."""
+        assert parse_year("2025P") == 2025
+        assert year_sort_key("2025P") == (2025, True)
+        assert sorted(["2026F", "2025P", "2025", "2024"], key=year_sort_key) == [
+            "2024",
+            "2025",
+            "2025P",
+            "2026F",
+        ]
+
     def test_parse_table_year_shape(self):
         """A time-series table yields item/column/value records with the year."""
         records = parse_table(YEAR_CSV, "finances_2021_26f")

openbb_platform/providers/government_us/tests/test_usda_livestock_and_meat_domestic_data.py

@@ -118,7 +118,7 @@ def test_catalog_contents(self):
         meatstats = [
             key
             for key, config in LIVESTOCK_AND_MEAT_FILES.items()
-            if config["member"] == "MeatStats.csv"
+            if config["member"] == ("Meat-Statistics.csv", "MeatStats.csv")
         ]
         assert len(meatstats) == 4
         for config in LIVESTOCK_AND_MEAT_FILES.values():
@@ -233,12 +233,13 @@ def test_parse_high_plains_keeps_textual_value(self):
         assert marketing["period_sort"] == 2
         assert any(r["column"].startswith("OK City ") for r in records)
 
-    def test_afetch_table(self, monkeypatch):
-        """afetch_table fetches the zip once and reads the requested member."""
+    @pytest.mark.parametrize("member", ["Meat-Supply-Disappearance.csv", "MeatSD.csv"])
+    def test_afetch_table(self, monkeypatch, member):
+        """afetch_table reads the renamed member, or the old one."""
         calls = []
         buffer = BytesIO()
         with zipfile.ZipFile(buffer, "w") as archive:
-            archive.writestr(f"{ZIP_DIR}/MeatSD.csv", MEATSD_CSV)
+            archive.writestr(f"{ZIP_DIR}/{member}", MEATSD_CSV)
             archive.writestr(f"{ZIP_DIR}/_Read_Me.txt", "readme")
         content = buffer.getvalue()
 

openbb_platform/providers/government_us/tests/test_usda_vegetables_and_pulses.py

@@ -15,7 +15,7 @@
 )
 from openbb_government_us.usda.utils import ers_vegetables_and_pulses
 from openbb_government_us.usda.utils.ers_vegetables_and_pulses import (
-    CSV_MEMBER,
+    CSV_MEMBERS,
     DATA_ZIP,
     DEFAULT_TABLE,
     PRODUCT_PAGE,
@@ -71,7 +71,7 @@ def test_catalog_contents(self):
         assert len(TABLE_LABELS) == 81
         assert DEFAULT_TABLE in TABLE_LABELS
         assert DATA_ZIP == "/media/5018/vegetables-and-pulses-machine-readable-data.zip"
-        assert CSV_MEMBER == "Vegetables_Pulses.csv"
+        assert CSV_MEMBERS == ("vegetables-pulses-data.csv", "Vegetables_Pulses.csv")
         assert PRODUCT_PAGE.endswith("vegetables-and-pulses-yearbook-tables")
         assert all(label for label in TABLE_LABELS.values())
 
@@ -137,12 +137,15 @@ def test_parse_rows_empty_dimensions_become_none(self):
         assert records[0]["geographical_level"] is None
         assert records[0]["value"] == 50.0
 
-    def test_afetch_table(self, monkeypatch):
-        """afetch_table fetches the zip through the ERS cache and reads the member."""
+    @pytest.mark.parametrize(
+        "member", ["vegetables-pulses-data.csv", "Vegetables_Pulses.csv"]
+    )
+    def test_afetch_table(self, monkeypatch, member):
+        """afetch_table reads the renamed member, or the old one."""
         calls = []
         buffer = BytesIO()
         with zipfile.ZipFile(buffer, "w") as archive:
-            archive.writestr(CSV_MEMBER, SAMPLE_CSV)
+            archive.writestr(member, SAMPLE_CSV)
             archive.writestr("readme.txt", "notes")
         content = buffer.getvalue()