From eff945e8952ab87e015162c9c8561cf62a3ffd76 Mon Sep 17 00:00:00 2001 From: artcashin Date: Fri, 9 Oct 2026 19:07:51 -0400 Subject: [PATCH] fix(government_us): read USDA ERS renamed zip members and preliminary years ERS renamed the CSV members of two machine-readable zips: Vegetables_Pulses.csv -> vegetables-pulses-data.csv, and in the livestock and meat zip MeatSD.csv -> Meat-Supply-Disappearance.csv and MeatStats.csv -> Meat-Statistics.csv. Reading the old names raised KeyError for every vegetables and pulses table and five livestock tables. The readers now try the new name, then the old one, case-insensitively; the CSV columns are unchanged. The farm household income tables now publish preliminary years such as 2025P. parse_year read only the forecast suffix F, so int("2025P") failed; it now strips P too and the sort key orders P like F, after the plain year. Co-Authored-By: Claude Opus 5.5 --- ...rm_household_income_and_characteristics.py | 4 ++-- .../ers_livestock_and_meat_domestic_data.py | 20 +++++++++++++------ .../usda/utils/ers_vegetables_and_pulses.py | 9 +++++++-- ...rm_household_income_and_characteristics.py | 11 ++++++++++ ...t_usda_livestock_and_meat_domestic_data.py | 9 +++++---- .../tests/test_usda_vegetables_and_pulses.py | 13 +++++++----- 6 files changed, 47 insertions(+), 19 deletions(-) diff --git a/openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_farm_household_income_and_characteristics.py b/openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_farm_household_income_and_characteristics.py index 0203fcddee8d..fa52b8d29dfc 100644 --- a/openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_farm_household_income_and_characteristics.py +++ b/openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_farm_household_income_and_characteristics.py @@ -170,7 +170,7 @@ def parse_year(column: str) -> int: int The four-digit year as an integer. """ - return int(column.rstrip("F")) + return int(column.rstrip("FP")) def year_sort_key(column: str) -> tuple[int, bool]: @@ -187,7 +187,7 @@ def year_sort_key(column: str) -> tuple[int, bool]: The integer year and whether the header is a forecast, so a forecast year sorts after a non-forecast year of the same value. """ - return (parse_year(column), column.endswith("F")) + return (parse_year(column), column.endswith(("F", "P"))) def parse_table(text: str, table: str) -> list[dict]: diff --git a/openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_livestock_and_meat_domestic_data.py b/openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_livestock_and_meat_domestic_data.py index 7c3f2ceea5ce..9ff82a30ff58 100644 --- a/openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_livestock_and_meat_domestic_data.py +++ b/openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_livestock_and_meat_domestic_data.py @@ -15,7 +15,7 @@ LIVESTOCK_AND_MEAT_FILES: dict[str, dict] = { "supply_and_disappearance": { "label": "Meat supply and disappearance", - "member": "MeatSD.csv", + "member": ("Meat-Supply-Disappearance.csv", "MeatSD.csv"), "table_name": None, "fold_dims": (("commodity_desc", "commodity"),), "col_dims": ("attribute_desc",), @@ -25,7 +25,7 @@ }, "slaughter": { "label": "Livestock and poultry slaughter", - "member": "MeatStats.csv", + "member": ("Meat-Statistics.csv", "MeatStats.csv"), "table_name": "Livestock and poultry slaughter", "fold_dims": (("commodity_desc", "commodity"),), "col_dims": ("attribute_desc",), @@ -35,7 +35,7 @@ }, "production": { "label": "Red meat and poultry production", - "member": "MeatStats.csv", + "member": ("Meat-Statistics.csv", "MeatStats.csv"), "table_name": "Red meat and poultry production", "fold_dims": (("commodity_desc", "commodity"),), "col_dims": ("attribute_desc",), @@ -45,7 +45,7 @@ }, "live_and_dressed_weights": { "label": "Livestock and poultry live and dressed weights", - "member": "MeatStats.csv", + "member": ("Meat-Statistics.csv", "MeatStats.csv"), "table_name": "Livestock and poultry live and dressed weights", "fold_dims": (("commodity_desc", "commodity"),), "col_dims": ("attribute_desc",), @@ -55,7 +55,7 @@ }, "cold_storage": { "label": "Red meat and poultry cold storage stocks", - "member": "MeatStats.csv", + "member": ("Meat-Statistics.csv", "MeatStats.csv"), "table_name": "Red meat and poultry beginning cold storage stocks", "fold_dims": (("commodity_desc", "commodity"),), "col_dims": ("attribute_desc",), @@ -286,6 +286,14 @@ async def afetch_table(table: str, **kwargs) -> list[dict]: config = LIVESTOCK_AND_MEAT_FILES[table] content = await afetch_ers_file(ZIP_MEDIA, product=PRODUCT_PAGE) + members = config["member"] + if isinstance(members, str): + members = (members,) with zipfile.ZipFile(BytesIO(content)) as archive: - raw = archive.read(f"{ZIP_DIR}/{config['member']}") + names = {name.lower(): name for name in archive.namelist()} + member = next( + (names[key] for m in members if (key := f"{ZIP_DIR}/{m}".lower()) in names), + f"{ZIP_DIR}/{members[0]}", + ) + raw = archive.read(member) return parse_table(raw.decode("utf-8-sig", errors="replace"), table) diff --git a/openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_vegetables_and_pulses.py b/openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_vegetables_and_pulses.py index 11adc5b23111..0c16f73409c2 100644 --- a/openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_vegetables_and_pulses.py +++ b/openbb_platform/providers/government_us/openbb_government_us/usda/utils/ers_vegetables_and_pulses.py @@ -9,7 +9,7 @@ "data-products/vegetables-and-pulses-data/vegetables-and-pulses-yearbook-tables" ) DATA_ZIP = "/media/5018/vegetables-and-pulses-machine-readable-data.zip" -CSV_MEMBER = "Vegetables_Pulses.csv" +CSV_MEMBERS = ("vegetables-pulses-data.csv", "Vegetables_Pulses.csv") DEFAULT_TABLE = "Table33_Potatoes" @@ -239,5 +239,10 @@ async def afetch_table(table: str, **kwargs) -> list[dict]: content = await afetch_ers_file(DATA_ZIP, product=PRODUCT_PAGE) with zipfile.ZipFile(BytesIO(content)) as archive: - raw = archive.read(CSV_MEMBER) + names = {name.lower(): name for name in archive.namelist()} + member = next( + (names[m.lower()] for m in CSV_MEMBERS if m.lower() in names), + CSV_MEMBERS[0], + ) + raw = archive.read(member) return parse_rows(raw.decode("utf-8-sig", errors="replace"), table) diff --git a/openbb_platform/providers/government_us/tests/test_usda_farm_household_income_and_characteristics.py b/openbb_platform/providers/government_us/tests/test_usda_farm_household_income_and_characteristics.py index 9ed3645d90f4..3e2c0ff079a9 100644 --- a/openbb_platform/providers/government_us/tests/test_usda_farm_household_income_and_characteristics.py +++ b/openbb_platform/providers/government_us/tests/test_usda_farm_household_income_and_characteristics.py @@ -194,6 +194,17 @@ def test_parse_year_and_sort_key(self): "2026F", ] + def test_parse_year_preliminary_suffix(self): + """A preliminary year such as 2025P parses and sorts like a forecast.""" + assert parse_year("2025P") == 2025 + assert year_sort_key("2025P") == (2025, True) + assert sorted(["2026F", "2025P", "2025", "2024"], key=year_sort_key) == [ + "2024", + "2025", + "2025P", + "2026F", + ] + def test_parse_table_year_shape(self): """A time-series table yields item/column/value records with the year.""" records = parse_table(YEAR_CSV, "finances_2021_26f") diff --git a/openbb_platform/providers/government_us/tests/test_usda_livestock_and_meat_domestic_data.py b/openbb_platform/providers/government_us/tests/test_usda_livestock_and_meat_domestic_data.py index 8456ef7340fe..611a528f39f4 100644 --- a/openbb_platform/providers/government_us/tests/test_usda_livestock_and_meat_domestic_data.py +++ b/openbb_platform/providers/government_us/tests/test_usda_livestock_and_meat_domestic_data.py @@ -118,7 +118,7 @@ def test_catalog_contents(self): meatstats = [ key for key, config in LIVESTOCK_AND_MEAT_FILES.items() - if config["member"] == "MeatStats.csv" + if config["member"] == ("Meat-Statistics.csv", "MeatStats.csv") ] assert len(meatstats) == 4 for config in LIVESTOCK_AND_MEAT_FILES.values(): @@ -233,12 +233,13 @@ def test_parse_high_plains_keeps_textual_value(self): assert marketing["period_sort"] == 2 assert any(r["column"].startswith("OK City ") for r in records) - def test_afetch_table(self, monkeypatch): - """afetch_table fetches the zip once and reads the requested member.""" + @pytest.mark.parametrize("member", ["Meat-Supply-Disappearance.csv", "MeatSD.csv"]) + def test_afetch_table(self, monkeypatch, member): + """afetch_table reads the renamed member, or the old one.""" calls = [] buffer = BytesIO() with zipfile.ZipFile(buffer, "w") as archive: - archive.writestr(f"{ZIP_DIR}/MeatSD.csv", MEATSD_CSV) + archive.writestr(f"{ZIP_DIR}/{member}", MEATSD_CSV) archive.writestr(f"{ZIP_DIR}/_Read_Me.txt", "readme") content = buffer.getvalue() diff --git a/openbb_platform/providers/government_us/tests/test_usda_vegetables_and_pulses.py b/openbb_platform/providers/government_us/tests/test_usda_vegetables_and_pulses.py index 7c45b6e87229..d28ee28ddb88 100644 --- a/openbb_platform/providers/government_us/tests/test_usda_vegetables_and_pulses.py +++ b/openbb_platform/providers/government_us/tests/test_usda_vegetables_and_pulses.py @@ -15,7 +15,7 @@ ) from openbb_government_us.usda.utils import ers_vegetables_and_pulses from openbb_government_us.usda.utils.ers_vegetables_and_pulses import ( - CSV_MEMBER, + CSV_MEMBERS, DATA_ZIP, DEFAULT_TABLE, PRODUCT_PAGE, @@ -71,7 +71,7 @@ def test_catalog_contents(self): assert len(TABLE_LABELS) == 81 assert DEFAULT_TABLE in TABLE_LABELS assert DATA_ZIP == "/media/5018/vegetables-and-pulses-machine-readable-data.zip" - assert CSV_MEMBER == "Vegetables_Pulses.csv" + assert CSV_MEMBERS == ("vegetables-pulses-data.csv", "Vegetables_Pulses.csv") assert PRODUCT_PAGE.endswith("vegetables-and-pulses-yearbook-tables") assert all(label for label in TABLE_LABELS.values()) @@ -137,12 +137,15 @@ def test_parse_rows_empty_dimensions_become_none(self): assert records[0]["geographical_level"] is None assert records[0]["value"] == 50.0 - def test_afetch_table(self, monkeypatch): - """afetch_table fetches the zip through the ERS cache and reads the member.""" + @pytest.mark.parametrize( + "member", ["vegetables-pulses-data.csv", "Vegetables_Pulses.csv"] + ) + def test_afetch_table(self, monkeypatch, member): + """afetch_table reads the renamed member, or the old one.""" calls = [] buffer = BytesIO() with zipfile.ZipFile(buffer, "w") as archive: - archive.writestr(CSV_MEMBER, SAMPLE_CSV) + archive.writestr(member, SAMPLE_CSV) archive.writestr("readme.txt", "notes") content = buffer.getvalue()