diff --git a/CHANGELOG.md b/CHANGELOG.md index baa9ec8..5dc6d45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,20 @@ This project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [0.6.5] — 2026-09-24 + ### Security - **The Census API key is no longer written to logs.** `geoinfo_from_params()` logged its request parameters at INFO after adding `key`, so the key appeared in notebook outputs (hundreds of times per call through the hierarchical geography lookup). It now logs before adding the key. Keys passed on to `morpc.req` are redacted there from morpc 0.7.5 (morpc/morpc-py#207). +### Fixed + +- **A Census API response of 204 No Content is treated as no rows** instead of failing the whole `CensusAPI` fetch. The Census API returns 204 when a request is valid but it has no data for the requested geographies; for example, ACS 5-year 2017 lists place/remainder parts (`070`) but publishes none. A warning is logged and the result has no rows for that request. Other HTTP errors still raise. + +### Changed + +- Requires morpc 0.7.5 or later, for its `HTTPError` response and API key redaction. + ## [0.6.4] — 2026-09-23 ### Fixed diff --git a/morpc_census/api.py b/morpc_census/api.py index 3a5ba24..373e8f5 100644 --- a/morpc_census/api.py +++ b/morpc_census/api.py @@ -934,6 +934,7 @@ def _fetch_variables(self, url: str, params: dict) -> pd.DataFrame: and all batch results are joined on GEO_ID into a single DataFrame. """ from morpc.req import get_json_safely + from requests import HTTPError BATCH_SIZE = 48 variables = self.variables @@ -959,7 +960,15 @@ def _fetch_variables(self, url: str, params: dict) -> pd.DataFrame: batch_params = {**params, 'get': ','.join(['GEO_ID', 'NAME'] + batch)} chunk_frames = [] for geo_chunk in geo_chunks: - records = get_json_safely(url, params={**batch_params, **geo_chunk}) + try: + records = get_json_safely(url, params={**batch_params, **geo_chunk}) + except HTTPError as e: + # 204 No Content: the request is valid but Census has no data for these geographies + # (e.g. ACS 5-year 2017 lists place/remainder parts but publishes none). + if e.response is None or e.response.status_code != 204: + raise + self.logger.warning(f"Census API returned no data for {self.name}; treating the request as no rows.") + records = [['GEO_ID', 'NAME'] + batch] columns = records.pop(0) chunk_frames.append(pd.DataFrame.from_records(records, columns=columns)) frames.append(pd.concat(chunk_frames).set_index(['GEO_ID', 'NAME'])) diff --git a/pyproject.toml b/pyproject.toml index 517dd2d..4903158 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "morpc>=0.5.4", + "morpc>=0.7.5", "numpy>=1.24", "pandas>=2.0", "geopandas>=0.14", diff --git a/reference/dev_notes.md b/reference/dev_notes.md index 2fb1357..c34e3a6 100644 --- a/reference/dev_notes.md +++ b/reference/dev_notes.md @@ -1106,3 +1106,11 @@ Tests: `tests/test_geos_hierarchical.py` — 3 new tests (scope filtering, one r **Note**: `dec/pl` does not publish 070 in any year; `dec/dhc` (2020) and `dec/sf1` (2010/2000) do. Their county subdivision totals match `dec/pl` except where parts of places that no longer exist are missing from the current-geography lookup (e.g. Hidden Lakes CDP). Tests: 1 new in `tests/test_geos_hierarchical.py`, 2 new in `tests/test_api.py`. 343 passing. Live: region15 070 returns 496 / 481 / 421 parts for 2020 / 2010 / 2000, covering all 237 MORPC-lookup 070 geographies each year (~145 s per call). + +## 2026-09-24 — Treat 204 No Content as no rows; stop logging the API key (0.6.5) + +**204**: ACS 5-year 2017 lists `place/remainder (or part)` (070) in its geography.json but returns 204 No Content for every 070 request, in both ucgid and for/in forms. Since morpc 0.7.3 that raised `HTTPError` and stopped the whole fetch (and the pop-collect notebook run). `_fetch_variables` now catches `HTTPError` whose `response.status_code` is 204 (morpc ≥ 0.7.4 attaches the response), logs a warning, and treats that request as zero rows. If every request is empty, `CensusAPI.long` is an empty frame. + +**API key**: `geoinfo_from_params()` logged its params at INFO after adding `key` (#7, fixed in #8). morpc 0.7.5 redacts `key`/`token`/`api_key` in all `morpc.req` logs and errors (morpc/morpc-py#207); morpc-census now requires it. + +Tests: 3 in `tests/test_api.py` (204 chunk skipped, all-204 returns empty frame, other errors raise) and 1 in `tests/test_geos_hierarchical.py` (key not logged). 347 passing. diff --git a/tests/test_api.py b/tests/test_api.py index 7a23dfe..6fcfae6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1108,6 +1108,45 @@ def test_pseudo_ucgid_is_not_chunked(self): mock.assert_called_once() assert mock.call_args.kwargs['params']['ucgid'] == ucgid + @staticmethod + def _no_content(): + from requests import HTTPError, Response + response = Response() + response.status_code = 204 + return HTTPError("Request failed with status 204", response=response) + + def test_no_content_chunk_is_treated_as_no_rows(self): + # The Census API answers 204 when it has no data for the requested geographies, + # e.g. ACS 5-year 2017 place/remainder parts. + api = self._make_api(1) + geoids = [f'0700000US39049{i:05d}99999' for i in range(150)] + + def respond(url, params): + chunk = params['ucgid'].split(',') + if chunk[0] == geoids[100]: + raise self._no_content() + return [['GEO_ID', 'NAME'] + api.variables] + [[g, g] + ['1'] for g in chunk] + + with patch('morpc.req.get_json_safely', side_effect=respond): + result = api._fetch_variables(api.request['url'], {'ucgid': ','.join(geoids)}) + assert sorted(result['GEO_ID']) == geoids[:100] + + def test_no_content_for_every_chunk_returns_empty_frame(self): + api = self._make_api(1) + with patch('morpc.req.get_json_safely', side_effect=self._no_content()): + result = api._fetch_variables(api.request['url'], {'ucgid': '0700000US390491800099999'}) + assert result.empty + assert list(result.columns) == ['GEO_ID', 'NAME'] + api.variables + + def test_other_http_errors_still_raise(self): + from requests import HTTPError, Response + api = self._make_api(1) + response = Response() + response.status_code = 400 + with patch('morpc.req.get_json_safely', side_effect=HTTPError("bad", response=response)): + with pytest.raises(HTTPError): + api._fetch_variables(api.request['url'], {'ucgid': '0700000US390491800099999'}) + class TestFetchDispatch: """Tests for _fetch choosing between the group() and variable-list paths."""