From d09c563d421ad4bef60c46e451aa4b95f9772a28 Mon Sep 17 00:00:00 2001 From: Gniewkob Date: Thu, 12 Nov 2026 23:56:51 +0100 Subject: [PATCH 1/5] research: probe original public Eurostat Polish government bond yield JSON-stat shape (#152) Original source and content hash only in bounded one-shot CI; zero model admission, no account or real financial paths. --- .../isl-eurostat-pl-yield-probe-v04.yml | 49 ++++++++ ...eurostat_pl_bond_yield_source_probe_v04.py | 117 ++++++++++++++++++ ...eurostat_pl_bond_yield_source_probe_v04.py | 62 ++++++++++ 3 files changed, 228 insertions(+) create mode 100644 .github/workflows/isl-eurostat-pl-yield-probe-v04.yml create mode 100644 scripts/eurostat_pl_bond_yield_source_probe_v04.py create mode 100644 tests/test_eurostat_pl_bond_yield_source_probe_v04.py diff --git a/.github/workflows/isl-eurostat-pl-yield-probe-v04.yml b/.github/workflows/isl-eurostat-pl-yield-probe-v04.yml new file mode 100644 index 00000000..b8a956d3 --- /dev/null +++ b/.github/workflows/isl-eurostat-pl-yield-probe-v04.yml @@ -0,0 +1,49 @@ +name: ISL v0.4 original official Eurostat PL bond yield source probe +on: + pull_request: + paths: + - 'scripts/eurostat_pl_bond_yield_source_probe_v04.py' + - 'tests/test_eurostat_pl_bond_yield_source_probe_v04.py' + - '.github/workflows/isl-eurostat-pl-yield-probe-v04.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + offline-contract: + runs-on: ubuntu-24.04 + timeout-minutes: 6 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: No-network schema-probe unit tests + run: | + python - <<'PY' + import socket, unittest + def deny(*args, **kwargs): + raise RuntimeError("UNEXPECTED_OFFLINE_NETWORK") + socket.socket.connect = deny + socket.socket.connect_ex = deny + socket.create_connection = deny + suite = unittest.defaultTestLoader.loadTestsFromName( + "tests.test_eurostat_pl_bond_yield_source_probe_v04") + result = unittest.TextTestRunner(verbosity=2).run(suite) + if not result.wasSuccessful() or result.skipped or result.testsRun != 5: + raise SystemExit("EUROSTAT_SOURCE_PROBE_UNIT_FAILURE") + PY + official-source-shape: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Official original public Eurostat data, schema-only stdout + run: | + python -m scripts.eurostat_pl_bond_yield_source_probe_v04 --allow-public-eurostat-read diff --git a/scripts/eurostat_pl_bond_yield_source_probe_v04.py b/scripts/eurostat_pl_bond_yield_source_probe_v04.py new file mode 100644 index 00000000..0c306836 --- /dev/null +++ b/scripts/eurostat_pl_bond_yield_source_probe_v04.py @@ -0,0 +1,117 @@ +"""Official Eurostat monthly Polish sovereign convergence-yield source shape probe. + +No model data, account access, trades, databases or local files. One-shot +explicit opt-in. Logs *schema only*, never values, yields or series. +""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +from hashlib import sha256 +import json +from urllib.parse import urlencode, urlsplit +from urllib.request import HTTPRedirectHandler, HTTPSHandler, Request, build_opener + +ROOT = "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/irt_lt_mcby_m" +ARGS = ( + ("lang", "EN"), ("freq", "M"), ("int_rt", "MCBY"), ("geo", "PL"), + ("sinceTimePeriod", "2021-11"), ("untilTimePeriod", "2026-09"), +) +URL = ROOT + "?" + urlencode(ARGS) +MAX_BYTES = 600_000 + + +class PublicEurostatProbeBlocked(ValueError): + pass + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, request, fp, code, msg, headers, newurl): + raise PublicEurostatProbeBlocked("REDIRECT_REJECTED") + + +def fetch_official() -> bytes: + """One official allowlisted HTTPS request, never redirect to third party.""" + req = Request(URL, headers={"Accept": "application/json", + "User-Agent": "ISL-edu-public-source/1"}) + opener = build_opener(NoRedirect(), HTTPSHandler()) + with opener.open(req, timeout=25) as response: + if response.status != 200 or response.geturl() != URL: + raise PublicEurostatProbeBlocked("OFFICIAL_RESPONSE_STATUS_OR_URL_CHANGED") + content = response.read(MAX_BYTES + 1) + if not content or len(content) > MAX_BYTES: + raise PublicEurostatProbeBlocked("SOURCE_RESPONSE_OUT_OF_BOUNDS") + return content + + +def inspect_source_shape(raw: object, *, observed_at: datetime) -> dict: + if type(raw) is not bytes or not raw or len(raw) > MAX_BYTES: + raise PublicEurostatProbeBlocked("SOURCE_RESPONSE_OUT_OF_BOUNDS") + if (not isinstance(observed_at, datetime) or observed_at.tzinfo is None + or observed_at.utcoffset() is None): + raise PublicEurostatProbeBlocked("UTC_AWARE_OBSERVATION_CLOCK_REQUIRED") + try: + data = json.loads(raw) + except (UnicodeError, json.JSONDecodeError) as exc: + raise PublicEurostatProbeBlocked("NOT_OFFICIAL_JSON") from exc + if (not isinstance(data, dict) or data.get("version") != "2.0" + or data.get("class") != "dataset" + or data.get("source") not in ("ESTAT", "Eurostat") + or not isinstance(data.get("id"), list) + or not isinstance(data.get("size"), list) + or not isinstance(data.get("dimension"), dict)): + raise PublicEurostatProbeBlocked("JSONSTAT_STRUCTURE_NOT_MATCHED") + dims = data["dimension"] + names = data["id"] + if (not 1 <= len(names) <= 7 or len(names) != len(set(names)) + or set(names) != set(dims) or len(data["size"]) != len(names)): + raise PublicEurostatProbeBlocked("DIMENSIONS_INCOHERENT") + codes = {} + for name, count in zip(names, data["size"]): + if (not isinstance(name, str) or not name.isidentifier() + or type(count) is not int or not 1 <= count <= 120 + or not isinstance(dims[name], dict)): + raise PublicEurostatProbeBlocked("DIMENSION_INVALID") + cat = dims[name].get("category") + ix = cat.get("index") if isinstance(cat, dict) else None + if not isinstance(ix, dict) or len(ix) != count: + raise PublicEurostatProbeBlocked("CATEGORIES_INVALID") + codes[name] = sorted(str(k) for k in ix) + # No raw data or complete time series is ever printed/returned. + return { + "schema": "ISL_PUBLIC_EUROSTAT_PL_BONDS_YIELD_SHAPE_PROBE_V04", + "source": "Eurostat", + "dataset": "irt_lt_mcby_m", + "query": URL, + "source_bytes_sha256": sha256(raw).hexdigest(), + "observed_by_runner_at_utc": observed_at.astimezone(timezone.utc).isoformat(), + "dimensions": names, + "sizes": data["size"], + "fixed_category_codes": {k: v for k, v in codes.items() if k != "time"}, + "time_first": codes["time"][0] if "time" in codes else None, + "time_last": codes["time"][-1] if "time" in codes else None, + "time_count": len(codes.get("time", [])), + "provider_updated_not_first_release": str(data.get("updated", "UNKNOWN"))[:50], + "actual_rate_values_in_output": False, + "historical_pit_certified": False, + "bond_holding_period_total_return": False, + "qualified_model_series": [], + "financial_operations": 0, + "database_writes": 0, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--allow-public-eurostat-read", action="store_true") + args = parser.parse_args() + if not args.allow_public_eurostat_read: + parser.error("explicit --allow-public-eurostat-read required") + result = inspect_source_shape(fetch_official(), + observed_at=datetime.now(timezone.utc)) + print("ISL_PUBLIC_EUROSTAT_PL_YIELD_SHAPE=" + + json.dumps(result, separators=(",", ":"), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_eurostat_pl_bond_yield_source_probe_v04.py b/tests/test_eurostat_pl_bond_yield_source_probe_v04.py new file mode 100644 index 00000000..7966ccc0 --- /dev/null +++ b/tests/test_eurostat_pl_bond_yield_source_probe_v04.py @@ -0,0 +1,62 @@ +"""Schema-only probe contract: no implicit network or investment operations.""" +import json +import unittest +from datetime import datetime, timezone +from scripts.eurostat_pl_bond_yield_source_probe_v04 import ( + inspect_source_shape, PublicEurostatProbeBlocked, URL, MAX_BYTES, +) + +CLOCK = datetime(2026, 11, 12, tzinfo=timezone.utc) + + +def sample(): + return { + "version": "2.0", "class": "dataset", "source": "ESTAT", + "id": ["freq", "int_rt", "geo", "time"], + "size": [1, 1, 1, 2], + "dimension": { + "freq": {"category": {"index": {"M": 0}}}, + "int_rt": {"category": {"index": {"MCBY": 0}}}, + "geo": {"category": {"index": {"PL": 0}}}, + "time": {"category": {"index": {"2026-08": 0, "2026-09": 1}}}, + }, + "value": {"0": 5.0, "1": 5.1}, + } + + +class EurostatPLBondYieldProbeTests(unittest.TestCase): + def test_source_metadata_no_raw_values(self): + out = inspect_source_shape(json.dumps(sample()).encode(), observed_at=CLOCK) + self.assertEqual(out["time_count"], 2) + self.assertEqual(out["fixed_category_codes"]["int_rt"], ["MCBY"]) + self.assertNotIn("5.0", json.dumps(out)) + self.assertFalse(out["bond_holding_period_total_return"]) + self.assertFalse(out["historical_pit_certified"]) + self.assertEqual(out["qualified_model_series"], []) + self.assertEqual(out["financial_operations"], 0) + + def test_only_official_fixed_query(self): + self.assertTrue(URL.startswith( + "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/irt_lt_mcby_m?")) + self.assertIn("geo=PL", URL) + self.assertIn("int_rt=MCBY", URL) + + def test_invalid_payload_and_oversize_blocked(self): + for raw in (b"", b"bad", b"{}", b"x" * (MAX_BYTES + 1)): + with self.subTest(size=len(raw)), self.assertRaises(PublicEurostatProbeBlocked): + inspect_source_shape(raw, observed_at=CLOCK) + + def test_naive_clock_blocked(self): + with self.assertRaisesRegex(PublicEurostatProbeBlocked, "CLOCK_REQUIRED"): + inspect_source_shape(json.dumps(sample()).encode(), + observed_at=datetime(2026, 11, 12)) + + def test_mismatched_dimensions_blocked(self): + p = sample() + p["size"] = [1, 1, 1, 3] + with self.assertRaisesRegex(PublicEurostatProbeBlocked, "CATEGORIES_INVALID"): + inspect_source_shape(json.dumps(p).encode(), observed_at=CLOCK) + + +if __name__ == "__main__": + unittest.main() From b6092063f473b63610c87215d89786ef00919ff4 Mon Sep 17 00:00:00 2001 From: Gniewkob Date: Thu, 12 Nov 2026 23:59:01 +0100 Subject: [PATCH 2/5] research: strict 59-month original Eurostat Polish sovereign yield parser after live source shape verification (#152) --- ...at_pl_government_bond_yield_context_v04.py | 219 ++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 scripts/eurostat_pl_government_bond_yield_context_v04.py diff --git a/scripts/eurostat_pl_government_bond_yield_context_v04.py b/scripts/eurostat_pl_government_bond_yield_context_v04.py new file mode 100644 index 00000000..fc6f47aa --- /dev/null +++ b/scripts/eurostat_pl_government_bond_yield_context_v04.py @@ -0,0 +1,219 @@ +"""Polish sovereign 10Y-ish bond YIELD context from original Eurostat JSON-stat. + +The Maastricht convergence series is a % p.a. interest rate (not holding- +period bond return, not TBSP, coupon data or an executable bond price). +A present-day official dataset is NOT proof of past first-publication versions. +This is read-only research context, never a trading or portfolio input. +""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone, date +from decimal import Decimal, InvalidOperation, localcontext +from hashlib import sha256 +import json +import re + +from scripts.eurostat_pl_bond_yield_source_probe_v04 import ( + fetch_official, URL, MAX_BYTES, +) +from scripts.public_first_prospective_source_observation_v03 import ( + _provider_update, +) + +SCHEMA = "ISL_EUROSTAT_PL_GOV_BOND_YIELD_CONTEXT_V04" +SINCE = "2021-11" +UNTIL = "2026-09" +MAX_MONTHS = 59 +REQUIRED = {"freq": "M", "int_rt": "MCBY", "geo": "PL"} +DIMS = set(REQUIRED) | {"time"} +MONTH = re.compile(r"^20\d{2}-(0[1-9]|1[0-2])$") + + +class PLYieldContextBlocked(ValueError): + """Unsupported, incomplete or semantically incompatible source evidence.""" + + +def _canon(data: object) -> bytes: + return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() + + +def _aware_utc(value: object) -> datetime: + if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None: + raise PLYieldContextBlocked("AWARE_UTC_OBSERVATION_CLOCK_REQUIRED") + return value.astimezone(timezone.utc) + + +def _month(value: object) -> date: + if not isinstance(value, str) or not MONTH.fullmatch(value): + raise PLYieldContextBlocked("SOURCE_MONTH_INVALID") + try: + return date.fromisoformat(value + "-01") + except ValueError as exc: + raise PLYieldContextBlocked("SOURCE_MONTH_INVALID") from exc + + +def _next_month(d: date) -> date: + return date(d.year + (d.month == 12), 1 if d.month == 12 else d.month + 1, 1) + + +def _indices(data: dict) -> tuple[list[str], list[int], dict[str, dict[str, int]]]: + ids, sizes, dims = data.get("id"), data.get("size"), data.get("dimension") + if (not isinstance(ids, list) or len(ids) != len(DIMS) + or set(ids) != DIMS or len(set(ids)) != len(ids) + or not isinstance(sizes, list) or len(sizes) != len(ids) + or not isinstance(dims, dict) or set(dims) != DIMS): + raise PLYieldContextBlocked("OFFICIAL_JSONSTAT_DIMENSION_CONFLICT") + categories = {} + for name, count in zip(ids, sizes): + if type(count) is not int or not 1 <= count <= MAX_MONTHS: + raise PLYieldContextBlocked("JSONSTAT_DIMENSION_SIZE_INVALID") + dim = dims[name] + cat = dim.get("category") if isinstance(dim, dict) else None + ix = cat.get("index") if isinstance(cat, dict) else None + if (not isinstance(ix, dict) or len(ix) != count + or any(type(v) is not int for v in ix.values()) + or set(ix.values()) != set(range(count))): + raise PLYieldContextBlocked("JSONSTAT_CATEGORY_POSITIONS_INVALID") + categories[name] = ix + if name in REQUIRED and ix != {REQUIRED[name]: 0}: + raise PLYieldContextBlocked("WRONG_COUNTRY_RATE_OR_FREQUENCY") + return ids, sizes, categories + + +def parse_original_yields(raw: bytes, *, observed_at: datetime) -> dict: + """Parse one original raw Eurostat response; output aggregates, not rows. + + 'observed_at' records our runner's completed HTTPS read or local fixture + processing time. It is never original historic first-publication time. + """ + observed_at = _aware_utc(observed_at) + if type(raw) is not bytes or not raw or len(raw) > MAX_BYTES: + raise PLYieldContextBlocked("SOURCE_BYTES_OUT_OF_BOUNDS") + try: + data = json.loads(raw) + except (UnicodeError, json.JSONDecodeError) as exc: + raise PLYieldContextBlocked("SOURCE_INVALID_JSON") from exc + if (not isinstance(data, dict) or data.get("version") != "2.0" + or data.get("class") != "dataset" + or data.get("source") not in ("ESTAT", "Eurostat")): + raise PLYieldContextBlocked("UNSUPPORTED_OFFICIAL_JSONSTAT_FORMAT") + ids, sizes, cats = _indices(data) + ordered = sorted(cats["time"].items(), key=lambda it: it[1]) + if (len(ordered) != MAX_MONTHS or ordered[0][0] != SINCE + or ordered[-1][0] != UNTIL + or _month(ordered[-1][0]) > observed_at.date().replace(day=1)): + raise PLYieldContextBlocked("INCOMPLETE_OR_FUTURE_SOURCE_WINDOW") + for idx, (month, pos) in enumerate(ordered): + if pos != idx or _month(month) != ( + _month(SINCE) if idx == 0 else _next_month(_month(ordered[idx-1][0]))): + raise PLYieldContextBlocked("MISSING_OR_REORDERED_MONTHS") + + values = data.get("value") + statuses = data.get("status", {}) + cells = 1 + for size in sizes: + cells *= size + if (not isinstance(values, (dict, list)) or type(values) is bool + or not isinstance(statuses, dict) + or isinstance(values, list) and len(values) != cells + or isinstance(values, dict) and any( + not isinstance(k, str) or not k.isdigit() + or not 0 <= int(k) < cells for k in values) + or any(not isinstance(k, str) or not k.isdigit() + or not 0 <= int(k) < cells for k in statuses)): + raise PLYieldContextBlocked("SOURCE_VALUE_VECTOR_MALFORMED") + rates = [] + flags = {} + for month, pos in ordered: + coords = [pos if k == "time" else 0 for k in ids] + offset = 0 + for coordinate, count in zip(coords, sizes): + offset = offset * count + coordinate + cell = values[offset] if isinstance(values, list) else values.get(str(offset)) + if (type(cell) not in (float, int, str) or cell is None + or isinstance(cell, str) and len(cell) > 32): + raise PLYieldContextBlocked("MISSING_OR_UNEXPECTED_YIELD") + try: + rate = Decimal(str(cell)) + except (InvalidOperation, ValueError) as exc: + raise PLYieldContextBlocked("INVALID_PERCENT_PER_ANNUM") from exc + # Negative Eurozone yields can occur, do not reject negative rates + # automatically; bound the official PL series against unit mistakes. + if not rate.is_finite() or not Decimal("-20") <= rate <= Decimal("50"): + raise PLYieldContextBlocked("YIELD_UNITS_OR_NUMERIC_RANGE_INVALID") + status = statuses.get(str(offset)) + if status is not None: + if type(status) is not str or len(status) > 12 or not status.isalnum(): + raise PLYieldContextBlocked("SOURCE_STATISTICAL_FLAG_INVALID") + flags[status] = flags.get(status, 0) + 1 + rates.append(rate) + # This field is the provider's most recent dataset update, NOT the first + # release timestamp of any constituent monthly observation. + try: + provider_update = _provider_update( + data.get("updated", "NOT_PROVIDED"), observed_at) + except ValueError as exc: + raise PLYieldContextBlocked("PROVIDER_TIMESTAMP_INCONSISTENT") from exc + with localcontext() as ctx: + ctx.prec = 36 + avg = sum(rates, Decimal(0)) / len(rates) + max_change_bps = max(abs((b-a) * Decimal(100)) + for a,b in zip(rates, rates[1:])) + out = { + "schema": SCHEMA, + "status": "PUBLIC_OFFICIAL_YIELD_CONTEXT_ONLY", + "source": "Eurostat", + "dataset": "irt_lt_mcby_m", + "original_source_url": URL, + "original_response_sha256": sha256(raw).hexdigest(), + "observed_by_runner_at_utc": observed_at.isoformat(), + "provider_reported_updated_not_first_release": provider_update, + "source_attribution": "Eurostat/ECB/National Central Banks: irt_lt_mcby_m", + "country": "PL", + "frequency": "M", + "interest_rate_code": "MCBY", + "source_unit": "PERCENT_PER_ANNUM", + "data_kind": "GOVERNMENT_BOND_CONVERGENCE_CRITERION_YIELD_NOT_RETURN", + "residual_maturity": "APPROXIMATELY_TEN_YEARS", + "first_month": SINCE, + "last_month": UNTIL, + "observed_months": len(rates), + "min_yield_pct_per_annum": str(min(rates)), + "max_yield_pct_per_annum": str(max(rates)), + "mean_yield_pct_per_annum": format(avg.quantize(Decimal("0.0001")), "f"), + "max_abs_adjacent_month_level_change_bps": format( + max_change_bps.quantize(Decimal("0.01")), "f"), + "source_statistical_flags_count": flags, + "present_snapshot_not_original_historic_vintages": True, + "historical_first_available_at_verified": False, + "historical_pit_certified": False, + "bond_holding_period_return_observed": False, + "tbsp_income_index_observed": False, + "estimated_coupon_reinvestment": False, + "qualified_model_series": [], + "legacy_m1_m2_m3": "INSUFFICIENT_DATA_HOLD_CASH", + "orders_sent": 0, + "financial_operations": 0, + "database_writes": 0, + "paper_events_created": 0, + } + out["receipt_sha256"] = sha256(_canon(out)).hexdigest() + return out + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--allow-public-eurostat-read", action="store_true") + args = parser.parse_args() + if not args.allow_public_eurostat_read: + parser.error("Explicit --allow-public-eurostat-read required") + raw = fetch_official() + receipt = parse_original_yields(raw, observed_at=datetime.now(timezone.utc)) + # No observations are printed, no data copied to files or GitHub artifacts. + print("ISL_PUBLIC_EUROSTAT_PL_YIELD_RECEIPT=" + + json.dumps(receipt, sort_keys=True, separators=(",", ":"))) + + +if __name__ == "__main__": + main() From eb064c5d9db18d7cdf2e9348b88cc211006cc3e4 Mon Sep 17 00:00:00 2001 From: Gniewkob Date: Thu, 12 Nov 2026 23:59:58 +0100 Subject: [PATCH 3/5] test: 12 strict source-typed Eurostat Polish yield-only negative cases, no network (#152) --- ...at_pl_government_bond_yield_context_v04.py | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 tests/test_eurostat_pl_government_bond_yield_context_v04.py diff --git a/tests/test_eurostat_pl_government_bond_yield_context_v04.py b/tests/test_eurostat_pl_government_bond_yield_context_v04.py new file mode 100644 index 00000000..0539fa58 --- /dev/null +++ b/tests/test_eurostat_pl_government_bond_yield_context_v04.py @@ -0,0 +1,166 @@ +"""Original-shape Eurostat MCBY/PL yield reference; offline/negative acceptance.""" +from __future__ import annotations + +from copy import deepcopy +from datetime import datetime, timezone +import json +import unittest +from unittest.mock import patch + +from scripts.eurostat_pl_government_bond_yield_context_v04 import ( + PLYieldContextBlocked, parse_original_yields, SINCE, UNTIL, MAX_MONTHS, +) + +OBSERVED = datetime(2026, 11, 12, 21, 30, tzinfo=timezone.utc) + + +def months(): + y, m = 2021, 11 + out = [] + for _ in range(MAX_MONTHS): + out.append(f"{y:04d}-{m:02d}") + y, m = (y + 1, 1) if m == 12 else (y, m + 1) + return out + + +def original_shape(*, dims=("freq", "int_rt", "geo", "time")): + calendar = months() + assert calendar[0] == SINCE and calendar[-1] == UNTIL + catalog = {"freq": {"M": 0}, "int_rt": {"MCBY": 0}, + "geo": {"PL": 0}, "time": {date: i for i, date in enumerate(calendar)}} + return { + "version": "2.0", "class": "dataset", "source": "ESTAT", + "updated": "2026-11-12T11:00:00+0100", + "id": list(dims), + "size": [len(catalog[x]) for x in dims], + "dimension": {k: {"category": {"index": catalog[k]}} for k in dims}, + "value": {str(i): round(4 + i * 0.01, 4) for i in range(MAX_MONTHS)}, + "status": {"1": "p"}, + } + + +def parse(data=None, *, at=OBSERVED): + return parse_original_yields( + json.dumps(data if data is not None else original_shape()).encode(), + observed_at=at) + + +class PLYieldContextTests(unittest.TestCase): + def test_original_shape_59_months_yield_semantics_provenance(self): + r = parse() + self.assertEqual(r["observed_months"], 59) + self.assertEqual((r["first_month"], r["last_month"]), (SINCE, UNTIL)) + self.assertEqual(r["source_unit"], "PERCENT_PER_ANNUM") + self.assertEqual(r["interest_rate_code"], "MCBY") + self.assertEqual(r["country"], "PL") + self.assertEqual(r["min_yield_pct_per_annum"], "4.0") + self.assertEqual(r["max_yield_pct_per_annum"], "4.58") + self.assertEqual(r["mean_yield_pct_per_annum"], "4.2900") + self.assertEqual(r["max_abs_adjacent_month_level_change_bps"], "1.00") + self.assertEqual(r["source_statistical_flags_count"], {"p": 1}) + self.assertRegex(r["original_response_sha256"], r"^[a-f0-9]{64}$") + self.assertRegex(r["receipt_sha256"], r"^[a-f0-9]{64}$") + self.assertFalse(r["historical_pit_certified"]) + self.assertFalse(r["bond_holding_period_return_observed"]) + self.assertFalse(r["tbsp_income_index_observed"]) + self.assertEqual(r["qualified_model_series"], []) + self.assertEqual(r["legacy_m1_m2_m3"], "INSUFFICIENT_DATA_HOLD_CASH") + self.assertEqual(r["financial_operations"], 0) + self.assertEqual(r["database_writes"], 0) + self.assertEqual(r["orders_sent"], 0) + self.assertNotIn('"observations"', json.dumps(r)) + self.assertNotIn('"value":', json.dumps(r)) + + def test_source_current_vintage_not_historical_first_release(self): + d = original_shape() + d["updated"] = "2026-11-12T11:00:00+0100" + r = parse(d) + self.assertTrue(r["present_snapshot_not_original_historic_vintages"]) + self.assertFalse(r["historical_first_available_at_verified"]) + self.assertEqual(r["provider_reported_updated_not_first_release"], d["updated"]) + + def test_deterministic_same_original_bytes_except_fetch_time(self): + self.assertEqual(parse(), parse()) + d = original_shape() + d["status"]["1"] = "e" + self.assertNotEqual(parse(d)["original_response_sha256"], + parse()["original_response_sha256"]) + + def test_dimension_order_independent(self): + d = original_shape(dims=("time", "geo", "freq", "int_rt")) + self.assertEqual(parse(d)["mean_yield_pct_per_annum"], "4.2900") + + def test_missing_month_and_index_hole_fail_closed(self): + d = original_shape() + d["dimension"]["time"]["category"]["index"].pop("2025-01") + d["size"][-1] -= 1 + with self.assertRaises(PLYieldContextBlocked): + parse(d) + d = original_shape() + d["dimension"]["time"]["category"]["index"]["2024-11"] = 900 + with self.assertRaises(PLYieldContextBlocked): + parse(d) + + def test_sparse_missing_and_invalid_index_fail_closed(self): + d = original_shape() + d["value"].pop("8") + with self.assertRaisesRegex(PLYieldContextBlocked, "MISSING"): + parse(d) + d = original_shape() + d["value"]["9999"] = 8 + with self.assertRaisesRegex(PLYieldContextBlocked, "VECTOR"): + parse(d) + + def test_wrong_geo_indicator_frequency_and_source_fail_closed(self): + for key, bad in (("geo", "DE"), ("int_rt", "OTHER"), ("freq", "A")): + d = original_shape() + d["dimension"][key]["category"]["index"] = {bad: 0} + with self.subTest(key=key), self.assertRaisesRegex( + PLYieldContextBlocked, "WRONG_COUNTRY"): + parse(d) + d = original_shape() + d["source"] = "MIRROR" + with self.assertRaisesRegex(PLYieldContextBlocked, "UNSUPPORTED_OFFICIAL"): + parse(d) + + def test_false_values_rejected_not_silently_zero_filled(self): + for bad in (None, True, float("nan"), float("inf"), "NA", 95, -30): + d = original_shape() + d["value"]["5"] = bad + with self.subTest(bad=str(bad)), self.assertRaises(PLYieldContextBlocked): + parse(d) + + def test_status_flag_validation(self): + d = original_shape() + d["status"]["1"] = "not-a-valid-statistic-flag" + with self.assertRaises(PLYieldContextBlocked): + parse(d) + d = original_shape() + d["status"]["10000"] = "p" + with self.assertRaisesRegex(PLYieldContextBlocked, "VECTOR"): + parse(d) + + def test_future_month_time_and_provider_update_rejected(self): + with self.assertRaisesRegex(PLYieldContextBlocked, "FUTURE"): + parse(at=datetime(2025, 1, 1, tzinfo=timezone.utc)) + d = original_shape() + d["updated"] = "2030-01-01T00:00:00Z" + with self.assertRaisesRegex(PLYieldContextBlocked, "PROVIDER_TIMESTAMP"): + parse(d) + with self.assertRaisesRegex(PLYieldContextBlocked, "CLOCK_REQUIRED"): + parse(at=datetime(2026, 11, 12)) + + def test_wrong_base_jsonstat_and_oversize_rejected(self): + for raw in (b"", b"not-json", b"{}", b"x" * 600001): + with self.subTest(size=len(raw)), self.assertRaises(PLYieldContextBlocked): + parse_original_yields(raw, observed_at=OBSERVED) + + def test_no_network_or_model_operation_in_parser(self): + def deny(*args, **kwargs): + raise AssertionError("Parser attempted to use network") + with patch("socket.socket.connect", side_effect=deny): + self.assertEqual(parse()["financial_operations"], 0) + + +if __name__ == "__main__": + unittest.main() From 16a54ac0ca6ab71b0388aeb47e04b31fc85f3d0f Mon Sep 17 00:00:00 2001 From: Gniewkob Date: Fri, 13 Nov 2026 00:01:19 +0100 Subject: [PATCH 4/5] ci/docs: original PL Eurostat yield aggregates and 17 offline cross-platform tests (#152) Source documented with exact ECB/Eurostat methodology and license caveats; trigger full regression and CodeQL. --- .../full-agent-staging-canary-ci.yml | 4 ++ .../isl-eurostat-pl-yield-probe-v04.yml | 62 +++++++++++++++---- ...isl-public-first-prospective-codeql-ci.yml | 2 + ...VEREIGN_YIELD_REFERENCE_V04_20261112_PL.md | 30 +++++++++ 4 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 docs/ISL_EUROSTAT_PL_SOVEREIGN_YIELD_REFERENCE_V04_20261112_PL.md diff --git a/.github/workflows/full-agent-staging-canary-ci.yml b/.github/workflows/full-agent-staging-canary-ci.yml index 9684a5ee..814ef2dd 100644 --- a/.github/workflows/full-agent-staging-canary-ci.yml +++ b/.github/workflows/full-agent-staging-canary-ci.yml @@ -24,6 +24,8 @@ on: - tests/test_shadow_offsite_export_v13.py - scripts/public_first_hicp_nbp_purchasing_power_v01.py - tests/test_public_first_hicp_nbp_purchasing_power_v01.py + - scripts/eurostat_pl_*.py + - tests/test_eurostat_pl_*.py - scripts/public_first_prospective_source_observation_v03.py - tests/test_public_first_prospective_source_observation_v03.py - scripts/public_first_nbp_reference_research_v01.py @@ -53,6 +55,8 @@ on: - tests/test_shadow_offsite_export_v13.py - scripts/public_first_hicp_nbp_purchasing_power_v01.py - tests/test_public_first_hicp_nbp_purchasing_power_v01.py + - scripts/eurostat_pl_*.py + - tests/test_eurostat_pl_*.py - scripts/public_first_prospective_source_observation_v03.py - tests/test_public_first_prospective_source_observation_v03.py - scripts/public_first_nbp_reference_research_v01.py diff --git a/.github/workflows/isl-eurostat-pl-yield-probe-v04.yml b/.github/workflows/isl-eurostat-pl-yield-probe-v04.yml index b8a956d3..3a091915 100644 --- a/.github/workflows/isl-eurostat-pl-yield-probe-v04.yml +++ b/.github/workflows/isl-eurostat-pl-yield-probe-v04.yml @@ -1,17 +1,26 @@ -name: ISL v0.4 original official Eurostat PL bond yield source probe +name: ISL v0.4 original Eurostat PL sovereign yield reference +# Public and official Eurostat, source-typed, no paid APIs, no credentials, +# no financial operations and no automatic schedules. on: pull_request: paths: - 'scripts/eurostat_pl_bond_yield_source_probe_v04.py' + - 'scripts/eurostat_pl_government_bond_yield_context_v04.py' - 'tests/test_eurostat_pl_bond_yield_source_probe_v04.py' + - 'tests/test_eurostat_pl_government_bond_yield_context_v04.py' + - 'docs/ISL_EUROSTAT_PL_SOVEREIGN_YIELD_REFERENCE_V04_20261112_PL.md' - '.github/workflows/isl-eurostat-pl-yield-probe-v04.yml' workflow_dispatch: permissions: contents: read jobs: offline-contract: - runs-on: ubuntu-24.04 - timeout-minutes: 6 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + runs-on: ${{ matrix.os }} + timeout-minutes: 8 steps: - uses: actions/checkout@v4 with: @@ -19,8 +28,9 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' - - name: No-network schema-probe unit tests + - name: No-network original-source source shape and typed yield cases run: | + set -euo pipefail python - <<'PY' import socket, unittest def deny(*args, **kwargs): @@ -28,13 +38,17 @@ jobs: socket.socket.connect = deny socket.socket.connect_ex = deny socket.create_connection = deny - suite = unittest.defaultTestLoader.loadTestsFromName( - "tests.test_eurostat_pl_bond_yield_source_probe_v04") + suite = unittest.TestSuite() + for name in ( + "tests.test_eurostat_pl_bond_yield_source_probe_v04", + "tests.test_eurostat_pl_government_bond_yield_context_v04", + ): + suite.addTests(unittest.defaultTestLoader.loadTestsFromName(name)) result = unittest.TextTestRunner(verbosity=2).run(suite) - if not result.wasSuccessful() or result.skipped or result.testsRun != 5: - raise SystemExit("EUROSTAT_SOURCE_PROBE_UNIT_FAILURE") + if not result.wasSuccessful() or result.skipped or result.testsRun != 17: + raise SystemExit("EUROSTAT_PL_YIELD_SOURCE_CONTRACT_FAILURE") PY - official-source-shape: + official-original-source-evidence: runs-on: ubuntu-24.04 timeout-minutes: 10 steps: @@ -44,6 +58,32 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' - - name: Official original public Eurostat data, schema-only stdout + - name: One official Eurostat HTTPS GET; print schema + yield aggregates only run: | - python -m scripts.eurostat_pl_bond_yield_source_probe_v04 --allow-public-eurostat-read + set -euo pipefail + python - <<'PY' + from datetime import datetime, timezone + import json + from scripts.eurostat_pl_bond_yield_source_probe_v04 import ( + fetch_official, inspect_source_shape, + ) + from scripts.eurostat_pl_government_bond_yield_context_v04 import ( + parse_original_yields, + ) + raw = fetch_official() + observed_at = datetime.now(timezone.utc) + shape = inspect_source_shape(raw, observed_at=observed_at) + receipt = parse_original_yields(raw, observed_at=observed_at) + if (shape["source_bytes_sha256"] != receipt["original_response_sha256"] + or shape["time_count"] != receipt["observed_months"] + or receipt["observed_months"] != 59 + or receipt["historical_pit_certified"] is not False + or receipt["qualified_model_series"] != [] + or receipt["orders_sent"] != 0 + or receipt["financial_operations"] != 0): + raise SystemExit("ORIGINAL_PL_EUROSTAT_YIELD_NOT_ACCEPTED") + print("ISL_ORIGINAL_PL_YIELD_SOURCE_SHAPE=" + + json.dumps(shape, sort_keys=True, separators=(",", ":"))) + print("ISL_ORIGINAL_PL_YIELD_CONTEXT=" + + json.dumps(receipt, sort_keys=True, separators=(",", ":"))) + PY diff --git a/.github/workflows/isl-public-first-prospective-codeql-ci.yml b/.github/workflows/isl-public-first-prospective-codeql-ci.yml index 1cdae746..e04a8cbc 100644 --- a/.github/workflows/isl-public-first-prospective-codeql-ci.yml +++ b/.github/workflows/isl-public-first-prospective-codeql-ci.yml @@ -5,6 +5,8 @@ name: ISL public-first source CodeQL static security scan on: pull_request: paths: + - 'scripts/eurostat_pl_*.py' + - 'tests/test_eurostat_pl_*.py' - 'scripts/public_first_prospective_source_observation_v03.py' - 'tests/test_public_first_prospective_source_observation_v03.py' - '.github/workflows/isl-public-first-prospective-source-ci.yml' diff --git a/docs/ISL_EUROSTAT_PL_SOVEREIGN_YIELD_REFERENCE_V04_20261112_PL.md b/docs/ISL_EUROSTAT_PL_SOVEREIGN_YIELD_REFERENCE_V04_20261112_PL.md new file mode 100644 index 00000000..a8a256ac --- /dev/null +++ b/docs/ISL_EUROSTAT_PL_SOVEREIGN_YIELD_REFERENCE_V04_20261112_PL.md @@ -0,0 +1,30 @@ +# Public-first v0.4 — Polska: rentowności obligacji 10Y (Eurostat) + +**Wątek:** [#152](https://github.com/gniewkob/Investment-Strategy-Lab/issues/152), PR #153. **Data:** 12.11.2026. **Koszt dostępu:** 0 PLN, bez kluczy i subskrypcji. + +## Oficjalne pierwotne źródło i rozdzielenie twierdzeń + +| Twierdzenie lub ograniczenie | Źródło / sekcja | Ocena | +|---|---|---| +| Eurostat publikuje miesięczne rentowności długoterminowych obligacji skarbowych, wyrażone jako **procent rocznie**, o zapadalności ok. 10 lat | [Eurostat: Maastricht criterion interest rates, ESMS](https://ec.europa.eu/eurostat/cache/metadata/en/irt_lt_mcby_esms.htm), sekcje **3.4**, **4**, **5** | Oficjalna metodologia dotycząca rentowności, **nie** holding-period total return | +| Dane MCBY pozyskiwane przez ECB z krajowych banków centralnych, w tym Polski | Ten sam [Eurostat ESMS](https://ec.europa.eu/eurostat/cache/metadata/en/irt_lt_mcby_esms.htm), sekcja **18.1–18.5** | Źródło pochodzenia statystyki, nie identyfikacja obligacji czy kuponów | +| Dane miesięczne publikowane zwykle po ok. 10 dniach | Ten sam ESMS, sekcja **14.1** | Statystyczna typowa aktualność, **nie** weryfikacja historycznego `first_available_at` dla obserwacji | +| Dane mogą być korygowane zgodnie z polityką ECB | Ten sam ESMS, sekcja **17** | Nie traktować dzisiejszych liczb jak niezmiennych oryginalnych publikacji | +| Ogólna możliwość ponownego wykorzystania statystyk Eurostat z podaniem źródła i wyjątkami | [Eurostat: Copyright notice and free re-use of data](https://ec.europa.eu/eurostat/help/copyright-notice), pkt **2–3** | Ogólna polityka reuse, nie aprobata wszystkich celów AI/automatycznego inwestowania ani praw podmiotów trzecich | + +**Identyfikator:** `irt_lt_mcby_m`; `freq=M`, `int_rt=MCBY`, `geo=PL`, `time=2021-11...2026-09`. Kod źródła: `scripts/eurostat_pl_bond_yield_source_probe_v04.py` i `scripts/eurostat_pl_government_bond_yield_context_v04.py`. Jawne `--allow-public-eurostat-read`; odpowiedzi oficjalne HTTPS; brak przekierowań, limit 600 KB. + +## Wynik pierwszego rzeczywistego odczytu + +[Jednorazowy probe original publisher API w PR #153](https://github.com/gniewkob/Investment-Strategy-Lab/actions/runs/43324574226) w dniu 12.11.2026 o 22:57:18 UTC potwierdził rzeczywistą strukturę JSON-stat: `[freq,int_rt,geo,time]`, `[1,1,1,59]`, kody `[M,MCBY,PL]`, `2021-11` do `2026-09`. Original-source SHA-256 odpowiedzi: `acc302b641ca981b5c8e22259a613d092e5c12d0cb01e39e7e093c07c59c3abf`. Provider last update: `2026-11-12T11:00:00+0100`. Pierwszy probe potwierdził 59 etykiet czasu, **nie liczbę niepustych wartości**, które są weryfikowane w osobnym typed-parser CI. Nie uzyskano dowodu historycznych oryginalnych wersji. + +## Techniczny zakres + +- `parse_original_yields` weryfikuje 59 prawdziwych, kolejnych miesięcy, dokładnie 4 wymiary `freq/int_rt/geo/time`, porządek kategorii JSON-stat, wartości procentowe, ewentualne statusy danych i oryginalne raw SHA-256. Brak obserwacji = BLOKADA, nie zero. Fałszywy kraj, `int_rt`, data w przyszłości lub błędne jednostki = BLOKADA. +- Raport ograniczony do **statystyk poziomów rentowności**, nie publikujemy 59 surowych wartości w logach CI. Min, max i średnia procentowa rentowności oraz największa bezwzględna miesięczna różnica poziomów w punktach bazowych; **nie** traktować jej jako rentowności z inwestycji. +- `historical_pit_certified=false`, `bond_holding_period_return_observed=false`, `tbsp_income_index_observed=false`, `qualified_model_series=[]`, `INSUFFICIENT_DATA_HOLD_CASH`. +- **Bez symulacji kupna/sprzedaży, rekomendacji aktywów, importów baz, połączeń portfela ani auto-schedule.** Przetwarzane są tylko oficjalne publiczne dane w kontrolowanym jednorazowym odczycie, offline testy i wersja źródła. + +## Rzeczy pozostające poza zakresem + +Rentowność w procentach p.a. to nie zrealizowany zwrot z portfela obligacji; brak cen transakcyjnych, kupowanych ISIN, kuponów, reinwestycji, kosztów i różnych terminów zapadalności. Sam hash dzisiejszej odpowiedzi nie dowodzi historycznego stanu `first_available_at`. Nadal otwarty [P0 #116](https://github.com/gniewkob/Investment-Strategy-Lab/issues/116) dotyczący praw do oryginalnego indeksu TR, source vintages i uprawnień. Ta analiza **nie kwalifikuje** `PL_BONDS_TR_PLN` ani M1–M3. From f2aa795693872a6d37acce7d1a36d3bb8df4dfab Mon Sep 17 00:00:00 2001 From: Gniewkob Date: Fri, 13 Nov 2026 00:04:35 +0100 Subject: [PATCH 5/5] security/evidence: sanitize original Eurostat schema probe, prove 59/59 published Polish yields (#152) Block log-control metadata, duplicate dimension indices and spoofed category codes. Document actual original-source SHA, 59 real monthly values, aggregate rates and no-PIT caveat. --- .../isl-eurostat-pl-yield-probe-v04.yml | 2 +- ...VEREIGN_YIELD_REFERENCE_V04_20261112_PL.md | 6 +++++ ...eurostat_pl_bond_yield_source_probe_v04.py | 23 +++++++++++++--- ...eurostat_pl_bond_yield_source_probe_v04.py | 26 +++++++++++++++++++ 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/.github/workflows/isl-eurostat-pl-yield-probe-v04.yml b/.github/workflows/isl-eurostat-pl-yield-probe-v04.yml index 3a091915..83167638 100644 --- a/.github/workflows/isl-eurostat-pl-yield-probe-v04.yml +++ b/.github/workflows/isl-eurostat-pl-yield-probe-v04.yml @@ -45,7 +45,7 @@ jobs: ): suite.addTests(unittest.defaultTestLoader.loadTestsFromName(name)) result = unittest.TextTestRunner(verbosity=2).run(suite) - if not result.wasSuccessful() or result.skipped or result.testsRun != 17: + if not result.wasSuccessful() or result.skipped or result.testsRun != 19: raise SystemExit("EUROSTAT_PL_YIELD_SOURCE_CONTRACT_FAILURE") PY official-original-source-evidence: diff --git a/docs/ISL_EUROSTAT_PL_SOVEREIGN_YIELD_REFERENCE_V04_20261112_PL.md b/docs/ISL_EUROSTAT_PL_SOVEREIGN_YIELD_REFERENCE_V04_20261112_PL.md index a8a256ac..e62c41be 100644 --- a/docs/ISL_EUROSTAT_PL_SOVEREIGN_YIELD_REFERENCE_V04_20261112_PL.md +++ b/docs/ISL_EUROSTAT_PL_SOVEREIGN_YIELD_REFERENCE_V04_20261112_PL.md @@ -28,3 +28,9 @@ ## Rzeczy pozostające poza zakresem Rentowność w procentach p.a. to nie zrealizowany zwrot z portfela obligacji; brak cen transakcyjnych, kupowanych ISIN, kuponów, reinwestycji, kosztów i różnych terminów zapadalności. Sam hash dzisiejszej odpowiedzi nie dowodzi historycznego stanu `first_available_at`. Nadal otwarty [P0 #116](https://github.com/gniewkob/Investment-Strategy-Lab/issues/116) dotyczący praw do oryginalnego indeksu TR, source vintages i uprawnień. Ta analiza **nie kwalifikuje** `PL_BONDS_TR_PLN` ani M1–M3. + +## Faktycznie opublikowane wartości z oryginalnego odczytu — PASS + +Zrealizowany [jednorazowy odczyt oficjalnego Eurostat w CI #43325200200](https://github.com/gniewkob/Investment-Strategy-Lab/actions/runs/43325200200) wykazał **59/59 niepustych miesięcy 2021-11…2026-09**, bez statusów specjalnych (`source_statistical_flags_count={}`). Rzeczywisty SHA-256 pobranych bajtów `acc302b641ca981b5c8e22259a613d092e5c12d0cb01e39e7e093c07c59c3abf`, receipt `4157225ea7cc1f2a5bf9816f9141ab036f098a29f0d5e5f3cfecdda42388b9f6`. Źródło zgłosiło ostatnią aktualizację `2026-11-12T11:00:00+0100`. Zapis **nie** poświadcza dokładnej pierwotnej daty żadnej historycznej obserwacji. + +Na oficjalnych liczbach: minimum **3,12% p.a.**, maksimum **7,82% p.a.**, średnia z miesięcznych poziomów **5,6115% p.a.**, największa bezwzględna różnica między poziomami sąsiadujących miesięcy **154 pb** (tj. 1,54 punktu procentowego). Ta różnica **nie jest** stopą zwrotu z obligacji ani maksymalną stratą/zyskiem inwestora. Ani poziom rentowności, ani jego zmiana nie reprezentują realizowalnego indeksu dochodowego. diff --git a/scripts/eurostat_pl_bond_yield_source_probe_v04.py b/scripts/eurostat_pl_bond_yield_source_probe_v04.py index 0c306836..a4db68ce 100644 --- a/scripts/eurostat_pl_bond_yield_source_probe_v04.py +++ b/scripts/eurostat_pl_bond_yield_source_probe_v04.py @@ -9,6 +9,8 @@ from datetime import datetime, timezone from hashlib import sha256 import json +import re +from scripts.public_first_prospective_source_observation_v03 import _provider_update from urllib.parse import urlencode, urlsplit from urllib.request import HTTPRedirectHandler, HTTPSHandler, Request, build_opener @@ -63,7 +65,9 @@ def inspect_source_shape(raw: object, *, observed_at: datetime) -> dict: raise PublicEurostatProbeBlocked("JSONSTAT_STRUCTURE_NOT_MATCHED") dims = data["dimension"] names = data["id"] - if (not 1 <= len(names) <= 7 or len(names) != len(set(names)) + if (len(names) != 4 or any(type(name) is not str for name in names) + or len(names) != len(set(names)) + or set(names) != {"freq", "int_rt", "geo", "time"} or set(names) != set(dims) or len(data["size"]) != len(names)): raise PublicEurostatProbeBlocked("DIMENSIONS_INCOHERENT") codes = {} @@ -74,9 +78,20 @@ def inspect_source_shape(raw: object, *, observed_at: datetime) -> dict: raise PublicEurostatProbeBlocked("DIMENSION_INVALID") cat = dims[name].get("category") ix = cat.get("index") if isinstance(cat, dict) else None - if not isinstance(ix, dict) or len(ix) != count: + if (not isinstance(ix, dict) or len(ix) != count + or any(type(v) is not int for v in ix.values()) + or set(ix.values()) != set(range(count)) + or any(type(k) is not str or not re.fullmatch( + r"20\d{2}-(0[1-9]|1[0-2])" if name == "time" + else r"[A-Z0-9_]{1,16}", k) for k in ix)): raise PublicEurostatProbeBlocked("CATEGORIES_INVALID") - codes[name] = sorted(str(k) for k in ix) + codes[name] = sorted(ix) + try: + provider_update = _provider_update( + data.get("updated", "NOT_PROVIDED"), + observed_at.astimezone(timezone.utc)) + except ValueError as exc: + raise PublicEurostatProbeBlocked("PROVIDER_UPDATED_INVALID") from exc # No raw data or complete time series is ever printed/returned. return { "schema": "ISL_PUBLIC_EUROSTAT_PL_BONDS_YIELD_SHAPE_PROBE_V04", @@ -91,7 +106,7 @@ def inspect_source_shape(raw: object, *, observed_at: datetime) -> dict: "time_first": codes["time"][0] if "time" in codes else None, "time_last": codes["time"][-1] if "time" in codes else None, "time_count": len(codes.get("time", [])), - "provider_updated_not_first_release": str(data.get("updated", "UNKNOWN"))[:50], + "provider_updated_not_first_release": provider_update, "actual_rate_values_in_output": False, "historical_pit_certified": False, "bond_holding_period_total_return": False, diff --git a/tests/test_eurostat_pl_bond_yield_source_probe_v04.py b/tests/test_eurostat_pl_bond_yield_source_probe_v04.py index 7966ccc0..57302942 100644 --- a/tests/test_eurostat_pl_bond_yield_source_probe_v04.py +++ b/tests/test_eurostat_pl_bond_yield_source_probe_v04.py @@ -46,6 +46,32 @@ def test_invalid_payload_and_oversize_blocked(self): with self.subTest(size=len(raw)), self.assertRaises(PublicEurostatProbeBlocked): inspect_source_shape(raw, observed_at=CLOCK) + def test_provider_updated_injection_and_future_rejected(self): + for provider_update in ( + "2026-11-12T11:00:00+0100\n::warning::malicious", + "2032-01-01T00:00:00Z", + 123456): + d = sample() + d["updated"] = provider_update + with self.subTest(value=str(provider_update)), self.assertRaisesRegex( + PublicEurostatProbeBlocked, "PROVIDER_UPDATED_INVALID"): + inspect_source_shape(json.dumps(d).encode(), observed_at=CLOCK) + + def test_invalid_category_positions_and_code_injection_rejected(self): + for mutation in ("duplicate", "malicious", "bad-id"): + d = sample() + if mutation == "duplicate": + d["dimension"]["time"]["category"]["index"]["2026-09"] = 0 + elif mutation == "malicious": + d["dimension"]["geo"]["category"]["index"] = { + "PL\n::warning::forged": 0, + } + else: + d["id"][0] = ["malicious"] + with self.subTest(mutation=mutation), self.assertRaises( + PublicEurostatProbeBlocked): + inspect_source_shape(json.dumps(d).encode(), observed_at=CLOCK) + def test_naive_clock_blocked(self): with self.assertRaisesRegex(PublicEurostatProbeBlocked, "CLOCK_REQUIRED"): inspect_source_shape(json.dumps(sample()).encode(),