diff --git a/addon/suncore/DOCS.md b/addon/suncore/DOCS.md index 45d835d..3a5ad2e 100644 --- a/addon/suncore/DOCS.md +++ b/addon/suncore/DOCS.md @@ -63,8 +63,11 @@ Supported architectures: `amd64` and `aarch64`. the assistant; **Earlier versions** restores a previous save; **Export configuration** and **Import configuration** back up everything as a file. The settings are stored in the app's `/data` folder and are part of the app -backup. This version uses database schema 3, which earlier versions cannot open: -export the configuration before updating. +backup. This version uses database schema 4, which earlier versions cannot open: +export the configuration before updating. Measurements, forecasts and prices +live in a separate `timeseries.db`, which is left out of the app backup; SunCore +rebuilds its quarter-hour history from Home Assistant's statistics when the file +is missing. ### Sign convention diff --git a/addon/suncore/config.yaml b/addon/suncore/config.yaml index fe73cfe..d84fefe 100644 --- a/addon/suncore/config.yaml +++ b/addon/suncore/config.yaml @@ -28,6 +28,7 @@ backup_exclude: - timeseries.db - timeseries.db-wal - timeseries.db-shm + - timeseries.db.corrupt* map: - type: homeassistant_config read_only: true diff --git a/addon/suncore_beta/DOCS.md b/addon/suncore_beta/DOCS.md index 7da86da..36975b2 100644 --- a/addon/suncore_beta/DOCS.md +++ b/addon/suncore_beta/DOCS.md @@ -66,8 +66,11 @@ Supported architectures: `amd64` and `aarch64`. the assistant; **Earlier versions** restores a previous save; **Export configuration** and **Import configuration** back up everything as a file. The settings are stored in the app's `/data` folder and are part of the app -backup. This version uses database schema 3, which earlier versions cannot open: -export the configuration before updating. +backup. This version uses database schema 4, which earlier versions cannot open: +export the configuration before updating. Measurements, forecasts and prices +live in a separate `timeseries.db`, which is left out of the app backup; SunCore +rebuilds its quarter-hour history from Home Assistant's statistics when the file +is missing. ### Sign convention diff --git a/addon/suncore_beta/config.yaml b/addon/suncore_beta/config.yaml index e70b4dd..0a4837d 100644 --- a/addon/suncore_beta/config.yaml +++ b/addon/suncore_beta/config.yaml @@ -26,6 +26,7 @@ backup_exclude: - timeseries.db - timeseries.db-wal - timeseries.db-shm + - timeseries.db.corrupt* map: - type: homeassistant_config read_only: true diff --git a/docs/STATUS.md b/docs/STATUS.md index d3f3843..b4ff05b 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -3,9 +3,9 @@ Read this first in every session, then `CLAUDE.md` (once it exists) and the ADRs relevant to your task. Keep it short; history lives in git. -**Last update:** 2026-10-16 · **Phase:** 4 in progress – PR 4-A contracts; Phase 3 (3-A +**Last update:** 2026-10-17 · **Phase:** 4 in progress – PR 4-A contracts, PR 4-B storage; Phase 3 (3-A `phase-3/a-safety`, 3-B `phase-3/b-controller`, 3-C `phase-3/c-ui`) implemented, waiting for the -maintainer's shadow test · **Branch:** `claude/zealous-mayer-j5oda9` +maintainer's shadow test · **Branch:** `phase-4/b-storage` (stacked on `claude/zealous-mayer-j5oda9`) ## Done @@ -29,6 +29,35 @@ maintainer's shadow test · **Branch:** `claude/zealous-mayer-j5oda9` - ADR-0013 "Battery as a controlled asset" **proposed** (prerequisite of WP10). - Review fixes: in progress (`fix(...)` commits on the branch). +- **Phase 4 – PR 4-B (WP3, storage and statistics; branch `phase-4/b-storage`, stacked on 4-A)**: + - `/data/timeseries.db` (own migrations and `user_version`; excluded from backups together with + its WAL/SHM and `timeseries.db.corrupt*`): `measurement` 48 h and `measurement_1m` 30 days + (primary key `(ts, key)`: about 5x fewer WAL bytes per flush than `(key, ts)`), `measurement_15m` + unlimited with a `derived` flag, `forecast` versioned by issue time (kept 45 days after the + slot), `price` 400 days, `meta`. `TimeseriesStore`: batched appends, series round trips, + `latest_forecast_before`, nightly job 03:30 local (wakes at most hourly, follows the time + zone): downsample (1-minute tier only for complete quarter hours), retention, `VACUUM` only + when worth it, `size_report`. + - Corruption handling: only "not a database"/"malformed"/a failing `quick_check` move the file + aside (with WAL/SHM, `timeseries.db.corrupt-`, newest two kept); locked, full, I/O or + permission errors and a file with a newer schema run SunCore without time series, nothing is + moved or deleted. + - Rebuild of 28 days of site history from `recorder/statistics_during_period` (period `hour`, + explicit units) as `derived` quarter hours (`suncore/ha/statistics.py`, + `suncore/app/history.py`), until it succeeded once (marker in `timeseries.db`), back-off 1, 5, + 15, then 60 min; `unknown_command` is final for the process. + - Controller measurements through a batched writer (flush every 60 s and at shutdown, only + OK-quality values; the tick hook cannot disturb control). + - `suncore.db` migration 0004 (`plan_snapshot` zlib JSON, 200 rows / 14 days; `learned_param`): + **DB schema 4, earlier versions refuse it – export before updating (release-risk note).** + - `ActionRateLimiter` per `domain.service[#sub-key]` for `call_action_response` (default 30 s, + `ActionRateLimitedError`, follow-up A3 done); boundary rule 7 admits only + `suncore.app.adapters`. + - Fixed: a cancelled `Database.transaction` could leave a transaction open on the shared + connection. + - Follow-ups: `size_report` is not on a diagnostics page yet; a too-new `timeseries.db` is only + logged (a UI hint follows in 4-H). + - **Fix after the first live test (2026-10-16, branch `fix/device-freshness`)**: on the maintainer's HA the SunSpec battery SoC (84 % for 25 min) and the `smartthinq_sensors` power sensors (5 W for a day) never advance `last_reported` for an unchanged value, so SunCore marked them "stale" and the @@ -308,7 +337,7 @@ maintainer's shadow test · **Branch:** `claude/zealous-mayer-j5oda9` (WP12, optional, after 4-E). Merge only after the Phase 3 acceptance. - **ADR-0013 must be accepted before WP10** (4-G). Its contract changes (`BatteryAction` `None` = baseline, `BatteryBand.grid_charge_max_w`/`discharge_lock`, `effective_reserve_pct`, - `battery_identity`) land in a WP2 fix before 4-B. + `battery_identity`) land at the start of 4-G, after the maintainer accepted it. - Maintainer questions from 4-A: plan §6 "Offene Fragen aus PR 4-A" (grid-charge schedule precedence, VAT on negative spot prices, export tiers, grid charging on the Kostal). @@ -501,6 +530,8 @@ maintainer's shadow test · **Branch:** `claude/zealous-mayer-j5oda9` | Kostal Modbus external battery management not enabled on the maintainer's inverter; how it is enabled; §3.5 fallback timer behaviour | `research/kostal-plenticore-modbus.md`, ADR-0013 | maintainer enables it and measures; `kostal_modbus` capped at assisted until then | | Creating a modbus hub from the HA UI (else YAML) | `research/ha-websocket-api.md` | maintainer's HA | | Supervisor proxy passes `recorder/*` and `energy/*` commands | `research/ha-websocket-api.md` | first Phase 4 run on HA OS | +| Supervisor proxy forwards `recorder/statistics_during_period` (the 4-B rebuild logs a warning and retries) | `suncore/ha/statistics.py` | first run on HA OS | +| `change`/`sum` rows of real energy counters (the rebuild reads them as verified from source, not tested against live counters); `backup_exclude` accepts the glob `timeseries.db.corrupt*` | `suncore/ha/statistics.py`, `addon/*/config.yaml` | maintainer's HA with an energy counter; a backup on HA OS | | Publication time of next-day prices (Tibber, Nord Pool, ENTSO-E, EPEX) | `research/integrations.md` | observe on the maintainer's HA | | VAT applied to negative spot prices | `suncore/core/tariff.py` | maintainer's bill / supplier terms | | KNN load model performance on aarch64 | plan WP12 | CI arm runner benchmark | diff --git a/frontend/src/lib/i18n/locales/de.json b/frontend/src/lib/i18n/locales/de.json index 9f24c8f..0a28c0f 100644 --- a/frontend/src/lib/i18n/locales/de.json +++ b/frontend/src/lib/i18n/locales/de.json @@ -408,6 +408,7 @@ "grid.power": "Netzleistung (positiv = Bezug)", "grid.power_hint": "Ein Sensor, der positiv ist, wenn du Strom beziehst, und negativ bei Einspeisung.", "grid.sensors": "Netzsensoren", + "ha.error.action_rate_limited": "Eine Home-Assistant-Aktion wurde zu oft angefordert und übersprungen; SunCore fragt nach dem Mindestabstand erneut an.", "log.all_devices": "Alle Geräte", "log.count": "{count, plural, =0 {Keine Einträge} one {# Eintrag} other {# Einträge}}", "log.device": "Gerät", diff --git a/frontend/src/lib/i18n/locales/en.json b/frontend/src/lib/i18n/locales/en.json index bbc7daf..0bf9c58 100644 --- a/frontend/src/lib/i18n/locales/en.json +++ b/frontend/src/lib/i18n/locales/en.json @@ -408,6 +408,7 @@ "grid.power": "Grid power (positive = import)", "grid.power_hint": "One sensor that is positive when you draw from the grid and negative when you feed in.", "grid.sensors": "Grid sensors", + "ha.error.action_rate_limited": "A Home Assistant action was requested too often and skipped; SunCore asks again after the minimum interval.", "log.all_devices": "All devices", "log.count": "{count, plural, =0 {No entries} one {# entry} other {# entries}}", "log.device": "Device", diff --git a/scripts/check_reasons.py b/scripts/check_reasons.py index 3fcf235..0bbf5cc 100644 --- a/scripts/check_reasons.py +++ b/scripts/check_reasons.py @@ -8,7 +8,7 @@ * string constants starting with ``api.error.`` (the error ``detail`` keys of the API) or ``api.validation.`` in ``suncore/api`` (validation issue keys), * string constants that are exactly one key with the prefix ``wizard.`` (check and import - reasons built from a variable) or ``detect.criterion.`` anywhere + reasons built from a variable), ``detect.criterion.`` or ``ha.error.`` anywhere in ``suncore/``, and ``controller.`` in ``suncore/controller/reasons.py`` (the controller's key constants). @@ -28,7 +28,7 @@ SOURCE = ROOT / "suncore" LOCALES = ROOT / "frontend" / "src" / "lib" / "i18n" / "locales" ERROR_PREFIXES = ("api.error.", "api.validation.") -KEY_PREFIXES = ("wizard.", "detect.criterion.") +KEY_PREFIXES = ("wizard.", "detect.criterion.", "ha.error.") CONTROLLER_KEYS = Path("controller") / "reasons.py" """The controller defines its keys as constants in one module (log event names elsewhere may share the ``controller.`` prefix).""" diff --git a/suncore/app/adapters.py b/suncore/app/adapters.py new file mode 100644 index 0000000..1c7e2d5 --- /dev/null +++ b/suncore/app/adapters.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Composition of the tariff and forecast adapters (Phase 4 WP4/WP5). + +This module will build the price and forecast sources from the configured templates and hand +them the Home Assistant connection. It is the only module of ``suncore.app`` that may call +``HAConnection.call_action_response`` (``tests/safety/test_boundary.py``, rule 7): the call is a +read-only action with ``return_response`` and is rate limited per ``(domain, service)`` by the +connection (``ActionRateLimiter``). An adapter passes the polling interval of its template as +``min_interval_s`` and treats ``ActionRateLimitedError`` as "use the last value". + +Empty on purpose until the first adapter lands (PR 4-C). +""" + +from __future__ import annotations diff --git a/suncore/app/control.py b/suncore/app/control.py index d4e2421..79f7164 100644 --- a/suncore/app/control.py +++ b/suncore/app/control.py @@ -49,7 +49,7 @@ DeviationEvent, ) from suncore.controller.loop import TickResult -from suncore.core.balance import compute_flows +from suncore.core.balance import FlowSnapshot, compute_flows from suncore.core.config import GoalsConfig from suncore.core.control import ( TEMPERATURE_ROLE, @@ -62,6 +62,14 @@ from suncore.core.decisions import Action, Decision, Reason from suncore.core.devices import DeviceConfig, control_roles from suncore.core.identity import identity_token +from suncore.core.measurements import ( + SITE_BATTERY_W, + SITE_GRID_W, + SITE_HOUSE_W, + SITE_PV_W, + SITE_SOC_PCT, + device_power_key, +) from suncore.core.signals import Quality, Sample from suncore.core.site import SiteConfig from suncore.devices.drivers import ClimateDriver, NumberDriver, create_driver @@ -101,6 +109,8 @@ """What ``create_driver`` returns, structurally: the gate's actuator plus ``device_state``.""" StateChangeListener = Callable[[], None] +MeasurementSink = Callable[[datetime, Mapping[str, float | None]], None] +"""Receives ``(tick time, {measurement key: value or None})`` after every tick.""" def probe_temperature_c(cache: EntityCache, device: DeviceConfig) -> float | None: @@ -169,6 +179,10 @@ class ControlStack: (``gate.recover(after_handover=True)``). predecessor: the gate of the replaced stack: devices it could not release (removed or re-mapped while Home Assistant was down) are released with its actuators. + on_measurements: called after every tick with the tick time and the site and device + measurements of that tick (``suncore.core.measurements`` keys; ``None`` = no valid + value). Synchronous and cheap by contract; an exception is logged and swallowed - + recording can never disturb the control loop. defer_recover: wait for :meth:`allow_recover` before ``gate.recover()`` (the runtime first learns Home Assistant's time zone, whose change would rebuild the stack in the middle of the recovery). @@ -199,8 +213,10 @@ def __init__( # noqa: PLR0915 - the composition root of the control layer after_handover: bool = False, predecessor: SafetyGate | None = None, defer_recover: bool = False, + on_measurements: MeasurementSink | None = None, ) -> None: self._connection = connection + self._on_measurements = on_measurements self._recover_allowed = not defer_recover self._after_handover = after_handover self._site = site @@ -687,11 +703,40 @@ async def tick(self) -> TickResult | None: ) self.estimates = estimates self.states = states + self._record_measurements(now, flows, estimates) self.last = await self.loop.tick(now, inputs) self._maybe_save_models(now) self._notify() return self.last + def _record_measurements( + self, + now: datetime, + flows: FlowSnapshot | None, + estimates: Mapping[str, PowerEstimate], + ) -> None: + """Hand this tick's values to the recorder; only ``OK`` data counts (a gap otherwise).""" + if self._on_measurements is None: + return + try: + values: dict[str, float | None] = {} + if flows is not None: + for key, flow in ( + (SITE_GRID_W, flows.grid_w), + (SITE_PV_W, flows.pv_w), + (SITE_BATTERY_W, flows.battery_w), + (SITE_SOC_PCT, flows.battery_soc_pct), + (SITE_HOUSE_W, flows.house_w), + ): + values[key] = flow.value if flow.quality is Quality.OK else None + for device_id, estimate in estimates.items(): + values[device_power_key(device_id)] = ( + estimate.value_w if estimate.quality is Quality.OK else None + ) + self._on_measurements(now, values) + except Exception: + log.exception("control.measurement_sink_failed") + async def drain(self) -> None: """Wait for the submissions and observations in flight (they wait on simulated time).""" await self.loop.drain() diff --git a/suncore/app/history.py b/suncore/app/history.py new file mode 100644 index 0000000..1773019 --- /dev/null +++ b/suncore/app/history.py @@ -0,0 +1,172 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Rebuild of the 15-minute site history from Home Assistant's statistics (Phase 4 WP3). + +Runs once after the first successful connect when ``measurement_15m`` is empty (a restore +without ``timeseries.db``, a fresh install). The configured site entities are asked for their +hourly statistics (``suncore.ha.statistics``); the answers are combined the way the live +balance does (``suncore.core.balance.compute_flows``): grid as ``import - export`` for a pair, +PV as the sum of the arrays, the house from its sensor or from ``pv + grid - battery``. The +rows are stored as *derived* quarter hours. + +Controlled devices are not rebuilt: their power comes from their own power models, which have +no statistic to read. ``UNVERIFIED``: whether the Supervisor proxy forwards ``recorder/*``; +an error is reported to the caller, which logs it and carries on without history. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta + +import structlog + +from suncore.core.measurements import ( + SITE_BATTERY_W, + SITE_GRID_W, + SITE_HOUSE_W, + SITE_PV_W, + SITE_SOC_PCT, +) +from suncore.core.series import Grid, SeriesUnit, TimeSeries, resample_power +from suncore.core.site import SiteConfig +from suncore.core.units import house_load_w +from suncore.ha.connection import HAConnection +from suncore.ha.statistics import StatisticSpec, fetch_quarter_history +from suncore.storage.timeseries import TimeseriesStore + +log = structlog.get_logger(__name__) + +REBUILD_DAYS = 28 +"""History that is rebuilt (the load forecast needs 14 days, the risk model starts at 30).""" + +_GRID, _GRID_IMPORT, _GRID_EXPORT = "grid", "grid_import", "grid_export" +_BATTERY, _SOC, _HOUSE = "battery", "soc", "house" + + +@dataclass(frozen=True, slots=True) +class RebuildResult: + """What a rebuild stored: quarter hours per measurement key, and the statistic ids + Home Assistant had no data for.""" + + stored: Mapping[str, int] + missing: tuple[str, ...] + + @property + def total(self) -> int: + return sum(self.stored.values()) + + +def site_statistic_specs(site: SiteConfig) -> list[StatisticSpec]: + """The statistics to ask for: every mapped power entity and the state of charge.""" + specs: list[StatisticSpec] = [] + grid = site.grid + if grid.power is not None: + specs.append(StatisticSpec(_GRID, grid.power.entity_id, invert=grid.power.invert_sign)) + else: + assert grid.import_power is not None + assert grid.export_power is not None + specs.append( + StatisticSpec( + _GRID_IMPORT, grid.import_power.entity_id, invert=grid.import_power.invert_sign + ) + ) + specs.append( + StatisticSpec( + _GRID_EXPORT, grid.export_power.entity_id, invert=grid.export_power.invert_sign + ) + ) + specs.extend( + StatisticSpec(f"pv{i}", ref.entity_id, invert=ref.invert_sign) + for i, ref in enumerate(site.pv_power_refs) + ) + if site.battery is not None: + power = site.battery.power + specs.append(StatisticSpec(_BATTERY, power.entity_id, invert=power.invert_sign)) + specs.append(StatisticSpec(_SOC, site.battery.soc.entity_id, unit=SeriesUnit.PERCENT)) + if site.house is not None: + specs.append(StatisticSpec(_HOUSE, site.house.entity_id, invert=site.house.invert_sign)) + return specs + + +def _common_grid(series: Sequence[TimeSeries]) -> Grid: + start = min(s.start for s in series) + end = max(s.end for s in series) + step = series[0].step + return Grid(start=start, step=step, count=int((end - start) / step)) + + +def _combine(parts: Sequence[TimeSeries | None], fn: Callable[..., float]) -> TimeSeries | None: + """Slot-wise ``fn`` over series on one grid; ``None`` if a part is missing in the slot.""" + present = [p for p in parts if p is not None] + if len(present) != len(parts) or not present: + return None + grid = _common_grid(present) + aligned = [resample_power(p, grid) for p in present] + values = tuple( + None if any(v is None for v in slot) else fn(*slot) + for slot in zip(*(a.values for a in aligned), strict=True) + ) + return TimeSeries(start=grid.start, step=grid.step, values=values, unit=aligned[0].unit) + + +def assemble_site_series( + site: SiteConfig, fetched: Mapping[str, TimeSeries] +) -> dict[str, TimeSeries]: + """Measurement key -> 15-minute series, combined like the live power balance.""" + get = fetched.get + if site.grid.power is not None: + grid = get(_GRID) + else: + grid = _combine([get(_GRID_IMPORT), get(_GRID_EXPORT)], lambda i, e: i - e) + pv_parts = [get(f"pv{i}") for i in range(len(site.pv_power_refs))] + pv = _combine(pv_parts, lambda *v: sum(v)) if pv_parts else None + if site.battery is None: + battery = soc = None + else: + battery, soc = get(_BATTERY), get(_SOC) + if site.house is not None: + house = get(_HOUSE) + else: + # pv + grid - battery; a site without PV or battery contributes 0 W for it. + has_pv, has_battery = bool(pv_parts), site.battery is not None + parts = [grid, *([pv] if has_pv else []), *([battery] if has_battery else [])] + + def balance(*v: float) -> float: + it = iter(v) + g = next(it) + return house_load_w(next(it) if has_pv else 0.0, g, next(it) if has_battery else 0.0) + + house = _combine(parts, balance) + named = { + SITE_GRID_W: grid, + SITE_PV_W: pv, + SITE_BATTERY_W: battery, + SITE_SOC_PCT: soc, + SITE_HOUSE_W: house, + } + return {key: series for key, series in named.items() if series is not None} + + +async def rebuild_history( + connection: HAConnection, + store: TimeseriesStore, + site: SiteConfig, + now: datetime, + *, + days: int = REBUILD_DAYS, +) -> RebuildResult: + """Fetch the site's hourly statistics and store them as derived quarter hours. + + Home Assistant errors propagate. Existing measured rows are never overwritten. + """ + end = now.astimezone(UTC).replace(minute=0, second=0, microsecond=0) + history = await fetch_quarter_history( + connection, site_statistic_specs(site), end - timedelta(days=days), end + ) + stored: dict[str, int] = {} + for key, series in assemble_site_series(site, history.series).items(): + stored[key] = await store.store_quarter_hours(key, series, derived=True) + result = RebuildResult(stored=stored, missing=history.missing) + log.info("history.rebuilt", quarter_hours=result.total, missing=list(result.missing)) + return result diff --git a/suncore/app/main.py b/suncore/app/main.py index 349641c..1b2a4a0 100644 --- a/suncore/app/main.py +++ b/suncore/app/main.py @@ -24,6 +24,7 @@ from suncore.api import create_app from suncore.app.demo import DemoClock, demo_tick_s from suncore.app.logconfig import configure_logging +from suncore.app.recorder import MeasurementRecorder from suncore.app.runtime import DB_FILENAME, Runtime, utc_now from suncore.app.settings import ENV_SUPERVISOR_TOKEN, Settings, load_settings, under_supervisor from suncore.core.control import ControlSettings @@ -40,9 +41,14 @@ ConfigStore, Database, DecisionLog, + LearnedParamStore, + PlanSnapshotStore, PowerModelStateStore, SqliteRuntimeStore, + TimeseriesStore, + open_timeseries_or_recover, ) +from suncore.storage.timeseries import TS_FILENAME from suncore.templates import load_templates log = structlog.get_logger(__name__) @@ -90,7 +96,7 @@ def _demo_wiring(settings: Settings) -> DemoWiring: ) -async def build_runtime( +async def build_runtime( # noqa: PLR0915 - the composition root settings: Settings, *, env: Mapping[str, str] | None = None, @@ -120,11 +126,14 @@ async def build_runtime( data_dir = settings.data_dir data_dir.mkdir(parents=True, exist_ok=True) db = await Database.open(data_dir / DB_FILENAME) + timeseries_db: Database | None = None try: + timeseries_db = await open_timeseries_or_recover(data_dir / TS_FILENAME) config = ConfigStore(db) if connection is None: connection = WebSocketConnection(transport_from_env(env)) decision_log = DecisionLog(db) + timeseries = None if timeseries_db is None else TimeseriesStore(timeseries_db) cache = EntityCache(clock=clock) catalogue = EntityCatalogue(connection, wall_clock=clock) user_templates = None if demo else data_dir / "templates" @@ -150,6 +159,11 @@ async def build_runtime( decision_log=decision_log, runtime_store=SqliteRuntimeStore(db, clock=clock), power_store=PowerModelStateStore(db, clock=clock), + plans=PlanSnapshotStore(db, clock=clock), + learned=LearnedParamStore(db, clock=clock), + timeseries_db=timeseries_db, + timeseries=timeseries, + recorder=MeasurementRecorder(timeseries), connection=connection, cache=cache, catalogue=catalogue, @@ -166,6 +180,8 @@ async def build_runtime( runtime._tempdir = tempdir await runtime.prepare() except BaseException: + if timeseries_db is not None: + await timeseries_db.close() await db.close() if tempdir is not None: tempdir.cleanup() diff --git a/suncore/app/recorder.py b/suncore/app/recorder.py new file mode 100644 index 0000000..dd8add0 --- /dev/null +++ b/suncore/app/recorder.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Batched writer of the controller's site measurements into ``timeseries.db`` (Phase 4 WP3). + +The controller calls :meth:`MeasurementRecorder.record` once per tick; it only appends to an +in-memory list. :meth:`flush` (every 60 s from the runtime, and once at shutdown) writes the +list in one transaction. A database error is logged and the batch is kept for the next flush +(bounded: the oldest rows are dropped first), so a broken disk can neither crash nor slow the +control loop and memory stays bounded. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Mapping +from datetime import datetime +from typing import Final + +import structlog + +from suncore.storage.timeseries import MeasurementRow, TimeseriesStore + +log = structlog.get_logger(__name__) + +FLUSH_INTERVAL_S: Final = 60.0 +MAX_BUFFERED_ROWS: Final = 50_000 +"""About 7 hours of a 5 s tick with 10 keys (2 rows/s): the cap for a database that stays broken.""" + + +class MeasurementRecorder: + """Collects ``(key, ts, value)`` rows and writes them in batches.""" + + def __init__( + self, store: TimeseriesStore | None, *, max_buffered: int = MAX_BUFFERED_ROWS + ) -> None: + self._store = store + self._max = max_buffered + self._buffer: list[MeasurementRow] = [] + self._lock = asyncio.Lock() + self.dropped = 0 + """Rows discarded because the buffer was full (a database that stays unavailable).""" + + @property + def enabled(self) -> bool: + return self._store is not None + + @property + def pending(self) -> int: + return len(self._buffer) + + def record(self, ts: datetime, values: Mapping[str, float | None]) -> None: + """Queue one tick's values; a ``None`` value is a gap and is not stored.""" + if self._store is None: + return + self._buffer.extend((key, ts, value) for key, value in values.items() if value is not None) + overflow = len(self._buffer) - self._max + if overflow > 0: + del self._buffer[:overflow] + self.dropped += overflow + + async def flush(self) -> int: + """Write the queued rows; returns how many were written (0 on error or nothing).""" + if self._store is None: + return 0 + async with self._lock: + batch, self._buffer = self._buffer, [] + if not batch: + return 0 + try: + return await self._store.append_measurements(batch) + except BaseException as exc: + # Also on cancellation (shutdown): the batch goes back, nothing is lost silently. + combined = batch + self._buffer + self.dropped += max(0, len(combined) - self._max) + self._buffer = combined[-self._max :] + if not isinstance(exc, Exception): + raise + log.exception("timeseries.flush_failed", rows=len(batch)) + return 0 diff --git a/suncore/app/runtime.py b/suncore/app/runtime.py index 8859f9b..8b82efb 100644 --- a/suncore/app/runtime.py +++ b/suncore/app/runtime.py @@ -8,7 +8,7 @@ import tempfile from collections.abc import Awaitable, Callable, Collection, Iterable, Mapping, Sequence from dataclasses import dataclass, field -from datetime import UTC, datetime, timedelta, tzinfo +from datetime import UTC, datetime, time, timedelta, tzinfo from pathlib import Path from typing import Any from zoneinfo import ZoneInfo, ZoneInfoNotFoundError @@ -28,7 +28,9 @@ ) from suncore.app.cadence import device_cadences, merge_cadences from suncore.app.control import ControlStack +from suncore.app.history import rebuild_history from suncore.app.modes import ConfigModes +from suncore.app.recorder import FLUSH_INTERVAL_S, MeasurementRecorder from suncore.app.settings import SUPERVISOR_INGRESS_PEER, Settings from suncore.core.balance import FlowSnapshot, compute_flows from suncore.core.config import DevicesConfig, GoalsConfig, OperationConfig, RegionConfig @@ -71,9 +73,13 @@ Database, DecisionLog, HistoryEntry, + LearnedParamStore, + PlanSnapshotStore, PowerModelStateStore, SqliteRuntimeStore, + TimeseriesStore, ) +from suncore.storage.timeseries import META_HISTORY_REBUILT from suncore.templates import TemplateCatalog LOOPBACK_PEERS: tuple[str, ...] = ("127.0.0.1", "::1") @@ -82,7 +88,13 @@ DB_FILENAME = "suncore.db" PRUNE_INTERVAL_S = 3600.0 -"""Decision-log retention runs hourly.""" +"""Decision-log and plan-snapshot retention run hourly.""" +REBUILD_BACKOFF_S = (60.0, 300.0, 900.0, 3600.0) +"""Retry delays of the statistics rebuild after a timeout or Home Assistant error.""" +MAINTENANCE_WAKE_S = 3600.0 +"""The nightly loop wakes at most this often to re-check the time (zone changes, clock jumps).""" +MAINTENANCE_TIME = time(3, 30) +"""Local time of the nightly time-series job (downsample, retention, vacuum if worth it).""" def utc_now() -> datetime: @@ -116,6 +128,15 @@ class Runtime: devices: DevicesConfig | None = None region: RegionConfig | None = None goals: GoalsConfig | None = None + timeseries_db: Database | None = None + """``timeseries.db``; ``None`` = unavailable (too new, unopenable): control is unaffected.""" + timeseries: TimeseriesStore | None = None + plans: PlanSnapshotStore | None = None + learned: LearnedParamStore | None = None + recorder: MeasurementRecorder = field(default_factory=lambda: MeasurementRecorder(None)) + """Batched writer of the controller's measurements (disabled without ``timeseries``).""" + flush_interval_s: float = FLUSH_INTERVAL_S + maintenance_time: time = MAINTENANCE_TIME sleep: Callable[[float], Awaitable[None]] = asyncio.sleep prune_interval_s: float = PRUNE_INTERVAL_S real_clock: Clock = utc_now @@ -136,6 +157,11 @@ class Runtime: _unsubscribers: list[Callable[[], None]] = field(default_factory=list) _tempdir: tempfile.TemporaryDirectory[str] | None = None _prune_task: asyncio.Task[None] | None = None + _flush_task: asyncio.Task[None] | None = None + _maintenance_task: asyncio.Task[None] | None = None + _history_task: asyncio.Task[None] | None = None + _history_done: bool = False + """The statistics rebuild ran (or was not needed) in this process.""" _stack_lock: asyncio.Lock = field(default_factory=asyncio.Lock) _operation_lock: asyncio.Lock = field(default_factory=asyncio.Lock) _started: bool = False @@ -182,6 +208,11 @@ async def start(self) -> None: self.stack.start_ticking() self._connect_task = asyncio.create_task(self._connect(), name="suncore-ha-connect") self._prune_task = asyncio.create_task(self._prune_loop(), name="suncore-prune") + if self.timeseries is not None: + self._flush_task = asyncio.create_task(self._flush_loop(), name="suncore-ts-flush") + self._maintenance_task = asyncio.create_task( + self._maintenance_loop(), name="suncore-ts-maintenance" + ) async def _load_live_sections(self) -> None: self.site = await self.config.get_site() @@ -237,6 +268,7 @@ def _new_stack( reserve_soc_pct=None if battery is None else battery.min_soc_pct, cadences=merge_cadences(device_cadences(devices), self.extra_cadences), sink=self._record_decision, + on_measurements=self.recorder.record, cache=self.cache, power_store=self.power_store, attach=False, @@ -271,6 +303,7 @@ async def _rebuild_stack(self) -> None: await new.start() new.start_ticking() self._notify_control() + self._start_history_rebuild() # a site configured after the first connect def _unchanged_devices(self, previous: Mapping[str, DeviceConfig]) -> frozenset[str]: """Ids of the devices that are the same device, controlled the same way, after a change.""" @@ -294,11 +327,124 @@ async def prune_decisions(self) -> int: log.exception("decision_log.prune_failed") return 0 + async def prune_plans(self) -> int: + """One retention pass over the plan snapshots; failures are logged, never raised.""" + if self.plans is None: + return 0 + try: + return await self.plans.prune(self.clock()) + except Exception: + log.exception("plan_snapshot.prune_failed") + return 0 + async def _prune_loop(self) -> None: while True: await self.prune_decisions() + await self.prune_plans() await self.sleep(self.prune_interval_s) + # --- time series (timeseries.db) --------------------------------------------- + + async def _flush_loop(self) -> None: + while True: + await self.sleep(self.flush_interval_s) + await self.recorder.flush() # logs its own errors + + async def maintain_timeseries(self) -> bool: + """The nightly job once: flush, downsample, retention, vacuum if worth it. + + Failures are logged, never raised; returns whether the job ran through. + """ + if self.timeseries is None: + return False + try: + await self.recorder.flush() # the newest samples are aggregated too + await self.timeseries.maintain(self.clock()) + except Exception: + log.exception("timeseries.maintenance_failed") + return False + return True + + def next_maintenance_after(self, moment: datetime) -> datetime: + """The first ``maintenance_time`` (site time zone, DST safe) strictly after ``moment``.""" + tz = self._timezone() + local = moment.astimezone(tz) + target = datetime.combine(local.date(), self.maintenance_time, tzinfo=tz) + if target.astimezone(UTC) <= moment.astimezone(UTC): + target = datetime.combine( + local.date() + timedelta(days=1), self.maintenance_time, tzinfo=tz + ) + return target.astimezone(UTC) + + def seconds_until_maintenance(self) -> float: + """Seconds to the next ``maintenance_time`` in the site's time zone.""" + now = self.clock() + return (self.next_maintenance_after(now) - now.astimezone(UTC)).total_seconds() + + async def _maintenance_loop(self) -> None: + """Run at start, then at every ``maintenance_time``; wakes at most hourly. + + The target is always "the first 03:30 after the last run" in the *current* time zone, so + a zone that becomes known later (Home Assistant's, after the first connect) moves the + target instead of leaving a stale sleep, and one run can never happen twice. + """ + await self.maintain_timeseries() + last_run = self.clock() + while True: + remaining = (self.next_maintenance_after(last_run) - self.clock()).total_seconds() + if remaining <= 0: + await self.maintain_timeseries() + last_run = self.clock() + continue + await self.sleep(min(remaining, MAINTENANCE_WAKE_S)) + + def _start_history_rebuild(self) -> None: + """Rebuild the 15-minute history from HA's statistics, once, in the background.""" + if ( + self._history_done + or not self._started + or self.timeseries is None + or self.site is None + or self.connection.state is not ConnectionState.CONNECTED + or (self._history_task is not None and not self._history_task.done()) + ): + return + self._history_task = asyncio.create_task(self._rebuild_history(), name="suncore-history") + + async def _rebuild_history(self) -> None: + """Rebuild until it succeeded once (marker in ``timeseries.db``), with back-off. + + A timeout or Home Assistant error is retried after 1, 5, 15 and then every 60 minutes; + ``unknown_command`` (the recorder commands are not available, e.g. a proxy that does not + forward them - UNVERIFIED) is final for this process and retried at the next start. + """ + attempt = 0 + while True: + store, site = self.timeseries, self.site + if store is None or site is None: + return + try: + if await store.get_meta(META_HISTORY_REBUILT) is None: + await rebuild_history(self.connection, store, site, self.clock()) + await store.set_meta(META_HISTORY_REBUILT, self.clock().isoformat()) + except asyncio.CancelledError: + raise + except CommandError as exc: + log.warning("history.rebuild_failed", reason=exc.code, error=str(exc)) + if exc.code == "unknown_command": + break + except (HAError, TimeoutError) as exc: + log.warning("history.rebuild_failed", reason=type(exc).__name__, error=str(exc)) + except Exception: + log.exception("history.rebuild_crashed") + break + else: + break + delay = REBUILD_BACKOFF_S[min(attempt, len(REBUILD_BACKOFF_S) - 1)] + attempt += 1 + await self.sleep(delay) + self._history_done = True + async def _connect(self) -> None: try: await self.connection.connect() @@ -308,6 +454,7 @@ async def _connect(self) -> None: log.error("ha.connect_failed", error_type=type(exc).__name__, error=str(exc)) return await self._prepare_control() + self._start_history_rebuild() def _time_zone_settled(self) -> bool: return self.region is not None or self._tz_learnt @@ -354,17 +501,37 @@ async def stop(self) -> None: for unsubscribe in self._unsubscribers: unsubscribe() self._unsubscribers.clear() - for task in (self._connect_task, self._prune_task, self._tz_task): + control_tasks = (self._connect_task, self._tz_task) + storage_tasks = ( + self._prune_task, + self._flush_task, + self._maintenance_task, + self._history_task, + ) + for task in (*control_tasks, *storage_tasks): if task is not None: task.cancel() + for task in control_tasks: + if task is not None: with contextlib.suppress(asyncio.CancelledError, Exception): await task self._connect_task = self._prune_task = self._tz_task = None + self._flush_task = self._maintenance_task = self._history_task = None + # Hardware release comes first: a running downsample or retention statement keeps the + # (shielded) rollback of its cancelled task waiting, and a Pi can need seconds for it. + # The storage tasks are awaited only after the devices are released. async with self._stack_lock: if self.stack is not None: await self.stack.shutdown() + for task in storage_tasks: + if task is not None: + with contextlib.suppress(asyncio.CancelledError, Exception): + await task await self.catalogue.stop() await self.connection.close() + await self.recorder.flush() + if self.timeseries_db is not None: + await self.timeseries_db.close() await self.db.close() if self._tempdir is not None: self._tempdir.cleanup() diff --git a/suncore/core/measurements.py b/suncore/core/measurements.py new file mode 100644 index 0000000..42551dd --- /dev/null +++ b/suncore/core/measurements.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Keys of the recorded site measurements (Phase 4 WP3). + +The time-series store (``timeseries.db``) and its readers (load forecast, accuracy +statistics, the rebuild from Home Assistant's statistics) agree on these keys. Units and +signs are the core conventions: W for power, % for the state of charge; grid + = import, +battery + = charging, device + = consumption, PV + = generation. +""" + +from __future__ import annotations + +from typing import Final + +SITE_GRID_W: Final = "site.grid_w" +SITE_PV_W: Final = "site.pv_w" +SITE_BATTERY_W: Final = "site.battery_w" +SITE_SOC_PCT: Final = "site.soc_pct" +SITE_HOUSE_W: Final = "site.house_w" + +SITE_KEYS: Final[tuple[str, ...]] = ( + SITE_GRID_W, + SITE_PV_W, + SITE_BATTERY_W, + SITE_SOC_PCT, + SITE_HOUSE_W, +) + +_DEVICE_PREFIX: Final = "device." +_DEVICE_SUFFIX: Final = ".power_w" + + +def device_power_key(device_id: str) -> str: + """Key of a controlled device's estimated power (W).""" + return f"{_DEVICE_PREFIX}{device_id}{_DEVICE_SUFFIX}" + + +def is_percent_key(key: str) -> bool: + """Whether the measurement is a state in percent (not a power in W).""" + return key == SITE_SOC_PCT + + +__all__ = [ + "SITE_BATTERY_W", + "SITE_GRID_W", + "SITE_HOUSE_W", + "SITE_KEYS", + "SITE_PV_W", + "SITE_SOC_PCT", + "device_power_key", + "is_percent_key", +] diff --git a/suncore/ha/__init__.py b/suncore/ha/__init__.py index 824ac0e..bdc6b0e 100644 --- a/suncore/ha/__init__.py +++ b/suncore/ha/__init__.py @@ -9,6 +9,8 @@ from suncore.ha.connection import ( REGISTRY_EVENT_TYPES, ActionNotAllowedError, + ActionRateLimitedError, + ActionRateLimiter, AuthError, CommandError, CommandTimeoutError, @@ -64,6 +66,8 @@ "REGISTRY_EVENT_TYPES", "SITE_ROLES", "ActionNotAllowedError", + "ActionRateLimitedError", + "ActionRateLimiter", "AuthError", "Backoff", "CachedEntity", diff --git a/suncore/ha/connection.py b/suncore/ha/connection.py index c43efe3..77dadb2 100644 --- a/suncore/ha/connection.py +++ b/suncore/ha/connection.py @@ -25,6 +25,7 @@ import time from collections.abc import Awaitable, Callable, Collection, Mapping, Sequence from dataclasses import dataclass, field +from datetime import datetime from enum import StrEnum from typing import Any, Final, Protocol, runtime_checkable @@ -72,6 +73,31 @@ class ActionNotAllowedError(HAError): """``call_action_response`` was asked for an action that is not on the read-only allowlist.""" +def action_rate_key(domain: str, service: str, rate_key: str | None = None) -> str: + """Limiter key of an action call: ``domain.service`` plus an optional caller sub-key.""" + base = f"{domain}.{service}" + return base if not rate_key else f"{base}#{rate_key}" + + +class ActionRateLimitedError(HAError): + """A read-only action was asked for again before its minimum interval elapsed. + + Nothing was sent to Home Assistant. ``retry_after_s`` is how long until the call would be + admitted. The user-visible text is the catalogue entry :data:`ERR_ACTION_RATE_LIMITED`. + """ + + code = "rate_limited" + + def __init__(self, action: str, retry_after_s: float) -> None: + super().__init__(f"{action} is rate limited, retry in {retry_after_s:.1f} s") + self.action = action + self.retry_after_s = retry_after_s + + +ERR_ACTION_RATE_LIMITED: Final = "ha.error.action_rate_limited" +"""Catalogue key of the refusal of a too frequent read-only action.""" + + # --- value types -------------------------------------------------------------- @@ -226,11 +252,44 @@ async def call_action_response( data: Mapping[str, Any] | None = None, *, target: Mapping[str, Any] | None = None, + min_interval_s: float | None = None, + max_wait_s: float = 0.0, + rate_key: str | None = None, ) -> dict[str, Any]: """Call a read-only action with ``return_response`` and return its response. Raises :class:`ActionNotAllowedError` unless ``domain.service`` is in :data:`suncore.core.actions.READ_ONLY_ACTIONS`. This is *not* a write path. + + Each ``(domain, service)`` is rate limited (:class:`ActionRateLimiter`): at most one + call per ``min_interval_s`` (default :data:`DEFAULT_ACTION_INTERVAL_S`; a caller passes + the interval of its configured polling). A call that is too early waits at most + ``max_wait_s`` (default: not at all) and otherwise raises + :class:`ActionRateLimitedError`; it never blocks indefinitely. ``rate_key`` (e.g. the + target entity or the requested date) gives calls of one service that ask for different + things their own slot: the limiter key is ``domain.service`` or ``domain.service#key``. + """ + ... + + async def statistics_during_period( + self, + statistic_ids: Sequence[str], + start: datetime, + end: datetime | None = None, + *, + period: str = "hour", + types: Sequence[str] | None = None, + units: Mapping[str, str] | None = None, + ) -> Mapping[str, Sequence[Mapping[str, Any]]]: + """``recorder/statistics_during_period`` (read only, not admin gated). + + ``start``/``end`` are sent as ISO 8601 with their UTC offset. The result maps each + statistic id *that has data* to its rows (``start``/``end`` in epoch milliseconds, + the requested ``types``); ids without data are absent. The response carries no unit: + pass ``units`` (e.g. ``{"power": "W", "energy": "Wh"}``) so the numbers do not change + when the user switches a sensor's display unit. Verified shapes: + ``docs/research/ha-websocket-api.md`` "Long-term statistics". + ``UNVERIFIED``: that the Supervisor proxy forwards ``recorder/*``. """ ... @@ -278,6 +337,75 @@ async def acquire(self) -> None: self._last = self._clock() +DEFAULT_ACTION_INTERVAL_S: Final = 30.0 +"""Default minimum spacing of two calls of the same read-only action (follow-up A3).""" + + +class ActionRateLimiter: + """Minimum spacing per key (``domain.service``) with try-acquire and a wait deadline. + + Slots are handed out in order: a caller that waits reserves the slot after the previous + one, so two concurrent callers do not both pass. A waiter that is cancelled keeps its + reservation (the slot stays taken until it would have elapsed): that is the safe direction + for an integration that must not be hammered. Clock and sleep are injected. A key that + is too busy is refused instead of queued (:meth:`acquire` returns ``False``), so a slow + integration can never stack up calls. + """ + + def __init__( + self, + default_interval_s: float = DEFAULT_ACTION_INTERVAL_S, + *, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + ) -> None: + if default_interval_s < 0: + msg = "default_interval_s must be >= 0" + raise ValueError(msg) + self._default = default_interval_s + self._clock = clock + self._sleep = sleep + self._next_ok: dict[str, float] = {} + + def retry_after(self, key: str) -> float: + """Seconds until ``key`` would be admitted (0 = now).""" + return max(0.0, self._next_ok.get(key, 0.0) - self._clock()) + + def try_acquire(self, key: str, interval_s: float | None = None) -> bool: + """Take the slot if it is free now; otherwise change nothing and return ``False``.""" + interval = self._interval(interval_s) + now = self._clock() + if self._next_ok.get(key, 0.0) > now: + return False + self._next_ok[key] = now + interval + return True + + async def acquire( + self, key: str, interval_s: float | None = None, *, max_wait_s: float = 0.0 + ) -> bool: + """Take the next slot of ``key`` if it comes within ``max_wait_s`` (waiting for it). + + Returns ``False`` without waiting and without reserving when the slot is further away. + """ + interval = self._interval(interval_s) + now = self._clock() + slot = max(now, self._next_ok.get(key, 0.0)) + wait = slot - now + if wait > max(0.0, max_wait_s): + return False + self._next_ok[key] = slot + interval # reserved before sleeping: no await in between + if wait > 0: + await self._sleep(wait) + return True + + def _interval(self, interval_s: float | None) -> float: + interval = self._default if interval_s is None else interval_s + if interval < 0: + msg = "interval_s must be >= 0" + raise ValueError(msg) + return interval + + @dataclass(frozen=True, slots=True) class ServiceResult: """Outcome of a service call: HA's response, the state read back afterwards and the diff --git a/suncore/ha/statistics.py b/suncore/ha/statistics.py new file mode 100644 index 0000000..44dc5e0 --- /dev/null +++ b/suncore/ha/statistics.py @@ -0,0 +1,251 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Quarter-hour history from Home Assistant's long-term statistics (Phase 4 WP3). + +When ``timeseries.db`` is missing or empty (a restore without it, a fresh install) the +15-minute history is rebuilt from ``recorder/statistics_during_period``. The finest period +that outlives the recorder's purge is ``hour`` (``5minute`` is purged after ``purge_keep_days``, +default 10; verified in ``docs/research/ha-websocket-api.md`` "Long-term statistics"), so the +rebuilt history is a **step-hold of the hourly values**: + +* a power or state sensor (``mean``): each of the four quarter hours of an hour gets the hour's + mean, which preserves the energy of the hour (4 x W x 0.25 h = W x 1 h); +* an energy counter (``change`` = the hour's delta of ``sum``, in Wh): each quarter hour gets + ``change / 1 h`` as mean power, which preserves the energy as well. + +These values are *derived*: the intra-hour shape is lost. They are stored with +``derived = 1`` and are replaced as soon as the controller measured the same slot itself. + +Request details (all verified): ``start_time``/``end_time`` are ISO 8601 **with** a UTC offset +(a value without one is read in HA's time zone), the response has ``start``/``end`` in integer +epoch **milliseconds** and no unit, so ``units`` is always sent. A statistic id without data is +absent from the result; a value can be ``null``. ``UNVERIFIED``: that the Supervisor proxy +forwards ``recorder/*`` (first run on Home Assistant OS); an error is the caller's to log. +""" + +from __future__ import annotations + +import math +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any, Final, Literal + +import structlog + +from suncore.core.series import MAX_POINTS, PLANNING_STEP, SeriesUnit, TimeSeries +from suncore.ha.connection import HAConnection + +log = structlog.get_logger(__name__) + +STATISTICS_UNITS: Final[Mapping[str, str]] = {"power": "W", "energy": "Wh"} +"""Target units, always sent so a display-unit change in HA does not change the numbers.""" + +DEFAULT_CHUNK_DAYS: Final = 7 +DEFAULT_BATCH_SIZE: Final = 20 +HOUR_MS: Final = 3_600_000 +QUARTER_MS: Final = 900_000 + +StatKind = Literal["mean", "energy"] +"""``mean``: power or state sensor (column ``mean``); ``energy``: counter (``change``/``sum``).""" + +_TYPES: Final[Mapping[str, tuple[str, ...]]] = { + "mean": ("mean",), + "energy": ("change", "sum"), +} + + +@dataclass(frozen=True, slots=True) +class StatisticSpec: + """One statistic to fetch: ``key`` names it in the result, ``statistic_id`` is the + sensor's entity id. ``invert`` flips the sign (a source with opposite polarity).""" + + key: str + statistic_id: str + kind: StatKind = "mean" + unit: SeriesUnit = SeriesUnit.W + invert: bool = False + + +@dataclass(frozen=True, slots=True) +class StatisticsHistory: + """Result of a fetch: 15-minute series per spec key and the ids Home Assistant had no + data for in the whole range.""" + + series: Mapping[str, TimeSeries] + missing: tuple[str, ...] + + +def _hour_start(row: Mapping[str, Any]) -> int | None: + start = row.get("start") + if isinstance(start, bool) or not isinstance(start, int | float): + return None + start_ms = int(start) + return start_ms if start_ms % QUARTER_MS == 0 else None + + +def _number(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float): + return None + number = float(value) + return number if math.isfinite(number) else None + + +def hourly_values(rows: Iterable[Mapping[str, Any]], kind: StatKind) -> dict[int, float]: + """``{hour start (epoch ms): value}`` of the usable rows. + + ``mean`` rows give the mean as is; ``energy`` rows give the hour's energy in Wh from + ``change``, else from the difference of ``sum`` to the previous row *if that row is the + previous hour*. A row with a missing value, a start off the quarter-hour grid or a + negative energy (a counter reset HA did not compensate) is skipped - a gap, never a zero. + """ + values: dict[int, float] = {} + previous_sum: tuple[int, float] | None = None + for row in sorted(rows, key=lambda r: _hour_start(r) or 0): + start = _hour_start(row) + if start is None: + continue + end = row.get("end") + if ( + isinstance(end, int | float) + and not isinstance(end, bool) + and int(end) - start != HOUR_MS + ): + continue # not an hour row: the request asked for period "hour" + if kind == "mean": + mean = _number(row.get("mean")) + if mean is not None: + values[start] = mean + continue + total = _number(row.get("sum")) + change = _number(row.get("change")) + if ( + change is None + and total is not None + and previous_sum is not None + and previous_sum[0] + HOUR_MS == start + ): + change = total - previous_sum[1] + if total is not None: + previous_sum = (start, total) + if change is not None and change >= 0: + values[start] = change # Wh in the hour == mean W over one hour + return values + + +def quarter_hours( + hourly: Mapping[int, float], *, unit: SeriesUnit, invert: bool = False +) -> TimeSeries | None: + """Step-hold the hourly values onto a 15-minute grid (``None`` where an hour is missing). + + The grid runs from the first to the last known hour. ``None`` if there is no value. + """ + if not hourly: + return None + first = min(hourly) + last = max(hourly) + HOUR_MS + count = (last - first) // QUARTER_MS + if count > MAX_POINTS: + msg = f"statistics range too long: {count} points exceed {MAX_POINTS}" + raise ValueError(msg) + sign = -1.0 if invert else 1.0 + slots: list[float | None] = [None] * count + for start_ms, value in hourly.items(): + offset = (start_ms - first) // QUARTER_MS + for i in range(HOUR_MS // QUARTER_MS): + slots[offset + i] = sign * value + 0.0 # + 0.0: no negative zero + return TimeSeries( + start=datetime.fromtimestamp(first / 1000, UTC), + step=PLANNING_STEP, + values=tuple(slots), + unit=unit, + ) + + +def _hour_floor(moment: datetime) -> datetime: + utc = moment.astimezone(UTC) + return utc.replace(minute=0, second=0, microsecond=0) + + +def _hour_ceil(moment: datetime) -> datetime: + floor = _hour_floor(moment) + return floor if floor == moment.astimezone(UTC) else floor + timedelta(hours=1) + + +def _windows(start: datetime, end: datetime, chunk: timedelta) -> list[tuple[datetime, datetime]]: + out: list[tuple[datetime, datetime]] = [] + cursor = start + while cursor < end: + out.append((cursor, min(cursor + chunk, end))) + cursor += chunk + return out + + +async def fetch_quarter_history( + connection: HAConnection, + specs: Sequence[StatisticSpec], + start: datetime, + end: datetime, + *, + chunk_days: int = DEFAULT_CHUNK_DAYS, + batch_size: int = DEFAULT_BATCH_SIZE, +) -> StatisticsHistory: + """Fetch hourly statistics in chunks and return them as 15-minute series per spec. + + Requests are split by time (``chunk_days``) and by statistic id (``batch_size``) so one + answer stays small, and by kind because the requested ``types`` differ. Errors of + Home Assistant (:class:`~suncore.ha.connection.HAError`) propagate: a rebuild is + all-or-nothing and the caller decides whether to retry. + """ + if not start < end: + msg = "statistics range: start must be before end" + raise ValueError(msg) + if chunk_days < 1 or batch_size < 1: + msg = "chunk_days and batch_size must be positive" + raise ValueError(msg) + start, end = _hour_floor(start), _hour_ceil(end) + rows: dict[str, list[Mapping[str, Any]]] = {} + by_id = {s.statistic_id: s for s in specs} + for kind in ("mean", "energy"): + ids = sorted({s.statistic_id for s in specs if s.kind == kind}) + for begin, stop in _windows(start, end, timedelta(days=chunk_days)): + for i in range(0, len(ids), batch_size): + batch = ids[i : i + batch_size] + result = await connection.statistics_during_period( + batch, + begin, + stop, + period="hour", + types=_TYPES[kind], + units=STATISTICS_UNITS, + ) + for statistic_id, statistic_rows in result.items(): + if statistic_id in by_id and statistic_rows: + rows.setdefault(statistic_id, []).extend(statistic_rows) + series: dict[str, TimeSeries] = {} + missing: list[str] = [] + for spec in specs: + built = quarter_hours( + hourly_values(rows.get(spec.statistic_id, ()), spec.kind), + unit=spec.unit, + invert=spec.invert, + ) + if built is None: + missing.append(spec.statistic_id) + else: + series[spec.key] = built + if missing: + log.info("statistics.no_data", statistic_ids=sorted(set(missing))) + return StatisticsHistory(series=series, missing=tuple(dict.fromkeys(missing))) + + +__all__ = [ + "DEFAULT_BATCH_SIZE", + "DEFAULT_CHUNK_DAYS", + "STATISTICS_UNITS", + "StatKind", + "StatisticSpec", + "StatisticsHistory", + "fetch_quarter_history", + "hourly_values", + "quarter_hours", +] diff --git a/suncore/ha/websocket.py b/suncore/ha/websocket.py index 7a49628..6a2d47a 100644 --- a/suncore/ha/websocket.py +++ b/suncore/ha/websocket.py @@ -77,6 +77,8 @@ DEFAULT_WRITE_INTERVAL_S, REGISTRY_EVENT_TYPES, ActionNotAllowedError, + ActionRateLimitedError, + ActionRateLimiter, AuthError, CommandError, CommandTimeoutError, @@ -94,6 +96,7 @@ StateContext, StateListener, Unsubscribe, + action_rate_key, ) from suncore.ha.redact import DEFAULT_REGISTRY, SecretRegistry from suncore.ha.transport import Transport @@ -106,6 +109,18 @@ """Period of the full ``get_states`` resync. It is also the real freshness of a sensor whose value does not change: the stream is silent, the resync sees its ``last_reported`` move.""" +STATISTIC_PERIODS: Final = frozenset({"5minute", "hour", "day", "week", "month", "year"}) +"""Periods of ``recorder/statistics_during_period`` (verified, research/ha-websocket-api.md).""" + + +def _iso_with_offset(moment: datetime) -> str: + """ISO 8601 with a UTC offset: a value without one is read in HA's time zone.""" + if moment.tzinfo is None or moment.utcoffset() is None: + msg = "statistics time range needs timezone-aware datetimes" + raise ValueError(msg) + return moment.isoformat() + + MAX_MESSAGE_BYTES: Final = 32 * 1024 * 1024 """A full ``get_states`` of a large installation exceeds aiohttp's 4 MiB default.""" @@ -312,6 +327,7 @@ def __init__( self._resync_interval = resync_interval self._stable_after = stable_after self._write_limiter = RateLimiter(write_interval, clock=clock, sleep=sleep) + self._action_limiter = ActionRateLimiter(clock=clock, sleep=sleep) secrets.register(transport.token) @@ -468,15 +484,24 @@ async def call_action_response( data: Mapping[str, Any] | None = None, *, target: Mapping[str, Any] | None = None, + min_interval_s: float | None = None, + max_wait_s: float = 0.0, + rate_key: str | None = None, ) -> dict[str, Any]: """Read-only action call with ``return_response``; see :class:`HAConnection`. - Deliberately not rate limited like writes (no device is touched) and not routed - through :class:`ServiceCaller`: the allowlist is the guard. + Not routed through :class:`ServiceCaller` (no device is touched): the allowlist is + the guard against writes and the per-action :class:`ActionRateLimiter` the guard + against hammering an integration (follow-up A3). """ if not is_read_only_action(domain, service): msg = f"{domain}.{service} is not a read-only action" raise ActionNotAllowedError(msg) + if self._ws is None or self._ws.closed: # before a limiter slot is taken + raise NotConnectedError("not connected to Home Assistant") + key = action_rate_key(domain, service, rate_key) + if not await self._action_limiter.acquire(key, min_interval_s, max_wait_s=max_wait_s): + raise ActionRateLimitedError(key, self._action_limiter.retry_after(key)) payload: dict[str, Any] = { "type": "call_service", "domain": domain, @@ -494,6 +519,44 @@ async def call_action_response( raise ProtocolError(msg) return dict(response) + async def statistics_during_period( + self, + statistic_ids: Sequence[str], + start: datetime, + end: datetime | None = None, + *, + period: str = "hour", + types: Sequence[str] | None = None, + units: Mapping[str, str] | None = None, + ) -> Mapping[str, Sequence[Mapping[str, Any]]]: + """``recorder/statistics_during_period``; see :class:`HAConnection` for the shape.""" + if period not in STATISTIC_PERIODS: + msg = f"unknown statistics period {period!r}" + raise ValueError(msg) + if not statistic_ids: + return {} + payload: dict[str, Any] = { + "type": "recorder/statistics_during_period", + "start_time": _iso_with_offset(start), + "statistic_ids": list(statistic_ids), + "period": period, + } + if end is not None: + payload["end_time"] = _iso_with_offset(end) + if types is not None: + payload["types"] = list(types) + if units is not None: + payload["units"] = dict(units) + result = await self._command(payload) + if not isinstance(result, Mapping): + msg = "recorder/statistics_during_period returned a non-object" + raise ProtocolError(msg) + return { + str(key): [row for row in rows if isinstance(row, Mapping)] + for key, rows in result.items() + if isinstance(rows, list) + } + # --- supervisor loop ------------------------------------------------------ async def _run(self) -> None: diff --git a/suncore/sim/fake_ha.py b/suncore/sim/fake_ha.py index 08fda53..41edd80 100644 --- a/suncore/sim/fake_ha.py +++ b/suncore/sim/fake_ha.py @@ -48,7 +48,7 @@ import heapq import itertools import logging -from collections.abc import Awaitable, Callable, Collection, Mapping +from collections.abc import Awaitable, Callable, Collection, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta from functools import partial @@ -60,6 +60,8 @@ from suncore.ha.connection import ( REGISTRY_EVENT_TYPES, ActionNotAllowedError, + ActionRateLimitedError, + ActionRateLimiter, CommandError, ConnectionState, EntityListener, @@ -73,6 +75,7 @@ StateContext, StateListener, Unsubscribe, + action_rate_key, ) from suncore.sim import registry from suncore.sim.demo_devices import ( @@ -90,6 +93,7 @@ validate_schema, ) from suncore.sim.house import House, HouseState +from suncore.sim.recorder import SimRecorder from suncore.sim.scenario import ManualEvent, Scenario, SimSignal _log = logging.getLogger(__name__) @@ -255,8 +259,24 @@ def __init__( self._event_time: datetime | None = None self._offset_s = 0 self._unavailable: frozenset[str] = frozenset() + self._action_limiter = ActionRateLimiter( + clock=lambda: self._clock().timestamp(), sleep=sleep + ) self.call_log: list[CallRecord] = [] """Every service call seen, oldest first.""" + self.recorder = SimRecorder( + scenario, + { + GRID_ID: lambda s: s.grid_w, + PV_ID: lambda s: s.pv_w, + BATTERY_ID: lambda s: s.battery_w, + SOC_ID: lambda s: s.soc_pct, + HOUSE_ID: lambda s: s.load_w, + }, + ) + """Hourly statistics behind :meth:`statistics_during_period`.""" + self.statistics_error: HAError | None = None + """Make ``statistics_during_period`` fail (e.g. a proxy that blocks ``recorder/*``).""" # --- HAConnection --------------------------------------------------------- @@ -388,6 +408,24 @@ def fire_registry_event(self, event_type: str = REGISTRY_EVENT_TYPES[0]) -> None except Exception: _log.exception("registry listener failed") + async def statistics_during_period( + self, + statistic_ids: Sequence[str], + start: datetime, + end: datetime | None = None, + *, + period: str = "hour", + types: Sequence[str] | None = None, + units: Mapping[str, str] | None = None, # noqa: ARG002 - the fake always answers in W + ) -> Mapping[str, Sequence[Mapping[str, Any]]]: + """``recorder/statistics_during_period`` from :attr:`recorder` (hourly means in W).""" + self._require_connected() + if self.statistics_error is not None: + raise self.statistics_error + if not statistic_ids: + return {} + return self.recorder.query(statistic_ids, start, end, period, types) + async def call_action_response( self, domain: str, @@ -395,12 +433,18 @@ async def call_action_response( data: Mapping[str, Any] | None = None, # noqa: ARG002 *, target: Mapping[str, Any] | None = None, # noqa: ARG002 + min_interval_s: float | None = None, + max_wait_s: float = 0.0, + rate_key: str | None = None, ) -> dict[str, Any]: """Read-only action responses built from the demo data (allowlist like the real one).""" if not is_read_only_action(domain, service): msg = f"{domain}.{service} is not a read-only action" raise ActionNotAllowedError(msg) self._require_connected() + key = action_rate_key(domain, service, rate_key) + if not await self._action_limiter.acquire(key, min_interval_s, max_wait_s=max_wait_s): + raise ActionRateLimitedError(key, self._action_limiter.retry_after(key)) now = self._house_state.time if self._house_state is not None else self._clock() if (domain, service) == ("tibber", "get_prices"): start = now.replace(minute=(now.minute // 15) * 15, second=0, microsecond=0) diff --git a/suncore/sim/recorder.py b/suncore/sim/recorder.py new file mode 100644 index 0000000..49a3bea --- /dev/null +++ b/suncore/sim/recorder.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A minimal fake of Home Assistant's recorder statistics for the simulator (Phase 4 WP3). + +Answers ``recorder/statistics_during_period`` with ``period: "hour"`` in the verified shape +(``docs/research/ha-websocket-api.md``): rows of ``{start, end, mean}`` with epoch +**milliseconds**, no unit, ids without data absent from the result. The history is the +simulated house run *before* the scenario start at 15-minute steps (clear sky, no devices), +aggregated to hourly means - a plausible week for the rebuild, not a replay of the scenario. +Tests can replace the rows of any id with :meth:`set_rows`. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from datetime import UTC, datetime, timedelta +from typing import Any, Final + +from suncore.ha.connection import CommandError +from suncore.sim.house import House, HouseState +from suncore.sim.scenario import Scenario + +HISTORY_STEP_S: Final = 900 +HOUR_MS: Final = 3_600_000 + + +class SimRecorder: + """Hourly statistics of the simulated house. + + Args: + scenario: supplies the house parameters and the end of the history (its start). + series: statistic id -> reader of the value from a house state (the sim sensors). + history_hours: how far back the history reaches. + """ + + def __init__( + self, + scenario: Scenario, + series: Mapping[str, Callable[[HouseState], float]], + *, + history_hours: int = 72, + ) -> None: + self._scenario = scenario + self._series = dict(series) + self._history_hours = history_hours + self._rows: dict[str, list[dict[str, Any]]] | None = None + self._overrides: dict[str, list[dict[str, Any]]] = {} + self.calls: list[dict[str, Any]] = [] + """Every query seen (evidence for the chunking tests).""" + + def set_rows(self, statistic_id: str, rows: Sequence[Mapping[str, Any]]) -> None: + """Replace (or add) the rows of one statistic; an empty list removes its data.""" + self._overrides[statistic_id] = [dict(r) for r in rows] + + def _build(self) -> dict[str, list[dict[str, Any]]]: + end = self._scenario.start.astimezone(UTC).replace(minute=0, second=0, microsecond=0) + begin = end - timedelta(hours=self._history_hours) + house = House(self._scenario.house, begin, HISTORY_STEP_S) + steps_per_hour = 3600 // HISTORY_STEP_S + rows: dict[str, list[dict[str, Any]]] = {sid: [] for sid in self._series} + for hour in range(self._history_hours): + states = [house.step() for _ in range(steps_per_hour)] + start_ms = int((begin + timedelta(hours=hour)).timestamp() * 1000) + for statistic_id, read in self._series.items(): + mean = sum(read(s) for s in states) / len(states) + rows[statistic_id].append( + {"start": start_ms, "end": start_ms + HOUR_MS, "mean": mean} + ) + return rows + + def query( + self, + statistic_ids: Sequence[str], + start: datetime, + end: datetime | None, + period: str, + types: Sequence[str] | None, + ) -> dict[str, list[dict[str, Any]]]: + """The ``result`` object of ``recorder/statistics_during_period``.""" + self.calls.append( + {"ids": list(statistic_ids), "start": start, "end": end, "period": period} + ) + if period != "hour": + raise CommandError("not_found", f"period {period!r} is not simulated") + if self._rows is None: + self._rows = self._build() + lo = int(start.timestamp() * 1000) + hi = None if end is None else int(end.timestamp() * 1000) + wanted = tuple(types) if types is not None else ("mean",) + result: dict[str, list[dict[str, Any]]] = {} + for statistic_id in statistic_ids: + source = self._overrides.get(statistic_id, self._rows.get(statistic_id, [])) + picked = [ + {k: v for k, v in row.items() if k in ("start", "end", *wanted)} + for row in source + if row["start"] >= lo and (hi is None or row["start"] < hi) + ] + if picked: + result[statistic_id] = picked + return result diff --git a/suncore/storage/__init__.py b/suncore/storage/__init__.py index 772ea6a..9713a82 100644 --- a/suncore/storage/__init__.py +++ b/suncore/storage/__init__.py @@ -15,7 +15,14 @@ StorageError, UnknownSectionError, ) +from suncore.storage.learned import LearnedParam, LearnedParamStore +from suncore.storage.plans import PlanSnapshot, PlanSnapshotMeta, PlanSnapshotStore from suncore.storage.runtime import PowerModelStateStore, SqliteRuntimeStore +from suncore.storage.timeseries import ( + TimeseriesStore, + open_timeseries, + open_timeseries_or_recover, +) __all__ = [ "ConfigCorruptError", @@ -25,11 +32,19 @@ "HistoryEntry", "HistoryEntryNotFoundError", "ImportRejectedError", + "LearnedParam", + "LearnedParamStore", "MigrationError", + "PlanSnapshot", + "PlanSnapshotMeta", + "PlanSnapshotStore", "PowerModelStateStore", "SchemaTooNewError", "SectionSpec", "SqliteRuntimeStore", "StorageError", + "TimeseriesStore", "UnknownSectionError", + "open_timeseries", + "open_timeseries_or_recover", ] diff --git a/suncore/storage/db.py b/suncore/storage/db.py index e9da307..71989fb 100644 --- a/suncore/storage/db.py +++ b/suncore/storage/db.py @@ -4,6 +4,8 @@ from __future__ import annotations import asyncio +import contextlib +import sqlite3 from collections.abc import AsyncIterator, Iterable, Sequence from contextlib import asynccontextmanager from pathlib import Path @@ -12,12 +14,29 @@ import aiosqlite -from suncore.storage.migrations import run_migrations +from suncore.storage.migrations import Migration, run_migrations SqlParam = str | int | float | bytes | None Row = tuple[object, ...] +async def _rollback(conn: aiosqlite.Connection) -> None: + """``ROLLBACK`` that completes even if the awaiting task is cancelled meanwhile. + + Statements of a cancelled caller are still executed by the connection's worker thread, so + the rollback is queued behind them and shielded; "no transaction is active" is fine. + """ + + async def run() -> None: + with contextlib.suppress(sqlite3.OperationalError): + await conn.execute("ROLLBACK") + + task = asyncio.ensure_future(run()) + # Retrieve the outcome even if this await is cancelled: no "exception never retrieved". + task.add_done_callback(lambda t: t.cancelled() or t.exception()) + await asyncio.shield(task) + + class Transaction: """Statements inside one ``BEGIN IMMEDIATE`` block; obtained from ``Database.transaction``.""" @@ -32,6 +51,11 @@ async def execute(self, sql: str, params: Sequence[SqlParam] = ()) -> int: finally: await cur.close() + async def executemany(self, sql: str, rows: Iterable[Sequence[SqlParam]]) -> None: + """Run a statement for every parameter row (one round trip to the SQLite thread).""" + cur = await self._conn.executemany(sql, rows) + await asyncio.shield(cur.close()) # closed even if this task is cancelled meanwhile + async def fetchall(self, sql: str, params: Sequence[SqlParam] = ()) -> list[Row]: async with self._conn.execute(sql, params) as cur: return [tuple(r) for r in await cur.fetchall()] @@ -42,7 +66,7 @@ async def fetchone(self, sql: str, params: Sequence[SqlParam] = ()) -> Row | Non class Database: - """One connection to ``suncore.db``; all access is serialised by a lock.""" + """One connection to a SQLite file (``suncore.db``, ``timeseries.db``), serialised by a lock.""" def __init__(self, path: Path | str) -> None: self._path = str(path) @@ -50,12 +74,16 @@ def __init__(self, path: Path | str) -> None: self._lock = asyncio.Lock() @classmethod - async def open(cls, path: Path | str) -> Self: - """Open (creating if needed), configure, and migrate the database.""" + async def open(cls, path: Path | str, *, migrations: list[Migration] | None = None) -> Self: + """Open (creating if needed), configure, and migrate the database. + + ``migrations`` selects the migration set (default: ``suncore.db``'s; the time-series + database passes its own). A database newer than the set raises ``SchemaTooNewError``. + """ db = cls(path) await db._connect() try: - await run_migrations(db._connection()) + await run_migrations(db._connection(), migrations) except BaseException: await db.close() raise @@ -116,8 +144,7 @@ async def fetchone(self, sql: str, params: Sequence[SqlParam] = ()) -> Row | Non async def executemany(self, sql: str, rows: Iterable[Sequence[SqlParam]]) -> None: async with self.transaction() as tx: - for row in rows: - await tx.execute(sql, row) + await tx.executemany(sql, rows) @asynccontextmanager async def transaction(self) -> AsyncIterator[Transaction]: @@ -131,14 +158,14 @@ async def transaction(self) -> AsyncIterator[Transaction]: try: await conn.execute("BEGIN IMMEDIATE") except BaseException: - # A cancelled caller may leave the BEGIN running in the worker thread. - if conn.in_transaction: - await conn.execute("ROLLBACK") + # A cancelled caller may leave the BEGIN queued in the worker thread: the + # rollback queues behind it, so it always sees the transaction (if any). + await _rollback(conn) raise try: yield Transaction(conn) except BaseException: - await conn.execute("ROLLBACK") + await _rollback(conn) raise else: await conn.execute("COMMIT") diff --git a/suncore/storage/learned.py b/suncore/storage/learned.py new file mode 100644 index 0000000..b693c27 --- /dev/null +++ b/suncore/storage/learned.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Learned parameters (Phase 4 WP3): bias tables, error distributions, load profiles, ... + +One row per ``(namespace, key)`` holding a JSON object. The module that owns a namespace +defines the payload and migrates old payloads by ``schema_version``; this repository only +stores, lists and deletes them. A payload that does not parse is treated as missing (logged), +so a learned value can never stop the application - the owner then starts learning again. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any, Final + +import structlog + +from suncore.storage.db import Database + +log = structlog.get_logger(__name__) + +NS_FORECAST_BIAS: Final = "forecast_bias" +NS_NET_ERROR: Final = "net_error" +NS_LOAD_PROFILE: Final = "load_profile" +NS_BLEND_WEIGHTS: Final = "blend_weights" +NS_ML_STATE: Final = "ml_state" + +KNOWN_NAMESPACES: Final[frozenset[str]] = frozenset( + {NS_FORECAST_BIAS, NS_NET_ERROR, NS_LOAD_PROFILE, NS_BLEND_WEIGHTS, NS_ML_STATE} +) +"""The namespaces Phase 4 uses; other well-formed names are accepted (later phases).""" + +_NAME_RE: Final = re.compile(r"^[a-z][a-z0-9_]{0,63}$") +_KEY_MAX: Final = 200 + + +def _now() -> datetime: + return datetime.now(UTC) + + +@dataclass(frozen=True, slots=True) +class LearnedParam: + namespace: str + key: str + payload: dict[str, Any] + schema_version: int + updated_at: datetime + + +def _check(namespace: str, key: str | None = None) -> None: + if not _NAME_RE.match(namespace): + msg = f"invalid learned-parameter namespace {namespace!r}" + raise ValueError(msg) + if key is not None and not 0 < len(key) <= _KEY_MAX: + msg = "learned-parameter key must be 1..200 characters" + raise ValueError(msg) + + +class LearnedParamStore: + """``learned_param`` table behind typed methods.""" + + def __init__(self, db: Database, clock: Callable[[], datetime] = _now) -> None: + self._db = db + self._clock = clock + + async def put( + self, namespace: str, key: str, payload: Mapping[str, Any], *, schema_version: int = 1 + ) -> None: + """Insert or replace one parameter set.""" + _check(namespace, key) + await self._db.execute( + "INSERT INTO learned_param (namespace, key, payload, schema_version, updated_at) " + "VALUES (?, ?, ?, ?, ?) ON CONFLICT(namespace, key) DO UPDATE SET " + "payload = excluded.payload, schema_version = excluded.schema_version, " + "updated_at = excluded.updated_at", + ( + namespace, + key, + json.dumps(payload, allow_nan=False, sort_keys=True), + schema_version, + self._clock().astimezone(UTC).isoformat(), + ), + ) + + async def get(self, namespace: str, key: str) -> LearnedParam | None: + _check(namespace, key) + row = await self._db.fetchone( + "SELECT namespace, key, payload, schema_version, updated_at FROM learned_param " + "WHERE namespace = ? AND key = ?", + (namespace, key), + ) + return None if row is None else self._decode(row) + + async def in_namespace(self, namespace: str) -> list[LearnedParam]: + """All readable entries of a namespace, ordered by key.""" + _check(namespace) + rows = await self._db.fetchall( + "SELECT namespace, key, payload, schema_version, updated_at FROM learned_param " + "WHERE namespace = ? ORDER BY key", + (namespace,), + ) + decoded = (self._decode(r) for r in rows) + return [p for p in decoded if p is not None] + + async def delete(self, namespace: str, key: str) -> bool: + _check(namespace, key) + deleted = await self._db.execute( + "DELETE FROM learned_param WHERE namespace = ? AND key = ?", (namespace, key) + ) + return deleted > 0 + + async def delete_namespace(self, namespace: str) -> int: + """Forget everything learned in a namespace (e.g. a reset button); returns the count.""" + _check(namespace) + return await self._db.execute("DELETE FROM learned_param WHERE namespace = ?", (namespace,)) + + @staticmethod + def _decode(row: tuple[object, ...]) -> LearnedParam | None: + try: + data = json.loads(str(row[2])) + except json.JSONDecodeError: + data = None + if not isinstance(data, dict): + log.warning("learned_param.unreadable", namespace=row[0], key=row[1]) + return None + return LearnedParam( + namespace=str(row[0]), + key=str(row[1]), + payload=data, + schema_version=int(str(row[3])), + updated_at=datetime.fromisoformat(str(row[4])), + ) + + +__all__ = [ + "KNOWN_NAMESPACES", + "NS_BLEND_WEIGHTS", + "NS_FORECAST_BIAS", + "NS_LOAD_PROFILE", + "NS_ML_STATE", + "NS_NET_ERROR", + "LearnedParam", + "LearnedParamStore", +] diff --git a/suncore/storage/migrations/0004_plans_learned.sql b/suncore/storage/migrations/0004_plans_learned.sql new file mode 100644 index 0000000..225e17c --- /dev/null +++ b/suncore/storage/migrations/0004_plans_learned.sql @@ -0,0 +1,25 @@ +-- SPDX-License-Identifier: Apache-2.0 +-- Phase 4 (WP3): plan snapshots and learned parameters. +-- Schema 4 cannot be opened by earlier versions (they refuse newer databases): export before +-- updating. The `learned` table of 0001 stays untouched (unused since Phase 1). + +-- Compressed JSON of a computed plan (zlib); retention in suncore.storage.plans. +CREATE TABLE plan_snapshot ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL, + source TEXT NOT NULL, + schema_version INTEGER NOT NULL, + payload BLOB NOT NULL +); +CREATE INDEX plan_snapshot_created_idx ON plan_snapshot (created_at); + +-- Learned parameters: forecast_bias, net_error, load_profile, blend_weights, ml_state, ... +-- The owner of a namespace defines its payload and migrates it by schema_version. +CREATE TABLE learned_param ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + payload TEXT NOT NULL CHECK (json_valid(payload)), + schema_version INTEGER NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (namespace, key) +); diff --git a/suncore/storage/plans.py b/suncore/storage/plans.py new file mode 100644 index 0000000..eace1dc --- /dev/null +++ b/suncore/storage/plans.py @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Plan snapshots: compressed JSON payloads of computed plans (Phase 4 WP3). + +The planner (WP9) stores each plan it computes; the UI's plan history and the accuracy +bookkeeping read them back. The payload is opaque here - the plan model owns its shape and +passes its ``schema_version``. Retention keeps at most :data:`KEEP_LAST` snapshots and none +older than :data:`KEEP_DAYS` days. +""" + +from __future__ import annotations + +import json +import zlib +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any, Final + +import structlog + +from suncore.storage.db import Database + +log = structlog.get_logger(__name__) + +KEEP_LAST: Final = 200 +KEEP_DAYS: Final = 14 +MAX_LIMIT: Final = 500 +COMPRESSION_LEVEL: Final = 6 + + +def _now() -> datetime: + return datetime.now(UTC) + + +@dataclass(frozen=True, slots=True) +class PlanSnapshotMeta: + """Header of a stored snapshot (no payload).""" + + id: int + created_at: datetime + source: str + schema_version: int + + +@dataclass(frozen=True, slots=True) +class PlanSnapshot: + """A stored snapshot with its decoded payload.""" + + id: int + created_at: datetime + source: str + schema_version: int + payload: dict[str, Any] + + +def encode_payload(payload: Mapping[str, Any]) -> bytes: + """JSON (finite numbers only) compressed with zlib.""" + text = json.dumps(payload, allow_nan=False, separators=(",", ":"), sort_keys=True) + return zlib.compress(text.encode("utf-8"), COMPRESSION_LEVEL) + + +def decode_payload(blob: bytes) -> dict[str, Any]: + """Inverse of :func:`encode_payload`; ``ValueError`` if the blob is not a JSON object.""" + try: + data = json.loads(zlib.decompress(blob).decode("utf-8")) + except (zlib.error, UnicodeDecodeError, json.JSONDecodeError) as exc: + msg = f"plan snapshot payload unreadable: {exc}" + raise ValueError(msg) from exc + if not isinstance(data, dict): + msg = "plan snapshot payload is not an object" + raise ValueError(msg) + return data + + +class PlanSnapshotStore: + """``plan_snapshot`` table behind typed methods.""" + + def __init__(self, db: Database, clock: Callable[[], datetime] = _now) -> None: + self._db = db + self._clock = clock + + async def save( + self, + payload: Mapping[str, Any], + *, + source: str, + schema_version: int, + created_at: datetime | None = None, + ) -> int: + """Store a snapshot and return its id (ids grow with time).""" + when = (created_at or self._clock()).astimezone(UTC) + blob = encode_payload(payload) + async with self._db.transaction() as tx: + await tx.execute( + "INSERT INTO plan_snapshot (created_at, source, schema_version, payload) " + "VALUES (?, ?, ?, ?)", + (when.isoformat(), source, schema_version, blob), + ) + row = await tx.fetchone("SELECT last_insert_rowid()") + assert row is not None + return int(str(row[0])) + + async def get(self, snapshot_id: int) -> PlanSnapshot | None: + """The snapshot, or ``None`` if it does not exist or its payload is unreadable.""" + row = await self._db.fetchone( + "SELECT id, created_at, source, schema_version, payload FROM plan_snapshot " + "WHERE id = ?", + (snapshot_id,), + ) + return None if row is None else self._decode(row) + + async def latest(self) -> PlanSnapshot | None: + """The newest readable snapshot.""" + rows = await self._db.fetchall( + "SELECT id, created_at, source, schema_version, payload FROM plan_snapshot " + "ORDER BY id DESC LIMIT 5" + ) + for row in rows: + snapshot = self._decode(row) + if snapshot is not None: + return snapshot + return None + + async def headers(self, limit: int = 50) -> list[PlanSnapshotMeta]: + """Headers, newest first; ``limit`` is clamped to ``1..MAX_LIMIT``.""" + bounded = max(1, min(int(limit), MAX_LIMIT)) + rows = await self._db.fetchall( + "SELECT id, created_at, source, schema_version FROM plan_snapshot " + "ORDER BY id DESC LIMIT ?", + (bounded,), + ) + return [ + PlanSnapshotMeta( + id=int(str(r[0])), + created_at=datetime.fromisoformat(str(r[1])), + source=str(r[2]), + schema_version=int(str(r[3])), + ) + for r in rows + ] + + async def count(self) -> int: + row = await self._db.fetchone("SELECT COUNT(*) FROM plan_snapshot") + return int(str(row[0])) if row else 0 + + async def prune( + self, now: datetime, keep_last: int = KEEP_LAST, keep_days: int = KEEP_DAYS + ) -> int: + """Drop snapshots older than ``keep_days`` and all but the newest ``keep_last``.""" + cutoff = (now.astimezone(UTC) - timedelta(days=keep_days)).isoformat() + async with self._db.transaction() as tx: + by_age = await tx.execute("DELETE FROM plan_snapshot WHERE created_at < ?", (cutoff,)) + by_count = await tx.execute( + "DELETE FROM plan_snapshot WHERE id <= " + "COALESCE((SELECT id FROM plan_snapshot ORDER BY id DESC LIMIT 1 OFFSET ?), 0)", + (max(0, keep_last),), + ) + deleted = by_age + by_count + if deleted: + log.info("plan_snapshot.pruned", deleted=deleted) + return deleted + + @staticmethod + def _decode(row: tuple[object, ...]) -> PlanSnapshot | None: + try: + payload = decode_payload(bytes(row[4])) # type: ignore[call-overload] + except ValueError as exc: + log.warning("plan_snapshot.unreadable", snapshot_id=row[0], error=str(exc)[:200]) + return None + return PlanSnapshot( + id=int(str(row[0])), + created_at=datetime.fromisoformat(str(row[1])), + source=str(row[2]), + schema_version=int(str(row[3])), + payload=payload, + ) + + +__all__ = [ + "KEEP_DAYS", + "KEEP_LAST", + "PlanSnapshot", + "PlanSnapshotMeta", + "PlanSnapshotStore", + "decode_payload", + "encode_payload", +] diff --git a/suncore/storage/timeseries.py b/suncore/storage/timeseries.py new file mode 100644 index 0000000..5d79606 --- /dev/null +++ b/suncore/storage/timeseries.py @@ -0,0 +1,794 @@ +# SPDX-License-Identifier: Apache-2.0 +"""``timeseries.db``: measurements, forecasts and prices (ADR-0004, Phase 4 WP3). + +A separate SQLite file next to ``suncore.db``, excluded from Home Assistant backups +(``backup_exclude``) and rebuildable: the 15-minute history comes back from HA's long-term +statistics (``suncore.app.history``). It has its own migrations (``ts_migrations``) and its own +``PRAGMA user_version``; a file that is newer than the code is refused (and left untouched). + +Tables and retention (all times injected, nothing reads the wall clock): + +======================== ====================================== ============= +table content kept +======================== ====================================== ============= +``measurement`` controller resolution, one row per tick 48 h +``measurement_1m`` mean/min/max/n per minute 30 days +``measurement_15m`` mean/min/max/n per quarter hour unlimited +``forecast`` P10/P50/P90 per slot, per issue time 45 days (slots) +``price`` import/export price per slot 400 days +======================== ====================================== ============= + +Conventions: timestamps are integer seconds since the UNIX epoch (UTC); a gap in a measurement +is a missing row (never a stored zero) and reads back as ``None``; the aggregates are +sample-weighted, so ``mean * n`` adds up exactly (a 15-minute mean is the sample mean over its +minutes) and the power-times-time energy of an interval survives the downsampling. +``downsample`` is idempotent: it recomputes every complete bucket from the finer table, and +the retention cut-offs are aligned to the bucket size, so a bucket is either fully present in +its source table or gone - a re-run never aggregates half a bucket. +""" + +from __future__ import annotations + +import math +import sqlite3 +import time +from collections import Counter +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from importlib import resources +from pathlib import Path +from typing import Final, Literal + +import structlog + +from suncore.core.measurements import is_percent_key +from suncore.core.series import ( + MAX_POINTS, + PriceIncludes, + PriceSeries, + QuantileSeries, + SeriesUnit, + TimeRange, + TimeSeries, +) +from suncore.storage.db import Database, Row, SqlParam +from suncore.storage.errors import MigrationError, SchemaTooNewError +from suncore.storage.migrations import Migration, load_migrations + +log = structlog.get_logger(__name__) + +TS_FILENAME: Final = "timeseries.db" + +META_HISTORY_REBUILT: Final = "history_rebuilt_at" +"""Set (ISO time) once the rebuild from HA's statistics succeeded; it dies with the file.""" + +RAW_KEEP: Final = timedelta(hours=48) +MINUTE_KEEP: Final = timedelta(days=30) +FORECAST_KEEP: Final = timedelta(days=45) +PRICE_KEEP: Final = timedelta(days=400) + +VACUUM_MIN_FREE_BYTES: Final = 8 * 1024 * 1024 +VACUUM_MIN_FREE_FRACTION: Final = 0.25 +"""``VACUUM`` rewrites the file; it runs only if at least this much (and this share) is free.""" + +MINUTE_S: Final = 60 +QUARTER_S: Final = 900 + +Resolution = Literal["1m", "15m"] +ForecastKind = Literal["pv", "load", "price"] +_STEP_OF: Final[Mapping[str, int]] = {"1m": MINUTE_S, "15m": QUARTER_S} +_TABLE_OF: Final[Mapping[str, str]] = {"1m": "measurement_1m", "15m": "measurement_15m"} + +MeasurementRow = tuple[str, datetime, float | None] +"""``(key, timestamp, value)``; a ``None`` value is a gap and is not stored.""" + + +# --- opening ------------------------------------------------------------------------ + + +def timeseries_migrations() -> list[Migration]: + """The migrations of ``timeseries.db``.""" + return load_migrations(resources.files("suncore.storage.ts_migrations")) + + +async def open_timeseries(path: Path | str) -> Database: + """Open (creating, migrating) ``timeseries.db``; raises ``SchemaTooNewError`` if too new.""" + return await Database.open(path, migrations=timeseries_migrations()) + + +KEEP_CORRUPT_FILES: Final = 2 +"""How many moved-aside corrupt databases are kept (older ones are deleted).""" + +_CORRUPT_CODES: Final = frozenset({sqlite3.SQLITE_CORRUPT, sqlite3.SQLITE_NOTADB}) +_CORRUPT_TEXT: Final = ("file is not a database", "malformed") + + +def _is_corruption(exc: BaseException) -> bool: + """Unambiguous corruption only; locked, busy, full, I/O and permission errors are not.""" + seen: BaseException | None = exc + while seen is not None: + code = getattr(seen, "sqlite_errorcode", None) + if isinstance(code, int) and code & 0xFF in _CORRUPT_CODES: + return True + if isinstance(seen, sqlite3.DatabaseError) and any( + t in str(seen).lower() for t in _CORRUPT_TEXT + ): + return True + seen = seen.__cause__ + return False + + +async def _quick_check(db: Database) -> bool: + row = await db.fetchone("PRAGMA quick_check(1)") + return row is not None and str(row[0]).lower() == "ok" + + +async def open_timeseries_or_recover( + path: Path | str, *, now: Callable[[], float] = time.time +) -> Database | None: + """Open ``timeseries.db`` for the application; ``None`` = run without time series. + + The file is rebuildable, but only *unambiguous corruption* (SQLite's "not a database" or + "malformed" errors, a failing ``PRAGMA quick_check``) moves it aside, together with its + ``-wal`` and ``-shm``, as ``timeseries.db.corrupt-`` (the newest + :data:`KEEP_CORRUPT_FILES` are kept) before a fresh file is created. Anything else - a + locked or busy file, a full disk, an I/O or permission error, a migration that fails for + another reason - is logged and the application runs without time series; the next start + tries again and nothing is moved. A file that is *newer* than this code is never + touched either. Control never depends on this file. + """ + target = Path(path) + for attempt in (1, 2): + try: + db = await open_timeseries(target) + except SchemaTooNewError as exc: + log.error("timeseries.schema_too_new", path=str(target), error=str(exc)) + return None + except (sqlite3.Error, MigrationError, OSError) as exc: + corrupt = _is_corruption(exc) + log.error("timeseries.open_failed", path=str(target), corrupt=corrupt, error=str(exc)) + else: + try: + if await _quick_check(db): + return db + corrupt = True + log.error("timeseries.quick_check_failed", path=str(target)) + except sqlite3.Error as exc: + corrupt = _is_corruption(exc) + log.error("timeseries.quick_check_error", path=str(target), error=str(exc)) + await db.close() + if not corrupt or attempt == 2: + return None + try: + _move_aside(target, now()) + except OSError: + log.exception("timeseries.move_aside_failed", path=str(target)) + return None + return None + + +def _move_aside(path: Path, now: float) -> None: + """Rename the database and its WAL/SHM to ``.corrupt-``; keep the newest two.""" + stamp = datetime.fromtimestamp(now, UTC).strftime("%Y%m%dT%H%M%SZ") + moved = path.with_name(f"{path.name}.corrupt-{stamp}") + for suffix in ("", "-wal", "-shm"): + source = path.with_name(path.name + suffix) + if source.exists(): + source.replace(moved.with_name(moved.name + suffix)) + sets = sorted( + p + for p in path.parent.glob(f"{path.name}.corrupt-*") + if not p.name.endswith(("-wal", "-shm")) + ) + for old in sets[:-KEEP_CORRUPT_FILES]: + for suffix in ("", "-wal", "-shm"): + old.with_name(old.name + suffix).unlink(missing_ok=True) + + +# --- helpers ------------------------------------------------------------------------ + + +def _epoch(moment: datetime) -> int: + if moment.tzinfo is None or moment.utcoffset() is None: + msg = "timestamps must be timezone-aware" + raise ValueError(msg) + return math.floor(moment.timestamp()) + + +def _from_epoch(seconds: int) -> datetime: + return datetime.fromtimestamp(seconds, UTC) + + +def _floor(seconds: int, step: int) -> int: + return seconds - seconds % step + + +def _num(value: object) -> float: + assert isinstance(value, int | float) + return float(value) + + +def _int(value: object) -> int: + assert isinstance(value, int | float) + return int(value) + + +def _opt(value: float | None) -> float | None: + return None if value is None or not math.isfinite(value) else value + + +def _opt_num(value: object) -> float | None: + return None if value is None else _num(value) + + +def _default_unit(key: str) -> SeriesUnit: + return SeriesUnit.PERCENT if is_percent_key(key) else SeriesUnit.W + + +# --- reports ------------------------------------------------------------------------ + + +@dataclass(frozen=True, slots=True) +class DownsampleReport: + minute_rows: int + quarter_rows: int + + +@dataclass(frozen=True, slots=True) +class RetentionReport: + raw: int + minute: int + forecast: int + price: int + + @property + def total(self) -> int: + return self.raw + self.minute + self.forecast + self.price + + +@dataclass(frozen=True, slots=True) +class MaintenanceReport: + downsample: DownsampleReport + retention: RetentionReport + vacuumed: bool + size_bytes: int + + +@dataclass(frozen=True, slots=True) +class TimeseriesSize: + """Size of ``timeseries.db`` for the diagnostics page.""" + + file_bytes: int + wal_bytes: int + free_bytes: int + """Pages inside the file that hold no data (reclaimed by ``VACUUM``).""" + rows: Mapping[str, int] + + @property + def total_bytes(self) -> int: + return self.file_bytes + self.wal_bytes + + +# --- the repository ----------------------------------------------------------------- + + +class TimeseriesStore: + """Repository over ``timeseries.db`` with typed methods and core series as results.""" + + def __init__(self, db: Database) -> None: + self._db = db + + @property + def database(self) -> Database: + return self._db + + # --- measurements ----------------------------------------------------------- + + async def append_measurements(self, rows: Iterable[MeasurementRow]) -> int: + """Store measurements in one batch; gaps (``None``, NaN, inf) are skipped. + + A second value for the same ``(key, second)`` replaces the first. Returns the number + of rows written. + """ + batch: list[Sequence[SqlParam]] = [] + for key, moment, value in rows: + checked = _opt(value) + if checked is not None: + batch.append((key, _epoch(moment), checked)) + if batch: + await self._db.executemany( + "INSERT OR REPLACE INTO measurement (key, ts, value) VALUES (?, ?, ?)", batch + ) + return len(batch) + + async def read_raw(self, key: str, window: TimeRange) -> list[tuple[datetime, float]]: + """Controller-resolution samples in ``window`` (oldest first).""" + rows = await self._db.fetchall( + "SELECT ts, value FROM measurement WHERE key = ? AND ts >= ? AND ts < ? ORDER BY ts", + (key, _epoch(window.start), _ceil_epoch(window.end)), + ) + return [(_from_epoch(_int(r[0])), _num(r[1])) for r in rows] + + async def read_series( + self, + key: str, + window: TimeRange, + resolution: Resolution = "15m", + unit: SeriesUnit | None = None, + *, + include_recent: bool = True, + ) -> TimeSeries: + """The mean of ``key`` per minute or quarter hour as a regular series. + + The grid starts at the bucket containing ``window.start`` and ends at the bucket + containing the last instant of the window. A bucket without data is ``None``. + With ``include_recent`` (15 minutes only), buckets that the nightly job has not + aggregated yet are computed on the fly from the 1-minute and then the raw rows, so + yesterday's data is available at once; a bucket still in progress is included with + what has been measured so far (end the window at the last complete bucket if that + matters). + """ + step = _STEP_OF[resolution] + first = _floor(_epoch(window.start), step) + end = _ceil_epoch(window.end) + count = max(1, -(-(end - first) // step)) + if count > MAX_POINTS: + msg = f"read_series: {count} points exceed the limit of {MAX_POINTS}" + raise ValueError(msg) + last = first + count * step + means: dict[int, float] = {} + if include_recent and resolution == "15m": + means.update(await self._recent_quarters(key, first, last)) + table = _TABLE_OF[resolution] + derived_col = ", derived" if resolution == "15m" else ", 0" + rows = await self._db.fetchall( + f"SELECT ts, mean{derived_col} FROM {table} WHERE key = ? AND ts >= ? AND ts < ?", + (key, first, last), + ) + for row in rows: + ts_ = _int(row[0]) + if _int(row[2]) and ts_ in means: + continue # a derived (rebuilt) value never beats what was measured + means[ts_] = _num(row[1]) + values = tuple(means.get(first + i * step) for i in range(count)) + return TimeSeries( + start=_from_epoch(first), + step=timedelta(seconds=step), + values=values, + unit=unit or _default_unit(key), + ) + + async def _recent_quarters(self, key: str, first: int, last: int) -> dict[int, float]: + """Quarter-hour means from the finer tables (a bucket is whole in one of them).""" + out: dict[int, float] = {} + raw = await self._db.fetchall( + "SELECT (ts / 900) * 900 AS b, AVG(value) FROM measurement " + "WHERE key = ? AND ts >= ? AND ts < ? GROUP BY b", + (key, first, last), + ) + out.update({_int(r[0]): _num(r[1]) for r in raw}) + minutes = await self._db.fetchall( + "SELECT (ts / 900) * 900 AS b, SUM(mean * n) / SUM(n) FROM measurement_1m " + "WHERE key = ? AND ts >= ? AND ts < ? GROUP BY b", + (key, first, last), + ) + out.update({_int(r[0]): _num(r[1]) for r in minutes}) + return out + + async def store_quarter_hours( + self, key: str, series: TimeSeries, *, derived: bool = False + ) -> int: + """Write a 15-minute series into ``measurement_15m`` (``None`` slots are skipped). + + ``derived=True`` marks values rebuilt from Home Assistant's statistics: they fill + free slots and replace earlier derived rows but never overwrite measured ones. + Returns the number of slots offered. + """ + if series.step != timedelta(seconds=QUARTER_S): + msg = "store_quarter_hours: needs a 15-minute series" + raise ValueError(msg) + batch: list[Sequence[SqlParam]] = [] + for begin, _end, value in series.slots(): + checked = _opt(value) + if checked is not None: + batch.append((key, _epoch(begin), checked, checked, checked, 1 if derived else 0)) + if not batch: + return 0 + guard = "WHERE measurement_15m.derived = 1" if derived else "" + await self._db.executemany( + "INSERT INTO measurement_15m (key, ts, mean, min, max, n, derived) " + "VALUES (?, ?, ?, ?, ?, 1, ?) ON CONFLICT(key, ts) DO UPDATE SET " + "mean = excluded.mean, min = excluded.min, max = excluded.max, " + f"n = excluded.n, derived = excluded.derived {guard}", + batch, + ) + return len(batch) + + async def get_meta(self, key: str) -> str | None: + """A fact about this file's content (see ``META_HISTORY_REBUILT``).""" + row = await self._db.fetchone("SELECT value FROM meta WHERE key = ?", (key,)) + return None if row is None else str(row[0]) + + async def set_meta(self, key: str, value: str) -> None: + await self._db.execute( + "INSERT INTO meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (key, value), + ) + + async def has_quarter_history(self) -> bool: + """Whether ``measurement_15m`` holds any row (the rebuild runs when it does not).""" + return await self._db.fetchone("SELECT 1 FROM measurement_15m LIMIT 1") is not None + + # --- forecasts ---------------------------------------------------------------- + + async def store_forecast( + self, + source_id: str, + kind: ForecastKind, + issued_at: datetime, + forecast: QuantileSeries | TimeSeries, + ) -> int: + """Store one issue of a forecast; an earlier store of the same issue is replaced. + + Every slot is stored, also an unknown one (all NULL), so the series reads back with + the same start and length. A plain ``TimeSeries`` fills ``p50`` only. + """ + p10: TimeSeries | None + p90: TimeSeries | None + if isinstance(forecast, QuantileSeries): + p10, p50, p90 = forecast.p10, forecast.p50, forecast.p90 + else: + p10 = p90 = None + p50 = forecast + step_s = int(p50.step.total_seconds()) + issued = _epoch(issued_at) + rows: list[Sequence[SqlParam]] = [] + for i, begin in enumerate(p50.slot_start(j) for j in range(len(p50))): + rows.append( + ( + source_id, + kind, + issued, + _epoch(begin), + step_s, + p50.unit.value, + None if p10 is None else p10.values[i], + p50.values[i], + None if p90 is None else p90.values[i], + ) + ) + async with self._db.transaction() as tx: + await tx.execute( + "DELETE FROM forecast WHERE source_id = ? AND kind = ? AND issued_at = ?", + (source_id, kind, issued), + ) + await tx.executemany( + "INSERT INTO forecast (source_id, kind, issued_at, slot_start, step_s, unit, " + "p10, p50, p90) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + rows, + ) + return len(rows) + + async def read_forecast( + self, + source_id: str, + kind: ForecastKind, + issued_at: datetime, + window: TimeRange | None = None, + ) -> QuantileSeries | None: + """One issue of a forecast (optionally clipped to ``window``); ``None`` if absent.""" + lo, hi = _bounds(window) + rows = await self._db.fetchall( + "SELECT slot_start, step_s, unit, p10, p50, p90 FROM forecast " + "WHERE source_id = ? AND kind = ? AND issued_at = ? AND slot_start >= ? " + "AND slot_start < ? ORDER BY slot_start", + (source_id, kind, _epoch(issued_at), lo, hi), + ) + return _quantile_series(rows) + + async def latest_forecast_before( + self, + source_id: str, + kind: ForecastKind, + before: datetime, + window: TimeRange | None = None, + ) -> QuantileSeries | None: + """Per slot the newest value issued at or before ``before`` (for the accuracy). + + To judge a forecast at lead time L, ask for ``before = slot_start - L``: every slot + then carries what was known L earlier, whichever issue that was. The result spans + the first to the last slot found; slots no issue covered are ``None``. + """ + lo, hi = _bounds(window) + limit = _epoch(before) + rows = await self._db.fetchall( + "SELECT f.slot_start, f.step_s, f.unit, f.p10, f.p50, f.p90 FROM forecast f " + "JOIN (SELECT slot_start, MAX(issued_at) AS issued FROM forecast " + "WHERE source_id = ? AND kind = ? AND issued_at <= ? AND slot_start >= ? " + "AND slot_start < ? GROUP BY slot_start) m " + "ON f.slot_start = m.slot_start AND f.issued_at = m.issued " + "WHERE f.source_id = ? AND f.kind = ? ORDER BY f.slot_start", + (source_id, kind, limit, lo, hi, source_id, kind), + ) + return _quantile_series(rows) + + async def forecast_issues( + self, source_id: str, kind: ForecastKind, since: datetime | None = None + ) -> list[datetime]: + """Issue times stored for a source, oldest first.""" + rows = await self._db.fetchall( + "SELECT DISTINCT issued_at FROM forecast WHERE source_id = ? AND kind = ? " + "AND issued_at >= ? ORDER BY issued_at", + (source_id, kind, 0 if since is None else _epoch(since)), + ) + return [_from_epoch(_int(r[0])) for r in rows] + + # --- prices ------------------------------------------------------------------- + + async def store_prices(self, prices: PriceSeries) -> int: + """Store prices; rows of the same source that overlap their time span are replaced.""" + step_s = int(prices.step.total_seconds()) + start = _epoch(prices.start) + end = start + step_s * len(prices) + export = prices.export_price + rows: list[Sequence[SqlParam]] = [ + ( + prices.source_id, + _epoch(prices.import_price.slot_start(i)), + step_s, + prices.price_includes, + prices.import_price.values[i], + None if export is None else export.values[i], + ) + for i in range(len(prices)) + ] + async with self._db.transaction() as tx: + await tx.execute( + "DELETE FROM price WHERE source_id = ? AND slot_start < ? " + "AND slot_start + step_s > ?", + (prices.source_id, end, start), + ) + await tx.executemany( + "INSERT INTO price (source_id, slot_start, step_s, includes, import_price, " + "export_price) VALUES (?, ?, ?, ?, ?, ?)", + rows, + ) + return len(rows) + + async def read_prices(self, source_id: str, window: TimeRange) -> list[PriceSeries]: + """Stored prices overlapping ``window`` as regular segments (a change of step, + ``price_includes`` or a gap that is not a whole number of slots starts a new one). + + A segment whose export prices are all unknown has ``export_price = None``. + """ + rows = await self._db.fetchall( + "SELECT slot_start, step_s, includes, import_price, export_price FROM price " + "WHERE source_id = ? AND slot_start < ? AND slot_start + step_s > ? " + "ORDER BY slot_start", + (source_id, _ceil_epoch(window.end), _epoch(window.start)), + ) + segments: list[list[Row]] = [] + for row in rows: + if segments and _continues(segments[-1], row): + segments[-1].append(row) + else: + segments.append([row]) + return [_price_segment(source_id, segment) for segment in segments] + + # --- maintenance -------------------------------------------------------------- + + async def downsample(self, now: datetime) -> DownsampleReport: + """Aggregate every complete quarter hour into the 1-minute and 15-minute tables. + + Idempotent (see module doc). The 1-minute tier is written only for buckets that end at + or before ``floor(now, 15 min)``, never for a partial quarter: a bucket is then either + entirely in ``measurement_1m`` or entirely in the raw rows, which keeps + :meth:`read_series` from mixing the two inside one quarter hour. + """ + now_s = _epoch(now) + async with self._db.transaction() as tx: + minutes = await tx.execute( + "INSERT INTO measurement_1m (key, ts, mean, min, max, n) " + "SELECT key, (ts / 60) * 60 AS b, AVG(value), MIN(value), MAX(value), COUNT(*) " + "FROM measurement WHERE ts < ? GROUP BY key, b " + "ON CONFLICT(ts, key) DO UPDATE SET mean = excluded.mean, min = excluded.min, " + "max = excluded.max, n = excluded.n", + (_floor(now_s, QUARTER_S),), + ) + quarters = await tx.execute( + "INSERT INTO measurement_15m (key, ts, mean, min, max, n, derived) " + "SELECT key, (ts / 900) * 900 AS b, SUM(mean * n) / SUM(n), MIN(min), MAX(max), " + "SUM(n), 0 FROM measurement_1m WHERE ts < ? GROUP BY key, b " + "ON CONFLICT(key, ts) DO UPDATE SET mean = excluded.mean, min = excluded.min, " + "max = excluded.max, n = excluded.n, derived = 0", + (_floor(now_s, QUARTER_S),), + ) + return DownsampleReport(minute_rows=minutes, quarter_rows=quarters) + + async def apply_retention(self, now: datetime) -> RetentionReport: + """Delete what is older than the retention of its table (cut-offs bucket-aligned).""" + now_s = _epoch(now) + raw_cut = _floor(now_s - int(RAW_KEEP.total_seconds()), MINUTE_S) + minute_cut = _floor(now_s - int(MINUTE_KEEP.total_seconds()), QUARTER_S) + forecast_cut = now_s - int(FORECAST_KEEP.total_seconds()) + price_cut = now_s - int(PRICE_KEEP.total_seconds()) + async with self._db.transaction() as tx: + raw = await tx.execute("DELETE FROM measurement WHERE ts < ?", (raw_cut,)) + minute = await tx.execute("DELETE FROM measurement_1m WHERE ts < ?", (minute_cut,)) + forecast = await tx.execute( + "DELETE FROM forecast WHERE slot_start < ?", (forecast_cut,) + ) + price = await tx.execute("DELETE FROM price WHERE slot_start < ?", (price_cut,)) + return RetentionReport(raw=raw, minute=minute, forecast=forecast, price=price) + + async def maintain(self, now: datetime) -> MaintenanceReport: + """The nightly job: downsample first (nothing is deleted before it is aggregated), + then retention, then ``VACUUM`` if enough of the file is free.""" + downsample = await self.downsample(now) + retention = await self.apply_retention(now) + vacuumed = await self._vacuum_if_worth_it() + size = await self.size_report() + report = MaintenanceReport( + downsample=downsample, + retention=retention, + vacuumed=vacuumed, + size_bytes=size.total_bytes, + ) + log.info( + "timeseries.maintained", + minute_rows=downsample.minute_rows, + quarter_rows=downsample.quarter_rows, + deleted=retention.total, + vacuumed=vacuumed, + size_bytes=report.size_bytes, + ) + return report + + async def _free_and_total(self) -> tuple[int, int]: + page_size = await self._pragma("page_size") + return ( + await self._pragma("freelist_count") * page_size, + await self._pragma("page_count") * page_size, + ) + + async def _pragma(self, name: str) -> int: + row = await self._db.fetchone(f"PRAGMA {name}") + return _int(row[0]) if row else 0 + + async def _vacuum_if_worth_it(self) -> bool: + free, total = await self._free_and_total() + if free < VACUUM_MIN_FREE_BYTES or total == 0 or free / total < VACUUM_MIN_FREE_FRACTION: + return False + await self._db.execute("VACUUM") + await self._db.fetchone("PRAGMA wal_checkpoint(TRUNCATE)") + return True + + async def size_report(self) -> TimeseriesSize: + """File, WAL and free bytes plus the row count per table.""" + path = Path(self._db.path) + free, _total = await self._free_and_total() + rows: dict[str, int] = {} + for table in ("measurement", "measurement_1m", "measurement_15m", "forecast", "price"): + row = await self._db.fetchone(f"SELECT COUNT(*) FROM {table}") + rows[table] = _int(row[0]) if row else 0 + return TimeseriesSize( + file_bytes=_size(path), + wal_bytes=_size(path.with_name(path.name + "-wal")), + free_bytes=free, + rows=rows, + ) + + +# --- row conversion ----------------------------------------------------------------- + + +def _size(path: Path) -> int: + try: + return path.stat().st_size + except OSError: + return 0 + + +def _ceil_epoch(moment: datetime) -> int: + seconds = moment.timestamp() + _epoch(moment) # validates awareness + return math.ceil(seconds) + + +def _bounds(window: TimeRange | None) -> tuple[int, int]: + if window is None: + return 0, 2**62 + return _epoch(window.start), _ceil_epoch(window.end) + + +def _quantile_series(rows: Sequence[Row]) -> QuantileSeries | None: + """Rebuild a series from slot rows; the most frequent step wins, gaps become ``None``.""" + if not rows: + return None + steps = Counter(_int(r[1]) for r in rows) + step_s = steps.most_common(1)[0][0] + if len(steps) > 1: + log.warning("timeseries.mixed_forecast_steps", steps=sorted(steps), used=step_s) + kept = [r for r in rows if _int(r[1]) == step_s] + units = Counter(str(r[2]) for r in kept) + unit = SeriesUnit(units.most_common(1)[0][0]) + kept = [r for r in kept if SeriesUnit(str(r[2])) is unit] + first = _int(kept[0][0]) + count = (_int(kept[-1][0]) - first) // step_s + 1 + grids: tuple[list[float | None], list[float | None], list[float | None]] = ( + [None] * count, + [None] * count, + [None] * count, + ) + for row in kept: + offset = _int(row[0]) - first + if offset % step_s: # off the grid of the chosen step: ignore the stray row + continue + index = offset // step_s + for column in range(3): + grids[column][index] = _opt_num(row[3 + column]) + start = _from_epoch(first) + step = timedelta(seconds=step_s) + p10, p50, p90 = (TimeSeries(start=start, step=step, values=tuple(g), unit=unit) for g in grids) + return QuantileSeries(p10=p10, p50=p50, p90=p90) + + +def _continues(segment: Sequence[Row], row: Row) -> bool: + last = segment[-1] + if _int(row[1]) != _int(last[1]) or str(row[2]) != str(last[2]): + return False + gap = _int(row[0]) - (_int(last[0]) + _int(last[1])) + return gap >= 0 and gap % _int(last[1]) == 0 + + +def _price_segment(source_id: str, segment: Sequence[Row]) -> PriceSeries: + step_s = _int(segment[0][1]) + first = _int(segment[0][0]) + count = (_int(segment[-1][0]) - first) // step_s + 1 + imports: list[float | None] = [None] * count + exports: list[float | None] = [None] * count + for row in segment: + index = (_int(row[0]) - first) // step_s + imports[index] = _opt_num(row[3]) + exports[index] = _opt_num(row[4]) + start = _from_epoch(first) + step = timedelta(seconds=step_s) + includes: PriceIncludes = "all_in" if str(segment[0][2]) == "all_in" else "spot_only" + export = ( + None + if all(v is None for v in exports) + else TimeSeries(start=start, step=step, values=tuple(exports), unit=SeriesUnit.PRICE) + ) + return PriceSeries( + source_id=source_id, + price_includes=includes, + import_price=TimeSeries( + start=start, step=step, values=tuple(imports), unit=SeriesUnit.PRICE + ), + export_price=export, + ) + + +__all__ = [ + "FORECAST_KEEP", + "META_HISTORY_REBUILT", + "MINUTE_KEEP", + "PRICE_KEEP", + "RAW_KEEP", + "TS_FILENAME", + "VACUUM_MIN_FREE_BYTES", + "VACUUM_MIN_FREE_FRACTION", + "DownsampleReport", + "ForecastKind", + "MaintenanceReport", + "MeasurementRow", + "Resolution", + "RetentionReport", + "TimeseriesSize", + "TimeseriesStore", + "open_timeseries", + "open_timeseries_or_recover", + "timeseries_migrations", +] diff --git a/suncore/storage/ts_migrations/0001_initial.sql b/suncore/storage/ts_migrations/0001_initial.sql new file mode 100644 index 0000000..7bbf905 --- /dev/null +++ b/suncore/storage/ts_migrations/0001_initial.sql @@ -0,0 +1,71 @@ +-- SPDX-License-Identifier: Apache-2.0 +-- timeseries.db (ADR-0004, Phase 4 WP3). All timestamps are integer seconds since the UNIX +-- epoch (UTC); a gap is the absence of a row, never a stored zero. + +-- Controller resolution, kept 48 h. The primary key is (ts, key): the writes of one flush are +-- consecutive in time, so they land on the last pages of the table instead of one page per key +-- (about 5x fewer WAL bytes per flush on an SD card). Per-key range reads scan the ts range. +CREATE TABLE measurement ( + key TEXT NOT NULL, + ts INTEGER NOT NULL, + value REAL NOT NULL, + PRIMARY KEY (ts, key) +) WITHOUT ROWID; + +-- 1-minute aggregates (ts = start of the minute), kept 30 days; only complete quarter hours. +CREATE TABLE measurement_1m ( + key TEXT NOT NULL, + ts INTEGER NOT NULL, + mean REAL NOT NULL, + min REAL NOT NULL, + max REAL NOT NULL, + n INTEGER NOT NULL CHECK (n > 0), + PRIMARY KEY (ts, key) +) WITHOUT ROWID; + +-- 15-minute aggregates (ts = start of the quarter hour), kept for ever. `derived` = 1 marks +-- rows rebuilt from Home Assistant's hourly statistics (a step-hold of the hourly mean). +CREATE TABLE measurement_15m ( + key TEXT NOT NULL, + ts INTEGER NOT NULL, + mean REAL NOT NULL, + min REAL NOT NULL, + max REAL NOT NULL, + n INTEGER NOT NULL CHECK (n > 0), + derived INTEGER NOT NULL DEFAULT 0 CHECK (derived IN (0, 1)), + PRIMARY KEY (key, ts) +) WITHOUT ROWID; + +-- Forecasts versioned by issue time. p10/p50/p90 are NULL where the value is unknown; +-- a price-like single value is stored in p50 (kind = 'price'). +CREATE TABLE forecast ( + source_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('pv', 'load', 'price')), + issued_at INTEGER NOT NULL, + slot_start INTEGER NOT NULL, + step_s INTEGER NOT NULL CHECK (step_s > 0), + unit TEXT NOT NULL, + p10 REAL, + p50 REAL, + p90 REAL, + PRIMARY KEY (source_id, kind, issued_at, slot_start) +) WITHOUT ROWID; +CREATE INDEX forecast_slot_idx ON forecast (source_id, kind, slot_start); + +-- Prices by slot; a later store of the same time span replaces the earlier rows. +CREATE TABLE price ( + source_id TEXT NOT NULL, + slot_start INTEGER NOT NULL, + step_s INTEGER NOT NULL CHECK (step_s > 0), + includes TEXT NOT NULL CHECK (includes IN ('spot_only', 'all_in')), + import_price REAL, + export_price REAL, + PRIMARY KEY (source_id, slot_start) +) WITHOUT ROWID; + +-- Small facts about the content of this file, e.g. that the history was rebuilt from Home +-- Assistant's statistics. They live here, not in suncore.db: a lost timeseries.db must lose them. +CREATE TABLE meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +) WITHOUT ROWID; diff --git a/suncore/storage/ts_migrations/__init__.py b/suncore/storage/ts_migrations/__init__.py new file mode 100644 index 0000000..5044629 --- /dev/null +++ b/suncore/storage/ts_migrations/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Ordered SQL migrations of ``timeseries.db`` (same file format as ``storage.migrations``). + +``timeseries.db`` has its own ``PRAGMA user_version``; it is excluded from backups and can be +rebuilt, so a database that is newer than the code is left untouched and not opened. +""" diff --git a/tests/app/test_history.py b/tests/app/test_history.py new file mode 100644 index 0000000..937743f --- /dev/null +++ b/tests/app/test_history.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Combination of the site's statistics into measurement series (rebuild, Phase 4 WP3).""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from suncore.app.history import assemble_site_series, rebuild_history, site_statistic_specs +from suncore.core.measurements import ( + SITE_BATTERY_W, + SITE_GRID_W, + SITE_HOUSE_W, + SITE_PV_W, + SITE_SOC_PCT, +) +from suncore.core.series import SeriesUnit, TimeRange, TimeSeries +from suncore.core.site import BatteryConfig, EntityRef, GridConfig, PvArray, SiteConfig +from suncore.storage import open_timeseries +from suncore.storage.timeseries import TimeseriesStore + +T0 = datetime(2026, 10, 10, tzinfo=UTC) +Q = timedelta(minutes=15) + + +def ts(values: list[float | None], start: datetime = T0) -> TimeSeries: + return TimeSeries(start=start, step=Q, values=tuple(values), unit=SeriesUnit.W) + + +def ref(name: str, *, invert: bool = False) -> EntityRef: + return EntityRef(entity_id=f"sensor.{name}", invert_sign=invert) + + +def full_site(*, house: bool = False) -> SiteConfig: + return SiteConfig( + grid=GridConfig(power=ref("grid", invert=True)), + pv=[ + PvArray(name="A", power=ref("pv_a")), + PvArray(name="B", power=ref("pv_b")), + PvArray(name="C"), + ], + battery=BatteryConfig(power=ref("bat"), soc=ref("soc")), + house=ref("house") if house else None, + ) + + +def test_specs_cover_every_mapped_power_entity_and_the_soc() -> None: + specs = site_statistic_specs(full_site(house=True)) + by_key = {s.key: s for s in specs} + assert set(by_key) == {"grid", "pv0", "pv1", "battery", "soc", "house"} # array C has no sensor + assert by_key["grid"].invert is True + assert by_key["soc"].unit is SeriesUnit.PERCENT + assert all(s.kind == "mean" for s in specs) + + +def test_pair_grid_uses_two_statistics_and_subtracts() -> None: + site = SiteConfig( + grid=GridConfig(import_power=ref("imp"), export_power=ref("exp")), + pv=[PvArray(name="A", power=ref("pv"))], + ) + assert {s.key for s in site_statistic_specs(site)} == {"grid_import", "grid_export", "pv0"} + out = assemble_site_series( + site, + { + "grid_import": ts([500.0, 0.0]), + "grid_export": ts([0.0, 300.0]), + "pv0": ts([100.0, 900.0]), + }, + ) + assert out[SITE_GRID_W].values == (500.0, -300.0) + # no battery: house = pv + grid + assert out[SITE_HOUSE_W].values == (600.0, 600.0) + assert SITE_BATTERY_W not in out + assert SITE_SOC_PCT not in out + + +def test_house_is_derived_like_the_live_balance() -> None: + fetched = { + "grid": ts([200.0, -100.0]), + "pv0": ts([1000.0, 800.0]), + "pv1": ts([500.0, 200.0]), + "battery": ts([300.0, -50.0]), + "soc": ts([40.0, 41.0]), + } + out = assemble_site_series(full_site(), fetched) + assert out[SITE_PV_W].values == (1500.0, 1000.0) + # pv + grid - battery + assert out[SITE_HOUSE_W].values == (1400.0, 950.0) + assert out[SITE_SOC_PCT].values == (40.0, 41.0) + + +def test_a_configured_house_sensor_wins() -> None: + out = assemble_site_series( + full_site(house=True), + { + "grid": ts([1.0]), + "pv0": ts([1.0]), + "pv1": ts([1.0]), + "battery": ts([0.0]), + "house": ts([42.0]), + }, + ) + assert out[SITE_HOUSE_W].values == (42.0,) + + +def test_missing_parts_become_gaps_not_zeros() -> None: + fetched = { + "grid": ts([200.0, 300.0, 400.0]), + "pv0": ts([1000.0, None, 1000.0]), + "pv1": ts([500.0, 500.0, 500.0]), + "battery": ts([0.0, 0.0], start=T0 + Q), # shorter and later: only slots 1 and 2 overlap + } + out = assemble_site_series(full_site(), fetched) + assert out[SITE_PV_W].values == (1500.0, None, 1500.0) + assert out[SITE_HOUSE_W].values == (None, None, 1900.0) + # a missing array sensor makes the whole PV sum unknown + partial = assemble_site_series( + full_site(), {"grid": ts([1.0]), "pv0": ts([1.0]), "battery": ts([0.0])} + ) + assert SITE_PV_W not in partial + assert SITE_HOUSE_W not in partial + + +def test_no_pv_and_no_battery_house_is_the_grid() -> None: + site = SiteConfig(grid=GridConfig(power=ref("grid"))) + out = assemble_site_series(site, {"grid": ts([700.0])}) + assert out == {SITE_GRID_W: ts([700.0]), SITE_HOUSE_W: ts([700.0])} + + +async def test_rebuild_stores_derived_rows_and_reports_missing_ids(tmp_path) -> None: # type: ignore[no-untyped-def] + from suncore.ha.connection import HAConnection # noqa: PLC0415 + + class Stats: + async def statistics_during_period(self, ids, *_args, **_kw): # type: ignore[no-untyped-def] + hour = int(T0.timestamp() * 1000) + rows = { + "sensor.grid": [{"start": hour, "end": hour + 3_600_000, "mean": -400.0}], + "sensor.house": [{"start": hour, "end": hour + 3_600_000, "mean": 900.0}], + } + return {i: rows[i] for i in ids if i in rows} + + site = SiteConfig(grid=GridConfig(power=ref("grid", invert=True)), house=ref("house")) + async with await open_timeseries(tmp_path / "t.db") as db: + store = TimeseriesStore(db) + connection: HAConnection = Stats() # type: ignore[assignment] + result = await rebuild_history(connection, store, site, T0 + timedelta(hours=1), days=1) + assert result.stored == {SITE_GRID_W: 4, SITE_HOUSE_W: 4} + assert result.total == 8 + assert result.missing == () + grid = await store.read_series( + SITE_GRID_W, TimeRange(start=T0, end=T0 + timedelta(hours=1)) + ) + assert grid.values == (400.0,) * 4 # invert_sign applied + assert await db.fetchone("SELECT MIN(derived), MAX(derived) FROM measurement_15m") == (1, 1) + # the energy of the hour is kept: 900 W x 1 h + house = await store.read_series( + SITE_HOUSE_W, TimeRange(start=T0, end=T0 + timedelta(hours=1)) + ) + assert house.total() == pytest.approx(900.0) diff --git a/tests/app/test_timeseries_runtime.py b/tests/app/test_timeseries_runtime.py new file mode 100644 index 0000000..7cfd5f7 --- /dev/null +++ b/tests/app/test_timeseries_runtime.py @@ -0,0 +1,687 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Wiring of ``timeseries.db`` into the runtime: recording, flushing, nightly job, rebuild.""" + +from __future__ import annotations + +import asyncio +import contextlib +from collections.abc import AsyncIterator, Callable +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import pytest +import structlog.testing + +from suncore.app.control import ControlStack +from suncore.app.history import site_statistic_specs +from suncore.app.main import build_runtime +from suncore.app.recorder import MeasurementRecorder +from suncore.app.runtime import Runtime +from suncore.app.settings import Settings +from suncore.core.measurements import ( + SITE_HOUSE_W, + SITE_KEYS, + SITE_SOC_PCT, + device_power_key, +) +from suncore.core.series import TimeRange +from suncore.ha.connection import CommandError, HAError +from suncore.storage.timeseries import META_HISTORY_REBUILT, TS_FILENAME, TimeseriesStore +from tests.app.conftest import RecordingConnection, demo_settings + +T0 = datetime(2026, 6, 21, 0, 0, tzinfo=UTC) + + +def prod_settings(tmp_path: Path) -> Settings: + return Settings(demo_mode=False, data_dir=tmp_path / "data", static_dir=tmp_path / "no-dist") + + +class GatedSleep: + """Sleep that records the requested delays and returns only when released.""" + + def __init__(self) -> None: + self.requested: list[float] = [] + self._events: list[asyncio.Event] = [] + + async def __call__(self, delay: float) -> None: + self.requested.append(delay) + event = asyncio.Event() + self._events.append(event) + await event.wait() + + async def release(self, delay: float) -> None: + """Let the oldest pending sleep of ``delay`` return.""" + for event, requested in zip(self._events, self.requested, strict=False): + if requested == delay and not event.is_set(): + event.set() + break + for _ in range(10): + await asyncio.sleep(0) + + +async def until(predicate: Callable[[], bool], timeout_s: float = 5.0) -> None: + async with asyncio.timeout(timeout_s): + while not predicate(): # noqa: ASYNC110 - polling a plain predicate + await asyncio.sleep(0.01) + + +async def idle(runtime: Runtime) -> ControlStack: + """Stop the background tick of the current stack (after a possible rebuild settled).""" + await until(lambda: runtime.stack is not None and runtime._tz_learnt) + stack = runtime.stack + assert stack is not None + await stack._stop_tasks() + return stack + + +def store_of(runtime: Runtime) -> TimeseriesStore: + assert runtime.timeseries is not None + return runtime.timeseries + + +@pytest.fixture +async def gated(tmp_path: Path) -> AsyncIterator[tuple[Runtime, GatedSleep, RecordingConnection]]: + conn = RecordingConnection(speed=6000.0) + runtime = await build_runtime(demo_settings(tmp_path), env={}, connection=conn) + sleep = GatedSleep() + runtime.sleep = sleep + runtime.clock = lambda: T0 + timedelta(hours=10) + await runtime.start() + try: + yield runtime, sleep, conn + finally: + await runtime.stop() + + +# --- opening and closing -------------------------------------------------------------------- + + +async def test_production_path_creates_timeseries_next_to_suncore_db(tmp_path: Path) -> None: + settings = prod_settings(tmp_path) + runtime = await build_runtime(settings, env={}, connection=RecordingConnection(6000.0)) + try: + assert (settings.data_dir / "suncore.db").is_file() + assert (settings.data_dir / TS_FILENAME).is_file() + assert runtime.timeseries is not None + assert runtime.timeseries_db is not None + assert await runtime.timeseries_db.user_version() >= 1 + assert runtime.plans is not None + assert runtime.learned is not None + assert runtime.recorder.enabled + finally: + await runtime.stop() + assert not await runtime.timeseries_db.check() # closed with the runtime + assert not await runtime.db.check() + + +async def test_demo_keeps_timeseries_in_its_throw_away_dir(tmp_path: Path) -> None: + settings = demo_settings(tmp_path) + runtime = await build_runtime(settings, env={}, connection=RecordingConnection(6000.0)) + try: + assert runtime.timeseries_db is not None + assert not Path(runtime.timeseries_db.path).is_relative_to(settings.data_dir) + assert Path(runtime.timeseries_db.path).name == TS_FILENAME + finally: + await runtime.stop() + assert not settings.data_dir.exists() + + +async def test_too_new_timeseries_db_disables_recording_but_not_the_app(tmp_path: Path) -> None: + import sqlite3 # noqa: PLC0415 + + settings = prod_settings(tmp_path) + settings.data_dir.mkdir(parents=True) + con = sqlite3.connect(settings.data_dir / TS_FILENAME) + con.execute("PRAGMA user_version = 99") + con.close() + runtime = await build_runtime(settings, env={}, connection=RecordingConnection(6000.0)) + try: + assert runtime.timeseries is None + assert not runtime.recorder.enabled + await runtime.start() # control and API come up without it + stack = await idle(runtime) + assert await stack.tick() is not None + assert runtime.recorder.pending == 0 + assert await runtime.maintain_timeseries() is False + finally: + await runtime.stop() + con = sqlite3.connect(settings.data_dir / TS_FILENAME) + assert con.execute("PRAGMA user_version").fetchone() == (99,) + con.close() + + +# --- recording --------------------------------------------------------------------------------- + + +async def test_ticks_are_buffered_and_not_written_per_tick( + gated: tuple[Runtime, GatedSleep, RecordingConnection], monkeypatch: pytest.MonkeyPatch +) -> None: + runtime, _sleep, _conn = gated + store = store_of(runtime) + writes: list[int] = [] + original = store.append_measurements + + async def spy(rows: Any) -> int: + batch = list(rows) + writes.append(len(batch)) + return await original(batch) + + monkeypatch.setattr(store, "append_measurements", spy) + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + for _ in range(5): + await stack.tick() + assert writes == [] # nothing written per tick + pending = runtime.recorder.pending + assert pending >= 5 * len(SITE_KEYS) # five site values per tick (+ the devices) + assert await runtime.recorder.flush() == pending + assert writes == [pending] # one batch + assert runtime.recorder.pending == 0 + assert await runtime.recorder.flush() == 0 + assert writes == [pending] + + +async def test_recorded_keys_cover_site_and_devices( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, _sleep, _conn = gated + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + await stack.tick() + await runtime.recorder.flush() + rows = await store_of(runtime).database.fetchall("SELECT DISTINCT key FROM measurement") + keys = {str(r[0]) for r in rows} + assert set(SITE_KEYS) <= keys + assert {device_power_key(d) for d in stack.models} <= keys + soc = await store_of(runtime).database.fetchone( + "SELECT value FROM measurement WHERE key = ?", (SITE_SOC_PCT,) + ) + assert soc is not None + assert 0.0 <= float(str(soc[0])) <= 100.0 + + +async def test_flush_cadence_is_60_seconds_and_a_stop_flushes_the_rest( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, sleep, _conn = gated + assert runtime.flush_interval_s == 60.0 + await until(lambda: 60.0 in sleep.requested) # the flush loop is waiting + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + await stack.tick() + assert runtime.recorder.pending > 0 + await sleep.release(60.0) # one flush interval elapses + await until(lambda: runtime.recorder.pending == 0) + count = await store_of(runtime).database.fetchone("SELECT COUNT(*) FROM measurement") + assert count is not None + assert int(str(count[0])) > 0 + await stack.tick() + assert runtime.recorder.pending > 0 + pending = runtime.recorder.pending + written: list[int] = [] + store = store_of(runtime) + original = store.append_measurements + + async def spy(rows: Any) -> int: + batch = list(rows) + written.append(len(batch)) + return await original(batch) + + store.append_measurements = spy # type: ignore[method-assign] + await runtime.stop() # the shutdown writes what is left, before the database is closed + assert written == [pending] + assert runtime.recorder.pending == 0 + + +async def test_a_database_error_never_reaches_the_controller( + gated: tuple[Runtime, GatedSleep, RecordingConnection], monkeypatch: pytest.MonkeyPatch +) -> None: + runtime, _sleep, _conn = gated + store = store_of(runtime) + + async def broken(_rows: Any) -> int: + msg = "disk full" + raise OSError(msg) + + monkeypatch.setattr(store, "append_measurements", broken) + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + assert await stack.tick() is not None + pending = runtime.recorder.pending + with structlog.testing.capture_logs() as logs: + assert await runtime.recorder.flush() == 0 # logged, not raised + assert any(e["event"] == "timeseries.flush_failed" for e in logs) + assert runtime.recorder.pending == pending # kept for the next flush + assert await stack.tick() is not None # control is unaffected + monkeypatch.undo() + assert await runtime.recorder.flush() > 0 # the database is back: the kept rows are written + + +async def test_a_failing_sink_does_not_break_the_tick( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, _sleep, _conn = gated + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + + def boom(_ts: datetime, _values: Any) -> None: + msg = "sink exploded" + raise RuntimeError(msg) + + stack._on_measurements = boom + with structlog.testing.capture_logs() as logs: + assert await stack.tick() is not None + assert any(e["event"] == "control.measurement_sink_failed" for e in logs) + + +async def test_only_ok_quality_values_are_recorded( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, _sleep, conn = gated + stack = await idle(runtime) + seen: list[dict[str, float | None]] = [] + stack._on_measurements = lambda _ts, values: seen.append(dict(values)) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + await stack.tick() + assert seen[0][SITE_HOUSE_W] is not None + # a dead house sensor is a gap (None), never a zero + from suncore.ha.connection import EntityState, EntityUpdate # noqa: PLC0415 + + stack.cache.ingest( + EntityUpdate( + changed={ + "sensor.sim_house_power": EntityState( + "sensor.sim_house_power", + "unavailable", + {"unit_of_measurement": "W"}, + last_changed=1.0, + last_updated=1.0, + ) + } + ) + ) + await stack.tick() + assert seen[1][SITE_HOUSE_W] is None + assert conn is not None + + +# --- the recorder itself ----------------------------------------------------------------------- + + +async def test_recorder_skips_gaps_and_is_a_noop_without_store() -> None: + off = MeasurementRecorder(None) + off.record(T0, {"k": 1.0}) + assert off.pending == 0 + assert await off.flush() == 0 + + +async def test_recorder_buffer_is_bounded_and_drops_the_oldest(tmp_path: Path) -> None: + from suncore.storage import open_timeseries # noqa: PLC0415 + + async with await open_timeseries(tmp_path / "t.db") as db: + recorder = MeasurementRecorder(TimeseriesStore(db), max_buffered=5) + for i in range(8): + recorder.record(T0 + timedelta(seconds=i), {"k": float(i), "gap": None}) + assert recorder.pending == 5 + assert recorder.dropped == 3 + assert await recorder.flush() == 5 + store = TimeseriesStore(db) + rows = await store.read_raw("k", TimeRange(start=T0, end=T0 + timedelta(minutes=1))) + assert [v for _, v in rows] == [3.0, 4.0, 5.0, 6.0, 7.0] # the oldest three are gone + + +async def test_recorder_retries_a_failed_batch_within_the_bound(tmp_path: Path) -> None: + from suncore.storage import open_timeseries # noqa: PLC0415 + + async with await open_timeseries(tmp_path / "t.db") as db: + store = TimeseriesStore(db) + recorder = MeasurementRecorder(store, max_buffered=4) + recorder.record(T0, {"k": 1.0}) + recorder.record(T0 + timedelta(seconds=1), {"k": 2.0}) + original = store.append_measurements + fail = True + + async def flaky(rows: Any) -> int: + if fail: + msg = "locked" + raise OSError(msg) + return await original(rows) + + store.append_measurements = flaky # type: ignore[method-assign] + assert await recorder.flush() == 0 + recorder.record(T0 + timedelta(seconds=2), {"k": 3.0}) + recorder.record(T0 + timedelta(seconds=3), {"k": 4.0}) + recorder.record(T0 + timedelta(seconds=4), {"k": 5.0}) + assert recorder.pending == 4 + assert recorder.dropped == 1 + fail = False + assert await recorder.flush() == 4 + rows = await original.__self__.read_raw( # type: ignore[attr-defined] + "k", TimeRange(start=T0, end=T0 + timedelta(minutes=1)) + ) + assert [v for _, v in rows] == [2.0, 3.0, 4.0, 5.0] + + +# --- the nightly job --------------------------------------------------------------------------- + + +async def test_nightly_job_runs_at_start_then_wakes_at_most_hourly( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, sleep, _conn = gated + # the clock is 10:00 UTC = 12:00 CEST (Europe/Berlin, the demo region): 15.5 h to 03:30 + await until(lambda: 3600.0 in sleep.requested) + assert runtime.seconds_until_maintenance() == 15.5 * 3600 + + +async def test_nightly_job_runs_once_per_night_and_follows_the_time_zone( + tmp_path: Path, +) -> None: + conn = RecordingConnection(speed=6000.0) + runtime = await build_runtime(demo_settings(tmp_path), env={}, connection=conn) + sleep = GatedSleep() + runtime.sleep = sleep + now = [datetime(2026, 6, 21, 10, 0, tzinfo=UTC)] # 12:00 CEST + runtime.clock = lambda: now[0] + runs: list[datetime] = [] + + async def spy(at: datetime) -> None: + runs.append(at) + + store_of(runtime).maintain = spy # type: ignore[method-assign,assignment] + try: + await runtime.start() + await until(lambda: len(runs) == 1) # at start + await until(lambda: 3600.0 in sleep.requested) + # six hourly wakes later it is 18:00 CEST: no run + for hour in range(1, 7): + now[0] += timedelta(hours=1) + await sleep.release(3600.0) + await until(lambda n=hour: len(sleep.requested) > n) + assert len(runs) == 1 + # 03:30 CEST the next day: exactly one run, and waking again does not repeat it + now[0] = datetime(2026, 6, 22, 1, 31, tzinfo=UTC) + await sleep.release(3600.0) + await until(lambda: len(runs) == 2) + await sleep.release(3600.0) + await sleep.release(3600.0) + await asyncio.sleep(0.05) + assert len(runs) == 2 + # the zone changes (e.g. Home Assistant's zone becomes known): the target moves with it + from suncore.core.config import RegionConfig # noqa: PLC0415 + + before = runtime.next_maintenance_after(now[0]) + runtime.region = RegionConfig(region_id="de", currency="EUR", timezone="Asia/Tokyo") + after = runtime.next_maintenance_after(now[0]) + assert before != after + assert after == datetime(2026, 6, 22, 18, 30, tzinfo=UTC) # 03:30 JST + finally: + await runtime.stop() + + +@pytest.mark.parametrize( + ("now", "expected_h"), + [ + (datetime(2026, 6, 21, 1, 29, tzinfo=UTC), 0.0 + 1 / 60), # 03:29 CEST -> 1 min + (datetime(2026, 6, 21, 1, 30, tzinfo=UTC), 24.0), # exactly 03:30: tomorrow's + (datetime(2026, 3, 28, 12, 0, tzinfo=UTC), 13.5), # DST night: 23 h day (03:30 CEST) + (datetime(2026, 10, 24, 12, 0, tzinfo=UTC), 14.5), # DST night: 25 h day (03:30 CET) + ], +) +async def test_seconds_until_maintenance_is_dst_safe( + gated: tuple[Runtime, GatedSleep, RecordingConnection], now: datetime, expected_h: float +) -> None: + runtime, _sleep, _conn = gated + runtime.clock = lambda: now + assert runtime.seconds_until_maintenance() == pytest.approx(expected_h * 3600, abs=1) + + +async def test_maintenance_downsamples_and_survives_errors( + gated: tuple[Runtime, GatedSleep, RecordingConnection], monkeypatch: pytest.MonkeyPatch +) -> None: + runtime, _sleep, _conn = gated + store = store_of(runtime) + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + await stack.tick() + later = datetime.now(UTC) + timedelta(days=5) # the stack stamps with the real clock + runtime.clock = lambda: later + assert await runtime.maintain_timeseries() is True + assert await store.database.fetchone("SELECT COUNT(*) FROM measurement") == (0,) + assert await store.database.fetchone("SELECT COUNT(*) FROM measurement_1m") != (0,) + + async def broken(_now: datetime) -> Any: + msg = "boom" + raise OSError(msg) + + monkeypatch.setattr(store, "maintain", broken) + with structlog.testing.capture_logs() as logs: + assert await runtime.maintain_timeseries() is False + assert any(e["event"] == "timeseries.maintenance_failed" for e in logs) + + +async def test_plan_snapshots_are_pruned_hourly( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, _sleep, _conn = gated + assert runtime.plans is not None + await runtime.plans.save( + {"x": 1}, source="t", schema_version=1, created_at=runtime.clock() - timedelta(days=30) + ) + await runtime.prune_plans() # the hourly loop may have been first: either way it is gone + assert await runtime.plans.count() == 0 + + +# --- the statistics rebuild --------------------------------------------------------------------- + + +@pytest.fixture +async def rebuild_env(tmp_path: Path) -> AsyncIterator[tuple[Runtime, RecordingConnection]]: + conn = RecordingConnection(speed=6000.0) + runtime = await build_runtime(demo_settings(tmp_path), env={}, connection=conn) + # the fake house's recorder history ends at the scenario start + runtime.clock = lambda: conn._scenario.start + try: + yield runtime, conn + finally: + await runtime.stop() + + +async def test_rebuild_runs_once_after_connect_when_the_15_minute_table_is_empty( + rebuild_env: tuple[Runtime, RecordingConnection], +) -> None: + runtime, conn = rebuild_env + store = store_of(runtime) + assert not await store.has_quarter_history() + await runtime.start() + await until(lambda: runtime._history_done) + assert await store.has_quarter_history() + keys = { + str(r[0]) for r in await store.database.fetchall("SELECT DISTINCT key FROM measurement_15m") + } + assert keys == set(SITE_KEYS) + derived = await store.database.fetchone("SELECT MIN(derived) FROM measurement_15m") + assert derived == (1,) + # 72 h of simulated history, all hours present + count = await store.database.fetchone( + "SELECT COUNT(*) FROM measurement_15m WHERE key = ?", (SITE_HOUSE_W,) + ) + assert count == (72 * 4,) + assert all(c["period"] == "hour" for c in conn.recorder.calls) + asked = {i for c in conn.recorder.calls for i in c["ids"]} + assert asked == {s.statistic_id for s in site_statistic_specs(runtime.site)} # type: ignore[arg-type] + calls = len(conn.recorder.calls) + runtime._start_history_rebuild() # a reconnect or a site edit does not rebuild again + await asyncio.sleep(0.05) + assert len(conn.recorder.calls) == calls + + +async def test_rebuild_is_skipped_once_the_marker_is_set( + rebuild_env: tuple[Runtime, RecordingConnection], +) -> None: + runtime, conn = rebuild_env + await store_of(runtime).set_meta(META_HISTORY_REBUILT, "2026-06-01T00:00:00+00:00") + await runtime.start() + await until(lambda: runtime._history_done) + assert conn.recorder.calls == [] + + +async def test_existing_quarter_history_does_not_suppress_the_rebuild( + rebuild_env: tuple[Runtime, RecordingConnection], +) -> None: + """Only a successful rebuild sets the marker: data (or a maintenance run) is no substitute.""" + runtime, conn = rebuild_env + from suncore.core.series import SeriesUnit, TimeSeries # noqa: PLC0415 + + store = store_of(runtime) + await store.store_quarter_hours( + SITE_HOUSE_W, + TimeSeries(start=T0, step=timedelta(minutes=15), values=(1.0,), unit=SeriesUnit.W), + ) + assert await store.has_quarter_history() + await runtime.start() # the maintenance job runs at start, too + await until(lambda: runtime._history_done) + assert conn.recorder.calls + assert await store.get_meta(META_HISTORY_REBUILT) is not None + + +async def test_rebuild_retries_with_backoff_until_it_succeeded_once( + rebuild_env: tuple[Runtime, RecordingConnection], +) -> None: + runtime, conn = rebuild_env + sleep = GatedSleep() + runtime.sleep = sleep + runtime.flush_interval_s = 61.0 # keep the flush loop out of the 60 s back-off + conn.statistics_error = TimeoutError("slow") + store = store_of(runtime) + await runtime.start() + await until(lambda: 60.0 in sleep.requested) # first retry after a minute + assert not runtime._history_done + assert await store.get_meta(META_HISTORY_REBUILT) is None + conn.statistics_error = HAError("down") + await sleep.release(60.0) + await until(lambda: 300.0 in sleep.requested) + conn.statistics_error = None # Home Assistant is back + await sleep.release(300.0) + await until(lambda: runtime._history_done) + assert await store.get_meta(META_HISTORY_REBUILT) is not None + assert await store.has_quarter_history() + + +async def test_rebuild_failure_is_logged_and_does_not_stop_the_app( + rebuild_env: tuple[Runtime, RecordingConnection], +) -> None: + runtime, conn = rebuild_env + conn.statistics_error = CommandError("unknown_command", "Unknown command.") + with structlog.testing.capture_logs() as logs: + await runtime.start() + await until(lambda: runtime._history_done) + warning = next(e for e in logs if e["event"] == "history.rebuild_failed") + assert warning["reason"] == "unknown_command" + assert not await store_of(runtime).has_quarter_history() + assert await store_of(runtime).get_meta(META_HISTORY_REBUILT) is None # retried next start + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + assert await stack.tick() is not None # control is unaffected + + +async def test_rebuild_waits_for_the_site( + tmp_path: Path, +) -> None: + conn = RecordingConnection(speed=6000.0) + runtime = await build_runtime(demo_settings(tmp_path), env={}, connection=conn) + runtime.clock = lambda: conn._scenario.start + site = runtime.site + runtime.site = None + try: + await runtime.start() + await asyncio.sleep(0.1) + assert not runtime._history_done + assert conn.recorder.calls == [] + runtime.site = site + runtime._start_history_rebuild() + await until(lambda: runtime._history_done) + assert conn.recorder.calls + finally: + await runtime.stop() + + +async def test_a_cancelled_flush_puts_the_batch_back(tmp_path: Path) -> None: + from suncore.storage import open_timeseries # noqa: PLC0415 + + async with await open_timeseries(tmp_path / "t.db") as db: + store = TimeseriesStore(db) + recorder = MeasurementRecorder(store) + recorder.record(T0, {"a": 1.0, "b": 2.0}) + entered = asyncio.Event() + + async def hang(_rows: Any) -> int: + entered.set() + await asyncio.Event().wait() + return 0 + + store.append_measurements = hang # type: ignore[method-assign] + task = asyncio.create_task(recorder.flush()) + await entered.wait() + assert recorder.pending == 0 # in flight + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert recorder.pending == 2 # back in the buffer, to be written by the final flush + del store.append_measurements + assert await recorder.flush() == 2 + + +async def test_devices_are_released_before_storage_tasks_are_awaited( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + """A storage task that is slow to cancel (its rollback waits behind a long SQL statement) + must not delay the hardware release of the control stack.""" + runtime, _sleep, _conn = gated + stack = await idle(runtime) + hold = asyncio.Event() + release = asyncio.Event() + in_maintain = asyncio.Event() + shut_down = asyncio.Event() + + async def stubborn(_now: datetime) -> None: + in_maintain.set() + try: + await hold.wait() + except asyncio.CancelledError: + await release.wait() # the shielded rollback behind the running statement + raise + + store_of(runtime).maintain = stubborn # type: ignore[method-assign,assignment] + runtime._maintenance_task.cancel() # type: ignore[union-attr] + with contextlib.suppress(asyncio.CancelledError): + await runtime._maintenance_task # type: ignore[misc] + runtime._maintenance_task = asyncio.create_task(runtime._maintenance_loop()) + await in_maintain.wait() + original = stack.shutdown + + async def shutdown() -> None: + await original() + shut_down.set() + + stack.shutdown = shutdown # type: ignore[method-assign] + stopping = asyncio.create_task(runtime.stop()) + async with asyncio.timeout(5): + await shut_down.wait() # the release ran although the storage task is still stuck + assert not stopping.done() + release.set() + await stopping + + +async def test_a_failure_while_building_the_values_does_not_break_the_tick( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, _sleep, _conn = gated + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + stack._on_measurements = lambda _t, _v: None + stack.estimates = None # type: ignore[assignment] # not a mapping: .items() would raise + with structlog.testing.capture_logs() as logs: + stack._record_measurements(runtime.clock(), None, None) # type: ignore[arg-type] + assert any(e["event"] == "control.measurement_sink_failed" for e in logs) diff --git a/tests/ha/fakes.py b/tests/ha/fakes.py index d272b8e..114e758 100644 --- a/tests/ha/fakes.py +++ b/tests/ha/fakes.py @@ -112,6 +112,8 @@ def __init__(self, token: str = TOKEN) -> None: self.device_registry: list[dict[str, Any]] = [] self.area_registry: list[dict[str, Any]] = [] self.energy_prefs: dict[str, Any] | None = None # None = "No prefs" (not_found) + # recorder/statistics_during_period: statistic_id -> rows (ms epoch, as HA answers). + self.statistics: dict[str, list[dict[str, Any]]] = {} self.service_response: Callable[[dict[str, Any]], Any] = lambda msg: {"echo": msg} self.event_subscriptions: dict[web.WebSocketResponse, dict[int, str]] = {} self._server: TestServer | None = None @@ -228,7 +230,7 @@ async def _reply(self, ws: web.WebSocketResponse, *messages: dict[str, Any]) -> for message in messages: await ws.send_str(json.dumps(message)) - async def _on_message( # noqa: PLR0912 + async def _on_message( # noqa: PLR0912, PLR0915 self, ws: web.WebSocketResponse, msg: dict[str, Any] ) -> None: self.received.append(msg) @@ -273,6 +275,10 @@ def ok(result: Any = None) -> dict[str, Any]: ) else: await self._reply(ws, ok(self.energy_prefs)) + elif kind == "recorder/statistics_during_period": + wanted_ids = msg["statistic_ids"] + result = {i: self.statistics[i] for i in wanted_ids if self.statistics.get(i)} + await self._reply(ws, ok(result)) elif kind == "subscribe_events": self.event_subscriptions.setdefault(ws, {})[mid] = msg["event_type"] await self._reply(ws, ok()) diff --git a/tests/ha/test_action_rate_limit.py b/tests/ha/test_action_rate_limit.py new file mode 100644 index 0000000..2bdf5a3 --- /dev/null +++ b/tests/ha/test_action_rate_limit.py @@ -0,0 +1,296 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Rate limit of ``call_action_response`` per (domain, service) (Phase 4 follow-up A3).""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator + +import pytest + +from suncore.ha.connection import ( + DEFAULT_ACTION_INTERVAL_S, + ERR_ACTION_RATE_LIMITED, + ActionNotAllowedError, + ActionRateLimitedError, + ActionRateLimiter, +) +from suncore.ha.transport import DirectTransport +from suncore.ha.websocket import WebSocketConnection +from tests.ha.fakes import TOKEN, FakeClock, FakeHA + + +def _limiter(clock: FakeClock, interval: float = 30.0) -> ActionRateLimiter: + return ActionRateLimiter(interval, clock=clock.monotonic, sleep=clock.sleep) + + +# --- the limiter --------------------------------------------------------------------- + + +def test_default_interval_is_30_seconds() -> None: + assert DEFAULT_ACTION_INTERVAL_S == 30.0 + + +def test_try_acquire_admits_once_per_interval() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert limiter.try_acquire("tibber.get_prices") + assert not limiter.try_acquire("tibber.get_prices") + assert limiter.retry_after("tibber.get_prices") == 30.0 + clock.now = 29.9 + assert not limiter.try_acquire("tibber.get_prices") + clock.now = 30.0 + assert limiter.try_acquire("tibber.get_prices") + assert limiter.retry_after("tibber.get_prices") == 30.0 + + +def test_a_refused_try_does_not_move_the_slot() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert limiter.try_acquire("a.b") + clock.now = 20.0 + assert not limiter.try_acquire("a.b") + assert limiter.retry_after("a.b") == 10.0 + + +def test_keys_are_independent() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert limiter.try_acquire("tibber.get_prices") + assert limiter.try_acquire("weather.get_forecasts") + assert not limiter.try_acquire("tibber.get_prices") + assert limiter.retry_after("never.used") == 0.0 + + +def test_a_caller_supplied_interval_applies_to_the_next_slot() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert limiter.try_acquire("a.b", 5.0) + assert limiter.retry_after("a.b") == 5.0 + clock.now = 5.0 + assert limiter.try_acquire("a.b", 0.0) + assert limiter.try_acquire("a.b", 0.0) # interval 0: never limited + + +def test_negative_intervals_are_rejected() -> None: + clock = FakeClock() + with pytest.raises(ValueError, match="default_interval_s"): + ActionRateLimiter(-1.0, clock=clock.monotonic, sleep=clock.sleep) + with pytest.raises(ValueError, match="interval_s"): + _limiter(clock).try_acquire("a.b", -1.0) + + +async def test_acquire_without_deadline_is_a_try() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert await limiter.acquire("a.b") + assert not await limiter.acquire("a.b") + assert clock.requested == [] # never slept + + +async def test_acquire_waits_for_a_slot_within_the_deadline() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert await limiter.acquire("a.b") + clock.now = 10.0 + task = asyncio.create_task(limiter.acquire("a.b", max_wait_s=25.0)) + await clock.settle() + assert clock.requested == [20.0] + assert not task.done() + await clock.advance(20.0) + assert await task is True + assert limiter.retry_after("a.b") == 30.0 + + +async def test_acquire_refuses_when_the_slot_is_beyond_the_deadline() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert await limiter.acquire("a.b") + assert not await limiter.acquire("a.b", max_wait_s=10.0) + assert clock.requested == [] + assert limiter.retry_after("a.b") == 30.0 # nothing was reserved by the refusal + + +async def test_concurrent_waiters_get_consecutive_slots() -> None: + clock = FakeClock() + limiter = _limiter(clock, 10.0) + assert await limiter.acquire("a.b") + first = asyncio.create_task(limiter.acquire("a.b", max_wait_s=100.0)) + second = asyncio.create_task(limiter.acquire("a.b", max_wait_s=100.0)) + third = asyncio.create_task(limiter.acquire("a.b", max_wait_s=15.0)) + await clock.settle() + assert sorted(clock.requested) == [10.0, 20.0] # two slots reserved in order + assert await third is False # its slot (30 s) is beyond its 15 s deadline + await clock.advance(20.0) + assert await first is True + assert await second is True + + +# --- on the connection --------------------------------------------------------------- + + +@pytest.fixture +async def ha() -> AsyncIterator[FakeHA]: + server = FakeHA() + await server.start() + yield server + await server.stop() + + +@pytest.fixture +def clock() -> FakeClock: + return FakeClock() + + +@pytest.fixture +async def conn(ha: FakeHA, clock: FakeClock) -> AsyncIterator[WebSocketConnection]: + connection = WebSocketConnection( + DirectTransport.from_base_url(ha.base_url, TOKEN), + clock=clock.monotonic, + sleep=clock.sleep, + rng=lambda: 0.5, + heartbeat_interval=777.0, + heartbeat_timeout=666.0, + resync_interval=555.0, + command_timeout=888.0, + connect_timeout=999.0, + ) + await asyncio.wait_for(connection.connect(), 5) + yield connection + await connection.close() + + +def _calls(ha: FakeHA) -> int: + return len([m for m in ha.received if m["type"] == "call_service"]) + + +async def test_second_call_within_the_interval_is_refused_not_sent( + conn: WebSocketConnection, ha: FakeHA, clock: FakeClock +) -> None: + ha.service_response = lambda _m: {"ok": True} + assert await conn.call_action_response("tibber", "get_prices") == {"ok": True} + with pytest.raises(ActionRateLimitedError) as info: + await conn.call_action_response("tibber", "get_prices") + assert info.value.code == "rate_limited" + assert info.value.action == "tibber.get_prices" + assert info.value.retry_after_s == pytest.approx(30.0) + assert _calls(ha) == 1 + clock.now += 30.0 + assert await conn.call_action_response("tibber", "get_prices") == {"ok": True} + assert _calls(ha) == 2 + + +async def test_other_actions_are_not_affected(conn: WebSocketConnection, ha: FakeHA) -> None: + ha.service_response = lambda _m: {} + await conn.call_action_response("tibber", "get_prices") + await conn.call_action_response("weather", "get_forecasts") + assert _calls(ha) == 2 + + +async def test_configured_interval_and_deadline( + conn: WebSocketConnection, ha: FakeHA, clock: FakeClock +) -> None: + ha.service_response = lambda _m: {} + await conn.call_action_response("tibber", "get_prices", min_interval_s=5.0) + with pytest.raises(ActionRateLimitedError) as info: + await conn.call_action_response("tibber", "get_prices", min_interval_s=5.0) + assert info.value.retry_after_s == pytest.approx(5.0) + waiter = asyncio.create_task( + conn.call_action_response("tibber", "get_prices", min_interval_s=5.0, max_wait_s=6.0) + ) + await clock.settle() + await clock.advance(5.0) + assert await waiter == {} + assert _calls(ha) == 2 + + +async def test_not_allowed_is_checked_before_the_limiter( + conn: WebSocketConnection, ha: FakeHA +) -> None: + with pytest.raises(ActionNotAllowedError): + await conn.call_action_response("switch", "turn_on") + with pytest.raises(ActionNotAllowedError): + await conn.call_action_response("switch", "turn_on") # not "rate limited" either + assert _calls(ha) == 0 + + +def test_the_refusal_has_a_catalogue_key() -> None: + import json # noqa: PLC0415 + from pathlib import Path # noqa: PLC0415 + + locales = Path(__file__).resolve().parents[2] / "frontend/src/lib/i18n/locales" + for name in ("en", "de"): + catalogue = json.loads((locales / f"{name}.json").read_text(encoding="utf-8")) + assert catalogue[ERR_ACTION_RATE_LIMITED] + + +def test_action_rate_key_default_and_sub_key() -> None: + from suncore.ha.connection import action_rate_key # noqa: PLC0415 + + assert action_rate_key("tibber", "get_prices") == "tibber.get_prices" + assert action_rate_key("tibber", "get_prices", None) == "tibber.get_prices" + assert action_rate_key("tibber", "get_prices", "") == "tibber.get_prices" + assert action_rate_key("weather", "get_forecasts", "weather.home") == ( + "weather.get_forecasts#weather.home" + ) + + +async def test_sub_keys_do_not_block_each_other(conn: WebSocketConnection, ha: FakeHA) -> None: + ha.service_response = lambda _m: {} + await conn.call_action_response("weather", "get_forecasts", rate_key="weather.home") + await conn.call_action_response("weather", "get_forecasts", rate_key="weather.cabin") + with pytest.raises(ActionRateLimitedError) as info: + await conn.call_action_response("weather", "get_forecasts", rate_key="weather.home") + assert info.value.action == "weather.get_forecasts#weather.home" + await conn.call_action_response("weather", "get_forecasts") # the plain key: its own slot + with pytest.raises(ActionRateLimitedError): + await conn.call_action_response("weather", "get_forecasts") + assert _calls(ha) == 3 + + +async def test_a_cancelled_waiter_keeps_its_reservation() -> None: + clock = FakeClock() + limiter = _limiter(clock, 10.0) + assert await limiter.acquire("a.b") + waiter = asyncio.create_task(limiter.acquire("a.b", max_wait_s=100.0)) + await clock.settle() + waiter.cancel() + await clock.settle() + assert limiter.retry_after("a.b") == 20.0 # its slot (10 s) + the interval stays taken + + +async def test_the_fake_connection_is_limited_like_the_real_one() -> None: + from datetime import UTC, datetime # noqa: PLC0415 + + from suncore.sim.fake_ha import FakeHAConnection # noqa: PLC0415 + from suncore.sim.scenario import load_scenario # noqa: PLC0415 + + now = [datetime(2026, 6, 21, tzinfo=UTC)] + fake = FakeHAConnection( + load_scenario("summer_day"), clock=lambda: now[0], autorun=False, speed=1.0 + ) + await fake.connect() + try: + await fake.call_action_response("tibber", "get_prices") + with pytest.raises(ActionRateLimitedError): + await fake.call_action_response("tibber", "get_prices") + await fake.call_action_response("tibber", "get_prices", rate_key="other") + now[0] = datetime(2026, 6, 21, 0, 0, 31, tzinfo=UTC) + await fake.call_action_response("tibber", "get_prices") + finally: + await fake.close() + + +async def test_not_connected_does_not_consume_a_limiter_slot(ha: FakeHA, clock: FakeClock) -> None: + from suncore.ha.connection import NotConnectedError # noqa: PLC0415 + + connection = WebSocketConnection( + DirectTransport.from_base_url(ha.base_url, TOKEN), + clock=clock.monotonic, + sleep=clock.sleep, + ) + with pytest.raises(NotConnectedError): + await connection.call_action_response("tibber", "get_prices") + assert connection._action_limiter.retry_after("tibber.get_prices") == 0.0 + with pytest.raises(NotConnectedError): # and not "rate limited" the second time + await connection.call_action_response("tibber", "get_prices") diff --git a/tests/ha/test_statistics.py b/tests/ha/test_statistics.py new file mode 100644 index 0000000..9ddff9a --- /dev/null +++ b/tests/ha/test_statistics.py @@ -0,0 +1,282 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Quarter-hour history from ``recorder/statistics_during_period`` (Phase 4 WP3).""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta, timezone +from typing import Any + +import pytest + +from suncore.core.series import SeriesUnit +from suncore.ha.connection import CommandError, NotConnectedError +from suncore.ha.statistics import ( + HOUR_MS, + STATISTICS_UNITS, + StatisticSpec, + fetch_quarter_history, + hourly_values, + quarter_hours, +) +from suncore.ha.transport import DirectTransport +from suncore.ha.websocket import WebSocketConnection +from tests.ha.fakes import TOKEN, FakeClock, FakeHA + +H0 = datetime(2026, 10, 10, 0, 0, tzinfo=UTC) +H0_MS = int(H0.timestamp() * 1000) + + +def _row(hour: int, **values: Any) -> dict[str, Any]: + start = H0_MS + hour * HOUR_MS + return {"start": start, "end": start + HOUR_MS, **values} + + +# --- parsing --------------------------------------------------------------------------- + + +def test_mean_rows_become_hourly_values_and_skip_null() -> None: + rows = [_row(0, mean=100.0), _row(1, mean=None), _row(2, mean=300.0)] + assert hourly_values(rows, "mean") == {H0_MS: 100.0, H0_MS + 2 * HOUR_MS: 300.0} + + +def test_rows_off_the_grid_or_not_hourly_are_skipped() -> None: + odd_start = {"start": H0_MS + 1000, "end": H0_MS + 1000 + HOUR_MS, "mean": 1.0} + five_minutes = {"start": H0_MS, "end": H0_MS + 300_000, "mean": 2.0} + no_start = {"mean": 3.0} + bad_type = {"start": "x", "end": 1, "mean": 4.0} + assert hourly_values([odd_start, five_minutes, no_start, bad_type], "mean") == {} + + +def test_energy_uses_change_else_the_difference_of_sum() -> None: + rows = [ + _row(0, change=500.0, sum=1000.0), + _row(1, change=None, sum=1800.0), # 800 from the sums of consecutive hours + _row(3, change=None, sum=3000.0), # previous row is not the previous hour: a gap + _row(4, change=-5.0, sum=3100.0), # negative: skipped + _row(5, change=0.0, sum=3100.0), + ] + assert hourly_values(rows, "energy") == { + H0_MS: 500.0, + H0_MS + HOUR_MS: 800.0, + H0_MS + 5 * HOUR_MS: 0.0, + } + + +def test_booleans_and_non_finite_numbers_are_not_values() -> None: + rows = [_row(0, mean=True), _row(1, mean=float("nan")), _row(2, mean=float("inf"))] + assert hourly_values(rows, "mean") == {} + + +# --- hourly -> quarter hours --------------------------------------------------------------- + + +def test_step_hold_preserves_energy_and_marks_missing_hours() -> None: + series = quarter_hours({H0_MS: 1000.0, H0_MS + 2 * HOUR_MS: 400.0}, unit=SeriesUnit.W) + assert series is not None + assert series.start == H0 + assert series.step == timedelta(minutes=15) + assert series.values == (1000.0,) * 4 + (None,) * 4 + (400.0,) * 4 + # energy of the covered hours: 1000 Wh + 400 Wh + assert sum(v * 0.25 for v in series.values if v is not None) == pytest.approx(1400.0) + + +def test_inversion_flips_the_sign_without_negative_zero() -> None: + series = quarter_hours({H0_MS: 250.0, H0_MS + HOUR_MS: 0.0}, unit=SeriesUnit.W, invert=True) + assert series is not None + assert series.values == (-250.0,) * 4 + (0.0,) * 4 + assert str(series.values[-1]) == "0.0" + + +def test_no_values_no_series_and_a_long_range_is_refused() -> None: + assert quarter_hours({}, unit=SeriesUnit.W) is None + with pytest.raises(ValueError, match="too long"): + quarter_hours({H0_MS: 1.0, H0_MS + 400 * 24 * HOUR_MS: 1.0}, unit=SeriesUnit.W) + + +# --- fetching through the fake connection (sim) ------------------------------------------------ + + +@pytest.fixture +async def sim() -> AsyncIterator[Any]: + from suncore.sim.fake_ha import FakeHAConnection # noqa: PLC0415 + from suncore.sim.scenario import load_scenario # noqa: PLC0415 + + scenario = load_scenario("sensor_dropout") + clock_time = scenario.start + + async def sleep(_delay: float) -> None: + await asyncio.Event().wait() # the feed never advances + + conn = FakeHAConnection(scenario, clock=lambda: clock_time, sleep=sleep, autorun=False) + await conn.connect() + yield conn + await conn.close() + + +async def test_fetch_from_the_sim_recorder_chunks_and_batches(sim: Any) -> None: + from suncore.sim.fake_ha import GRID_ID, HOUSE_ID, PV_ID # noqa: PLC0415 + + end = sim._scenario.start.replace(minute=0, second=0, microsecond=0) + specs = [ + StatisticSpec("house", HOUSE_ID), + StatisticSpec("pv", PV_ID), + StatisticSpec("grid", GRID_ID, invert=True), + StatisticSpec("ghost", "sensor.does_not_exist"), + ] + result = await fetch_quarter_history( + sim, specs, end - timedelta(hours=48), end, chunk_days=1, batch_size=2 + ) + assert set(result.series) == {"house", "pv", "grid"} + assert result.missing == ("sensor.does_not_exist",) + house = result.series["house"] + assert house.start == end - timedelta(hours=48) + assert len(house) == 48 * 4 + assert None not in house.values + # the sign of `grid` is inverted + plain = ( + await fetch_quarter_history( + sim, [StatisticSpec("g", GRID_ID)], end - timedelta(hours=48), end + ) + ).series["g"] + assert result.series["grid"].values == tuple(-v for v in plain.values if v is not None) + # 2 chunks (1 day each) x 2 batches (ids 2 + 2) + calls = sim.recorder.calls + assert len({c["start"] for c in calls}) == 2 + assert all(len(c["ids"]) <= 2 for c in calls) + assert all(c["period"] == "hour" for c in calls) + + +async def test_fetch_aligns_a_ragged_range_to_whole_hours(sim: Any) -> None: + from suncore.sim.fake_ha import HOUSE_ID # noqa: PLC0415 + + end = sim._scenario.start.replace(minute=0, second=0, microsecond=0) + await fetch_quarter_history( + sim, + [StatisticSpec("h", HOUSE_ID)], + end - timedelta(hours=5, minutes=20), + end + timedelta(minutes=10), + ) + call = sim.recorder.calls[0] + assert call["start"] == end - timedelta(hours=6) + assert call["end"] == end + timedelta(hours=1) + + +async def test_fetch_validates_its_arguments(sim: Any) -> None: + with pytest.raises(ValueError, match="before end"): + await fetch_quarter_history(sim, [], H0, H0) + with pytest.raises(ValueError, match="positive"): + await fetch_quarter_history(sim, [], H0, H0 + timedelta(hours=1), chunk_days=0) + + +async def test_sim_recorder_overrides_errors_and_connection_state(sim: Any) -> None: + sim.recorder.set_rows("sensor.custom", [_row(0, mean=5.0), _row(1, mean=6.0)]) + got = await sim.statistics_during_period(["sensor.custom"], H0, H0 + timedelta(hours=1)) + assert got == {"sensor.custom": [_row(0, mean=5.0)]} + sim.recorder.set_rows("sensor.custom", []) + assert await sim.statistics_during_period(["sensor.custom"], H0) == {} + assert await sim.statistics_during_period([], H0) == {} + with pytest.raises(CommandError): + await sim.statistics_during_period(["sensor.custom"], H0, period="5minute") + sim.statistics_error = CommandError("unauthorized", "no") + with pytest.raises(CommandError): + await sim.statistics_during_period(["sensor.custom"], H0) + await sim.close() + sim.statistics_error = None + with pytest.raises(NotConnectedError): + await sim.statistics_during_period(["sensor.custom"], H0) + + +# --- the WebSocket command --------------------------------------------------------------------- + + +@pytest.fixture +async def ha() -> AsyncIterator[FakeHA]: + server = FakeHA() + await server.start() + yield server + await server.stop() + + +@pytest.fixture +async def conn(ha: FakeHA) -> AsyncIterator[WebSocketConnection]: + clock = FakeClock() + connection = WebSocketConnection( + DirectTransport.from_base_url(ha.base_url, TOKEN), + clock=clock.monotonic, + sleep=clock.sleep, + rng=lambda: 0.5, + heartbeat_interval=777.0, + heartbeat_timeout=666.0, + resync_interval=555.0, + command_timeout=888.0, + connect_timeout=999.0, + ) + await asyncio.wait_for(connection.connect(), 5) + yield connection + await connection.close() + + +async def test_command_shape_offsets_units_and_absent_ids( + conn: WebSocketConnection, ha: FakeHA +) -> None: + ha.statistics["sensor.house"] = [_row(0, mean=100.0), _row(1, mean=200.0)] + berlin = timezone(timedelta(hours=2)) + got = await conn.statistics_during_period( + ["sensor.house", "sensor.missing"], + datetime(2026, 10, 10, 2, 0, tzinfo=berlin), + datetime(2026, 10, 10, 4, 0, tzinfo=berlin), + period="hour", + types=["mean"], + units=STATISTICS_UNITS, + ) + assert got == {"sensor.house": [_row(0, mean=100.0), _row(1, mean=200.0)]} # missing id absent + sent = next(m for m in ha.received if m["type"] == "recorder/statistics_during_period") + assert sent["start_time"] == "2026-10-10T02:00:00+02:00" # offset kept: never HA-local naive + assert sent["end_time"] == "2026-10-10T04:00:00+02:00" + assert sent["period"] == "hour" + assert sent["types"] == ["mean"] + assert sent["units"] == {"power": "W", "energy": "Wh"} + assert sent["statistic_ids"] == ["sensor.house", "sensor.missing"] + + +async def test_command_optional_fields_validation_and_errors( + conn: WebSocketConnection, ha: FakeHA +) -> None: + await conn.statistics_during_period(["sensor.a"], H0) + sent = next(m for m in ha.received if m["type"] == "recorder/statistics_during_period") + assert "end_time" not in sent + assert "types" not in sent + assert "units" not in sent + assert await conn.statistics_during_period([], H0) == {} + with pytest.raises(ValueError, match="period"): + await conn.statistics_during_period(["sensor.a"], H0, period="minute") + with pytest.raises(ValueError, match="timezone-aware"): + await conn.statistics_during_period(["sensor.a"], datetime(2026, 1, 1)) # noqa: DTZ001 + ha.fail_commands["recorder/statistics_during_period"] = ("unknown_command", "Unknown command.") + with pytest.raises(CommandError): + await conn.statistics_during_period(["sensor.a"], H0) + + +async def test_fetch_through_the_websocket_with_energy_and_missing_ids( + conn: WebSocketConnection, ha: FakeHA +) -> None: + ha.statistics["sensor.house_power"] = [_row(h, mean=100.0 * (h + 1)) for h in range(3)] + ha.statistics["sensor.pv_energy"] = [ + _row(0, change=400.0, sum=10_000.0), + _row(1, change=None, sum=10_900.0), + ] + specs = [ + StatisticSpec("house", "sensor.house_power"), + StatisticSpec("pv", "sensor.pv_energy", kind="energy"), + StatisticSpec("gone", "sensor.gone"), + ] + result = await fetch_quarter_history(conn, specs, H0, H0 + timedelta(hours=3)) + assert result.series["house"].values == (100.0,) * 4 + (200.0,) * 4 + (300.0,) * 4 + assert result.series["pv"].values == (400.0,) * 4 + (900.0,) * 4 + assert result.missing == ("sensor.gone",) + types = { + tuple(m["types"]) for m in ha.received if m["type"] == "recorder/statistics_during_period" + } + assert types == {("mean",), ("change", "sum")} diff --git a/tests/safety/test_boundary.py b/tests/safety/test_boundary.py index 90b0947..28daa62 100644 --- a/tests/safety/test_boundary.py +++ b/tests/safety/test_boundary.py @@ -30,8 +30,12 @@ operating modes only through the injected ``ModeProvider`` (Phase 3 WP3) and the runtime only through the injected ``RuntimeStore``. 7. ``call_action_response`` (read-only action with ``return_response``) is defined and used - only in ``suncore.ha`` (implementation), ``suncore.sim`` (fake) and ``suncore.app`` - (wiring of the tariff/forecast adapters later); ``READ_ONLY_ACTIONS`` is the fixed allowlist. + only in ``suncore.ha`` (implementation, rate limited per action), ``suncore.sim`` (fake) and + the one adapter wiring module ``suncore.app.adapters`` (Phase 4); no other part of + ``suncore.app`` may use it. ``READ_ONLY_ACTIONS`` is the fixed allowlist. + +8. ``statistics_during_period`` (read-only recorder statistics) is used only in ``suncore.ha``, + and ``suncore.sim`` (``suncore.app.history`` goes through ``suncore.ha.statistics``). The scan is robust to packages that are still empty (``suncore/ha`` until WP3) because it only looks at the files that exist. @@ -63,7 +67,8 @@ API_FORBIDDEN = (SAFETY, HA, "suncore.devices") WRITE_CAPABILITY_NAMES = frozenset({"ServiceCaller", "ServiceCallerProvider"}) SAFETY_FORBIDDEN_IMPORTS = ("suncore.core.config", "suncore.storage") -ACTION_RESPONSE_OWNERS = ("suncore.ha", "suncore.sim") +STATISTICS_OWNERS = ("suncore.ha", "suncore.sim") +ACTION_RESPONSE_OWNERS = ("suncore.ha", "suncore.sim", "suncore.app.adapters") def module_name(path: Path) -> str: @@ -325,12 +330,35 @@ def test_safety_does_not_read_config_or_storage(path: Path) -> None: @pytest.mark.parametrize("path", SOURCES, ids=lambda p: str(p.relative_to(ROOT))) def test_call_action_response_is_confined(path: Path) -> None: - """Rule 7: the response-returning call lives in ha (+ the fake); app joins later.""" + """Rule 7: the response-returning call lives in ha (+ the fake) and app.adapters only.""" module = module_name(path) if any(in_package(module, owner) for owner in ACTION_RESPONSE_OWNERS): return offenders = fmt(path, attribute_refs(parse(path), "call_action_response")) - assert not offenders, f"call_action_response referenced outside ha/sim: {offenders}" + assert not offenders, ( + f"call_action_response referenced outside ha/sim/app.adapters: {offenders}" + ) + + +@pytest.mark.parametrize("path", SOURCES, ids=lambda p: str(p.relative_to(ROOT))) +def test_statistics_during_period_is_confined(path: Path) -> None: + """Rule 8: the recorder statistics read lives in ha (+ the fake) only.""" + module = module_name(path) + if any(in_package(module, owner) for owner in STATISTICS_OWNERS): + return + offenders = fmt(path, attribute_refs(parse(path), "statistics_during_period")) + assert not offenders, f"statistics_during_period referenced outside ha/sim: {offenders}" + + +def test_action_response_owner_is_the_adapter_module_not_the_whole_app() -> None: + """Rule 7 admits ``suncore.app.adapters`` only: its siblings in ``suncore.app`` stay out.""" + assert "suncore.app" not in ACTION_RESPONSE_OWNERS + assert (PACKAGE / "app" / "adapters.py").is_file() + for owner in ACTION_RESPONSE_OWNERS: + assert not in_package("suncore.app.control", owner) + assert not in_package("suncore.app.runtime", owner) + assert not in_package("suncore.app.adapters_extra", owner) + assert in_package("suncore.app.adapters", "suncore.app.adapters") def test_read_only_action_allowlist_is_fixed() -> None: diff --git a/tests/storage/test_config_sections.py b/tests/storage/test_config_sections.py index 132c239..bd63283 100644 --- a/tests/storage/test_config_sections.py +++ b/tests/storage/test_config_sections.py @@ -108,7 +108,7 @@ async def test_v1_export_file_imports(db: Database) -> None: assert site.grid.is_pair assert [a.power for a in site.pv] == [EntityRef(entity_id="sensor.pv_total")] exported = await store.export_json() - assert exported["db_schema_version"] == latest_version() == 3 + assert exported["db_schema_version"] == latest_version() == 4 assert exported["sections"]["site"]["schema_version"] == 2 diff --git a/tests/storage/test_db_cancellation.py b/tests/storage/test_db_cancellation.py new file mode 100644 index 0000000..c3adf25 --- /dev/null +++ b/tests/storage/test_db_cancellation.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A cancelled writer must never leave a transaction open on the shared connection.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from suncore.storage import Database + + +@pytest.mark.parametrize("turns", range(8)) +async def test_cancelling_a_transaction_leaves_the_database_usable( + db: Database, turns: int +) -> None: + await db.execute("CREATE TABLE t (x INTEGER)") + rows = [(i,) for i in range(5000)] + task = asyncio.create_task(db.executemany("INSERT INTO t VALUES (?)", rows)) + for _ in range(turns): + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + # the next writer starts a transaction without "within a transaction" and sees either + # none or all of the cancelled batch (it was rolled back or committed as a whole) + await db.executemany("INSERT INTO t VALUES (?)", [(-1,)]) + row = await db.fetchone("SELECT COUNT(*) FROM t") + assert row in {(1,), (5001,)} + + +async def test_cancelled_while_waiting_for_the_lock_is_harmless(db: Database) -> None: + await db.execute("CREATE TABLE t (x INTEGER)") + async with db.transaction() as tx: + waiter = asyncio.create_task(db.executemany("INSERT INTO t VALUES (?)", [(1,)])) + await asyncio.sleep(0) + waiter.cancel() + await tx.execute("INSERT INTO t VALUES (7)") + with pytest.raises(asyncio.CancelledError): + await waiter + assert await db.fetchone("SELECT COUNT(*) FROM t") == (1,) diff --git a/tests/storage/test_plans_learned.py b/tests/storage/test_plans_learned.py new file mode 100644 index 0000000..722d41b --- /dev/null +++ b/tests/storage/test_plans_learned.py @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Schema 4 of ``suncore.db``: plan snapshots and learned parameters.""" + +from __future__ import annotations + +import sqlite3 +import zlib +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import aiosqlite +import pytest + +from suncore.storage import ( + Database, + LearnedParamStore, + PlanSnapshotStore, + SchemaTooNewError, +) +from suncore.storage.learned import NS_BLEND_WEIGHTS, NS_FORECAST_BIAS, NS_ML_STATE +from suncore.storage.migrations import latest_version, load_migrations, run_migrations +from suncore.storage.plans import decode_payload, encode_payload + +NOW = datetime(2026, 10, 17, 12, 0, tzinfo=UTC) + + +# --- migration 0004 --------------------------------------------------------------------- + + +async def test_v3_database_upgrades_to_v4_and_keeps_data(tmp_path: Path) -> None: + path = tmp_path / "v3.db" + async with aiosqlite.connect(path, isolation_level=None) as conn: + assert await run_migrations(conn, load_migrations()[:3]) == 3 + await conn.execute("INSERT INTO learned VALUES ('k', '1', 't')") + await conn.execute("INSERT INTO safety_state VALUES ('emergency_stop', '0', 't')") + async with await Database.open(path) as db: + assert await db.user_version() == latest_version() == 4 + assert await db.fetchone("SELECT value FROM learned") == ("1",) + assert await db.fetchone("SELECT value FROM safety_state") == ("0",) + tables = {r[0] for r in await db.fetchall("SELECT name FROM sqlite_master")} + assert {"plan_snapshot", "learned_param"} <= tables + + +async def test_fresh_database_has_schema_4(tmp_path: Path) -> None: + async with await Database.open(tmp_path / "s.db") as db: + assert await db.user_version() == 4 + + +async def test_reader_of_schema_3_refuses_schema_4(tmp_path: Path) -> None: + path = tmp_path / "v4.db" + async with await Database.open(path): + pass + async with aiosqlite.connect(path, isolation_level=None) as conn: + with pytest.raises(SchemaTooNewError) as info: + await run_migrations(conn, load_migrations()[:3]) + assert (info.value.db_version, info.value.code_version) == (4, 3) + con = sqlite3.connect(path) + assert con.execute("PRAGMA user_version").fetchone() == (4,) + con.close() + + +# --- plan snapshots ----------------------------------------------------------------------- + + +@pytest.fixture +def plans(db: Database) -> PlanSnapshotStore: + return PlanSnapshotStore(db, clock=lambda: NOW) + + +async def test_snapshot_round_trip_and_compression(db: Database, plans: PlanSnapshotStore) -> None: + payload = {"intervals": [{"t": i, "w": i * 1.5} for i in range(500)], "name": "plän"} + snapshot_id = await plans.save(payload, source="planner", schema_version=2) + got = await plans.get(snapshot_id) + assert got is not None + assert got.payload == payload + assert (got.source, got.schema_version, got.created_at) == ("planner", 2, NOW) + row = await db.fetchone("SELECT length(payload) FROM plan_snapshot") + assert row is not None + assert int(str(row[0])) < len(str(payload)) / 3 # compressed + assert await plans.get(snapshot_id + 1) is None + + +async def test_payload_codec_is_strict() -> None: + assert decode_payload(encode_payload({"a": 1})) == {"a": 1} + with pytest.raises(ValueError, match="not an object"): + decode_payload(zlib.compress(b"[1, 2]")) + with pytest.raises(ValueError, match="unreadable"): + decode_payload(b"garbage") + with pytest.raises(ValueError, match="unreadable"): + decode_payload(zlib.compress(b"{not json")) + with pytest.raises(ValueError): # noqa: PT011 - json refuses NaN with allow_nan=False + encode_payload({"x": float("nan")}) + + +async def test_latest_headers_and_unreadable_rows(db: Database, plans: PlanSnapshotStore) -> None: + first = await plans.save({"n": 1}, source="a", schema_version=1, created_at=NOW) + second = await plans.save({"n": 2}, source="b", schema_version=2, created_at=NOW) + await db.execute( + "INSERT INTO plan_snapshot (created_at, source, schema_version, payload) " + "VALUES (?, 'bad', 1, ?)", + (NOW.isoformat(), b"not zlib"), + ) + latest = await plans.latest() + assert latest is not None + assert latest.id == second # the unreadable newest row is skipped + headers = await plans.headers() + assert [h.source for h in headers] == ["bad", "b", "a"] + assert [h.id for h in await plans.headers(limit=1)] == [headers[0].id] + assert await plans.count() == 3 + assert first < second + + +async def test_latest_is_none_without_readable_snapshots(plans: PlanSnapshotStore) -> None: + assert await plans.latest() is None + + +async def test_prune_keeps_at_most_n_and_14_days(plans: PlanSnapshotStore) -> None: + for i in range(10): + await plans.save( + {"i": i}, source="p", schema_version=1, created_at=NOW - timedelta(hours=i) + ) + assert await plans.prune(NOW, keep_last=4) == 6 + assert [h.id for h in await plans.headers()] == [10, 9, 8, 7] + await plans.save({"old": 1}, source="p", schema_version=1, created_at=NOW - timedelta(days=20)) + # the newest id is the old one: the age rule removes it, the others stay + assert await plans.prune(NOW) == 1 + assert await plans.count() == 4 + + +async def test_prune_defaults_are_200_and_14_days(plans: PlanSnapshotStore) -> None: + from suncore.storage.plans import KEEP_DAYS, KEEP_LAST # noqa: PLC0415 + + assert (KEEP_LAST, KEEP_DAYS) == (200, 14) + assert await plans.prune(NOW) == 0 + + +async def test_prune_by_age(plans: PlanSnapshotStore) -> None: + await plans.save({"a": 1}, source="p", schema_version=1, created_at=NOW - timedelta(days=15)) + await plans.save({"b": 1}, source="p", schema_version=1, created_at=NOW - timedelta(days=13)) + assert await plans.prune(NOW) == 1 + remaining = await plans.latest() + assert remaining is not None + assert remaining.payload == {"b": 1} + + +# --- learned parameters -------------------------------------------------------------------- + + +@pytest.fixture +def learned(db: Database) -> LearnedParamStore: + return LearnedParamStore(db, clock=lambda: NOW) + + +async def test_learned_round_trip_and_replace(learned: LearnedParamStore) -> None: + await learned.put(NS_FORECAST_BIAS, "solcast", {"hour_12": 1.05}, schema_version=2) + got = await learned.get(NS_FORECAST_BIAS, "solcast") + assert got is not None + assert (got.payload, got.schema_version, got.updated_at) == ({"hour_12": 1.05}, 2, NOW) + await learned.put(NS_FORECAST_BIAS, "solcast", {"hour_12": 1.1}) + replaced = await learned.get(NS_FORECAST_BIAS, "solcast") + assert replaced is not None + assert (replaced.payload, replaced.schema_version) == ({"hour_12": 1.1}, 1) + assert await learned.get(NS_FORECAST_BIAS, "other") is None + + +async def test_learned_namespaces_are_separate(learned: LearnedParamStore) -> None: + await learned.put(NS_FORECAST_BIAS, "b", {"x": 1}) + await learned.put(NS_FORECAST_BIAS, "a", {"x": 2}) + await learned.put(NS_BLEND_WEIGHTS, "a", {"w": 0.5}) + listed = await learned.in_namespace(NS_FORECAST_BIAS) + assert [p.key for p in listed] == ["a", "b"] + assert await learned.in_namespace(NS_ML_STATE) == [] + assert await learned.delete(NS_FORECAST_BIAS, "a") is True + assert await learned.delete(NS_FORECAST_BIAS, "a") is False + assert await learned.delete_namespace(NS_FORECAST_BIAS) == 1 + assert len(await learned.in_namespace(NS_BLEND_WEIGHTS)) == 1 + + +async def test_learned_unreadable_payload_is_missing( + db: Database, learned: LearnedParamStore +) -> None: + await db.execute( + "INSERT INTO learned_param VALUES ('net_error', 'list', '[1, 2]', 1, ?)", (NOW.isoformat(),) + ) + await learned.put("net_error", "ok", {"q": 1}) + assert await learned.get("net_error", "list") is None + assert [p.key for p in await learned.in_namespace("net_error")] == ["ok"] + + +async def test_learned_rejects_bad_names_and_non_json( + db: Database, learned: LearnedParamStore +) -> None: + with pytest.raises(ValueError, match="namespace"): + await learned.put("Bad Name", "k", {}) + with pytest.raises(ValueError, match="key"): + await learned.put(NS_ML_STATE, "", {}) + with pytest.raises(ValueError, match="key"): + await learned.get(NS_ML_STATE, "x" * 201) + with pytest.raises(ValueError, match="namespace"): + await learned.in_namespace("../x") + with pytest.raises(ValueError, match=r"NaN|Out of range"): + await learned.put(NS_ML_STATE, "k", {"x": float("nan")}) + assert await db.fetchone("SELECT COUNT(*) FROM learned_param") == (0,) diff --git a/tests/storage/test_runtime_store.py b/tests/storage/test_runtime_store.py index eb8f0a0..06d0a38 100644 --- a/tests/storage/test_runtime_store.py +++ b/tests/storage/test_runtime_store.py @@ -44,7 +44,7 @@ async def test_v2_database_migrates_to_v3_and_keeps_data(tmp_path: Path) -> None assert await run_migrations(conn, migrations) == 2 await conn.execute("INSERT INTO learned VALUES ('k', '1', 't')") async with await Database.open(path) as db: - assert await db.user_version() == latest_version() == 3 + assert await db.user_version() == latest_version() == 4 assert await db.fetchone("SELECT value FROM learned") == ("1",) tables = {r[0] for r in await db.fetchall("SELECT name FROM sqlite_master")} assert {"device_runtime", "safety_state", "power_model_state"} <= tables @@ -52,8 +52,8 @@ async def test_v2_database_migrates_to_v3_and_keeps_data(tmp_path: Path) -> None async def test_reader_expecting_v2_refuses_v3(tmp_path: Path) -> None: path = tmp_path / "v3.db" - async with await Database.open(path): - pass + async with aiosqlite.connect(path, isolation_level=None) as conn: + assert await run_migrations(conn, load_migrations()[:3]) == 3 async with aiosqlite.connect(path, isolation_level=None) as conn: with pytest.raises(SchemaTooNewError) as info: await run_migrations(conn, load_migrations()[:2]) diff --git a/tests/storage/test_timeseries.py b/tests/storage/test_timeseries.py new file mode 100644 index 0000000..3d53f26 --- /dev/null +++ b/tests/storage/test_timeseries.py @@ -0,0 +1,650 @@ +# SPDX-License-Identifier: Apache-2.0 +"""``timeseries.db``: migrations, repository round trips, downsampling and retention.""" + +from __future__ import annotations + +import asyncio +import sqlite3 +from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta +from pathlib import Path +from zoneinfo import ZoneInfo + +import pytest + +from suncore.core.measurements import SITE_GRID_W, SITE_SOC_PCT, device_power_key +from suncore.core.series import ( + PriceSeries, + QuantileSeries, + SeriesUnit, + TimeRange, + TimeSeries, + floor_to_step, +) +from suncore.storage import Database, SchemaTooNewError +from suncore.storage.migrations import latest_version +from suncore.storage.timeseries import ( + MINUTE_KEEP, + RAW_KEEP, + TimeseriesStore, + open_timeseries, + open_timeseries_or_recover, + timeseries_migrations, +) + +T0 = datetime(2026, 10, 1, 0, 0, tzinfo=UTC) +Q = timedelta(minutes=15) + + +@pytest.fixture +async def tsdb(tmp_path: Path) -> AsyncIterator[Database]: + db = await open_timeseries(tmp_path / "timeseries.db") + yield db + await db.close() + + +@pytest.fixture +def store(tsdb: Database) -> TimeseriesStore: + return TimeseriesStore(tsdb) + + +def _series(values: list[float | None], start: datetime = T0, step: timedelta = Q) -> TimeSeries: + return TimeSeries(start=start, step=step, values=tuple(values), unit=SeriesUnit.W) + + +def _quantiles( + p10: list[float | None], p50: list[float | None], p90: list[float | None], start: datetime = T0 +) -> QuantileSeries: + return QuantileSeries(p10=_series(p10, start), p50=_series(p50, start), p90=_series(p90, start)) + + +# --- migrations ----------------------------------------------------------------------- + + +async def test_fresh_timeseries_db_is_migrated_wal(tmp_path: Path) -> None: + async with await open_timeseries(tmp_path / "t.db") as db: + assert await db.user_version() == len(timeseries_migrations()) >= 1 + assert await db.fetchone("PRAGMA journal_mode") == ("wal",) + tables = {r[0] for r in await db.fetchall("SELECT name FROM sqlite_master")} + assert {"measurement", "measurement_1m", "measurement_15m", "forecast", "price"} <= tables + + +async def test_timeseries_migrations_are_independent_of_suncore_db(tmp_path: Path) -> None: + async with await open_timeseries(tmp_path / "t.db") as db: + tables = {r[0] for r in await db.fetchall("SELECT name FROM sqlite_master")} + assert "config_doc" not in tables + assert "plan_snapshot" not in tables + assert latest_version() >= 4 # suncore.db has its own, longer history + + +async def test_reopen_keeps_data(tmp_path: Path) -> None: + path = tmp_path / "t.db" + async with await open_timeseries(path) as db: + await TimeseriesStore(db).append_measurements([("k", T0, 1.0)]) + async with await open_timeseries(path) as db: + assert await TimeseriesStore(db).read_raw("k", TimeRange(start=T0, end=T0 + Q)) == [ + (T0, 1.0) + ] + + +async def test_newer_timeseries_db_is_refused_and_untouched(tmp_path: Path) -> None: + path = tmp_path / "t.db" + con = sqlite3.connect(path) + con.execute("PRAGMA user_version = 99") + con.close() + with pytest.raises(SchemaTooNewError): + await open_timeseries(path) + assert await open_timeseries_or_recover(path) is None + con = sqlite3.connect(path) + assert con.execute("PRAGMA user_version").fetchone() == (99,) + con.close() + assert not (tmp_path / "t.db.corrupt").exists() + + +async def test_corrupt_timeseries_db_is_moved_aside_with_its_wal(tmp_path: Path) -> None: + path = tmp_path / "t.db" + path.write_bytes(b"this is not a database" * 100) + db = await open_timeseries_or_recover(path, now=lambda: 1_800_000_000.0) + assert db is not None + try: + assert await db.user_version() >= 1 + finally: + await db.close() + moved = tmp_path / "t.db.corrupt-20270115T080000Z" + assert moved.read_bytes().startswith(b"this is not") + + +def test_move_aside_takes_wal_and_shm_along_and_never_unlinks_them(tmp_path: Path) -> None: + from suncore.storage.timeseries import _move_aside # noqa: PLC0415 + + path = tmp_path / "t.db" + for suffix, data in (("", b"db"), ("-wal", b"wal"), ("-shm", b"shm")): + (tmp_path / f"t.db{suffix}").write_bytes(data) + _move_aside(path, 1_800_000_000.0) + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "t.db.corrupt-20270115T080000Z", + "t.db.corrupt-20270115T080000Z-shm", + "t.db.corrupt-20270115T080000Z-wal", + ] + assert (tmp_path / "t.db.corrupt-20270115T080000Z-wal").read_bytes() == b"wal" + + +async def test_failing_quick_check_counts_as_corruption( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import suncore.storage.timeseries as module # noqa: PLC0415 + + path = tmp_path / "t.db" + await (await open_timeseries(path)).close() # a valid file first + calls = {"n": 0} + + async def fake_check(_db: Database) -> bool: + calls["n"] += 1 + return calls["n"] > 1 # the first (damaged) file fails, the fresh one passes + + monkeypatch.setattr(module, "_quick_check", fake_check) + db = await open_timeseries_or_recover(path, now=lambda: 1_800_000_000.0) + assert db is not None + await db.close() + assert (tmp_path / "t.db.corrupt-20270115T080000Z").exists() + + +@pytest.mark.parametrize( + "error", + [ + sqlite3.OperationalError("database is locked"), + sqlite3.OperationalError("disk I/O error"), + sqlite3.OperationalError("database or disk is full"), + sqlite3.OperationalError("unable to open database file"), + PermissionError("denied"), + ], +) +async def test_transient_errors_never_move_the_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, error: Exception +) -> None: + import suncore.storage.timeseries as module # noqa: PLC0415 + + path = tmp_path / "t.db" + async with await open_timeseries(path) as db: + await TimeseriesStore(db).append_measurements([("k", T0, 1.0)]) + + async def failing(_path: Path | str) -> Database: + raise error + + monkeypatch.setattr(module, "open_timeseries", failing) + assert await open_timeseries_or_recover(path) is None + monkeypatch.undo() + assert not await asyncio.to_thread(lambda: list(tmp_path.glob("*.corrupt*"))) + async with await open_timeseries(path) as db: # still there, data intact, next start works + assert await TimeseriesStore(db).read_raw("k", TimeRange(start=T0, end=T0 + Q)) == [ + (T0, 1.0) + ] + + +async def test_corrupt_files_are_rotated_keeping_two(tmp_path: Path) -> None: + path = tmp_path / "t.db" + for i in range(4): + path.write_bytes(b"garbage" * 100) + (tmp_path / "t.db-wal").write_bytes(b"w") + db = await open_timeseries_or_recover(path, now=lambda i=i: 1_800_000_000.0 + i * 60) + assert db is not None + await db.close() + path.unlink() + for suffix in ("-wal", "-shm"): + (tmp_path / f"t.db{suffix}").unlink(missing_ok=True) + kept = await asyncio.to_thread(lambda: sorted(p.name for p in tmp_path.glob("t.db.corrupt-*"))) + assert kept == ["t.db.corrupt-20270115T080200Z", "t.db.corrupt-20270115T080300Z"] + + +async def test_unopenable_path_gives_none(tmp_path: Path) -> None: + assert await open_timeseries_or_recover(tmp_path / "missing" / "dir" / "t.db") is None + assert not await asyncio.to_thread(lambda: list(tmp_path.glob("**/*.corrupt*"))) + + +# --- measurements --------------------------------------------------------------------- + + +async def test_append_skips_gaps_and_replaces_duplicates(store: TimeseriesStore) -> None: + written = await store.append_measurements( + [ + ("k", T0, 1.0), + ("k", T0 + timedelta(seconds=5), None), + ("k", T0 + timedelta(seconds=10), float("nan")), + ("k", T0 + timedelta(seconds=15), float("inf")), + ("k", T0 + timedelta(seconds=20), 2.0), + ("k", T0 + timedelta(seconds=20, milliseconds=400), 3.0), # same second: replaces + ] + ) + assert written == 3 + rows = await store.read_raw("k", TimeRange(start=T0, end=T0 + timedelta(minutes=1))) + assert rows == [(T0, 1.0), (T0 + timedelta(seconds=20), 3.0)] + + +async def test_naive_timestamps_are_rejected(store: TimeseriesStore) -> None: + with pytest.raises(ValueError, match="timezone-aware"): + await store.append_measurements([("k", datetime(2026, 1, 1), 1.0)]) # noqa: DTZ001 + + +async def test_downsample_is_sample_weighted_and_energy_preserving(store: TimeseriesStore) -> None: + # 5 s ticks for 30 minutes; a ramp, so every minute has a different mean + rows = [("k", T0 + timedelta(seconds=5 * i), float(i % 97)) for i in range(360)] + await store.append_measurements(rows) + now = T0 + timedelta(minutes=45) + report = await store.downsample(now) + assert report.minute_rows == 30 + assert report.quarter_rows == 2 + raw_mean = sum(v for _, _, v in rows[:180]) / 180 + q = await store.read_series( + "k", TimeRange(start=T0, end=T0 + timedelta(minutes=30)), "15m", include_recent=False + ) + assert q.values[0] == pytest.approx(raw_mean) + # energy of the first quarter hour: mean W x 0.25 h equals the sum of the samples x 5 s + sample_wh = sum(v for _, _, v in rows[:180]) * 5 / 3600 + assert q.values[0] * 0.25 == pytest.approx(sample_wh) + row = await store.database.fetchone( + "SELECT min, max, n FROM measurement_15m WHERE ts = ?", (int(T0.timestamp()),) + ) + assert row == (0.0, 96.0, 180) + + +async def test_downsample_is_idempotent(store: TimeseriesStore) -> None: + await store.append_measurements( + [("k", T0 + timedelta(seconds=10 * i), float(i)) for i in range(200)] + ) + now = T0 + timedelta(hours=1) + await store.downsample(now) + first = await store.database.fetchall("SELECT * FROM measurement_15m ORDER BY key, ts") + first_1m = await store.database.fetchall("SELECT * FROM measurement_1m ORDER BY key, ts") + await store.downsample(now) + await store.downsample(now + timedelta(hours=1)) + assert await store.database.fetchall("SELECT * FROM measurement_15m ORDER BY key, ts") == first + assert ( + await store.database.fetchall("SELECT * FROM measurement_1m ORDER BY key, ts") == first_1m + ) + + +async def test_incomplete_buckets_are_not_aggregated(store: TimeseriesStore) -> None: + await store.append_measurements( + [("k", T0 + timedelta(seconds=s), 1.0) for s in range(0, 100, 10)] + ) + # now = 00:01:40: the quarter hour 00:00 is incomplete, so not even its minutes are written + report = await store.downsample(T0 + timedelta(seconds=100)) + assert (report.minute_rows, report.quarter_rows) == (0, 0) + + +async def test_a_job_inside_a_quarter_hour_does_not_skew_the_quarter( + store: TimeseriesStore, +) -> None: + """Review example: job at 03:35, 5 min x 100 W before it, 10 min x 1000 W after it.""" + quarter = T0 + timedelta(hours=3, minutes=30) + window = TimeRange(start=quarter, end=quarter + Q) + low = [("k", quarter + timedelta(seconds=10 * i), 100.0) for i in range(30)] + high = [("k", quarter + timedelta(minutes=5, seconds=10 * i), 1000.0) for i in range(60)] + await store.append_measurements(low) + await store.downsample(quarter + timedelta(minutes=5)) # the 03:35 job + await store.append_measurements(high) + expected = (30 * 100 + 60 * 1000) / 90 # 700 + assert (await store.read_series("k", window)).values == (pytest.approx(expected),) + await store.downsample(quarter + timedelta(minutes=20)) # a later job completes it + assert (await store.read_series("k", window, include_recent=False)).values == ( + pytest.approx(expected), + ) + assert (await store.read_series("k", window)).values == (pytest.approx(expected),) + + +async def test_primary_keys_are_time_first_and_range_reads_use_them(store: TimeseriesStore) -> None: + """(ts, key) keeps one flush on consecutive pages (SD wear); reads still seek on ts.""" + db = store.database + for table in ("measurement", "measurement_1m"): + columns = await db.fetchall(f"PRAGMA table_info({table})") + order = sorted((int(str(c[5])), str(c[1])) for c in columns if int(str(c[5])) > 0) + assert [name for _, name in order] == ["ts", "key"] + plan = await db.fetchall( + "EXPLAIN QUERY PLAN SELECT ts, value FROM measurement WHERE key = ? AND ts >= ? AND ts < ?", + ("k", 0, 10), + ) + assert any("PRIMARY KEY" in str(r[3]) and "ts>" in str(r[3]) for r in plan) + + +async def test_measured_beats_derived_when_merging_recent_data(store: TimeseriesStore) -> None: + window = TimeRange(start=T0, end=T0 + Q) + await store.store_quarter_hours("k", _series([99.0]), derived=True) + await store.append_measurements([("k", T0 + timedelta(seconds=10 * i), 5.0) for i in range(30)]) + # not yet aggregated: the live data wins over the rebuilt value + assert (await store.read_series("k", window)).values == (5.0,) + assert (await store.read_series("k", window, include_recent=False)).values == (99.0,) + # a derived row alone is still used, and an aggregated measured row beats live data + other = TimeRange(start=T0 + Q, end=T0 + 2 * Q) + await store.store_quarter_hours("k", _series([7.0], start=T0 + Q), derived=True) + assert (await store.read_series("k", other)).values == (7.0,) + + +async def test_15m_weights_minutes_by_sample_count(store: TimeseriesStore) -> None: + # minute 0: 6 samples of 10; minute 1: 1 sample of 70 -> sample mean (60 + 70) / 7 + rows = [("k", T0 + timedelta(seconds=10 * i), 10.0) for i in range(6)] + rows.append(("k", T0 + timedelta(seconds=60), 70.0)) + await store.append_measurements(rows) + await store.downsample(T0 + timedelta(minutes=30)) + series = await store.read_series("k", TimeRange(start=T0, end=T0 + Q), include_recent=False) + assert series.values == (pytest.approx(130 / 7),) + + +async def test_retention_downsamples_before_it_deletes(store: TimeseriesStore) -> None: + old = T0 + await store.append_measurements( + [("k", old + timedelta(seconds=30 * i), 5.0) for i in range(60)] + ) + later = old + RAW_KEEP + timedelta(days=3) + report = await store.maintain(later) + assert report.retention.raw == 60 + assert await store.database.fetchone("SELECT COUNT(*) FROM measurement") == (0,) + # the aggregates survived the deletion of the raw rows + series = await store.read_series("k", TimeRange(start=old, end=old + timedelta(minutes=30))) + assert series.values[:2] == (5.0, 5.0) + + +async def test_retention_cutoffs_per_table(store: TimeseriesStore) -> None: + now = T0 + timedelta(days=100) + db = store.database + for table in ("measurement_1m", "measurement_15m"): + await db.execute( + f"INSERT INTO {table} (key, ts, mean, min, max, n) VALUES ('k', ?, 1, 1, 1, 1)", + (int((now - MINUTE_KEEP - timedelta(days=1)).timestamp()) // 900 * 900,), + ) + await db.execute( + f"INSERT INTO {table} (key, ts, mean, min, max, n) VALUES ('k', ?, 1, 1, 1, 1)", + (int((now - timedelta(days=1)).timestamp()) // 900 * 900,), + ) + report = await store.apply_retention(now) + assert report.minute == 1 + assert await db.fetchone("SELECT COUNT(*) FROM measurement_1m") == (1,) + assert await db.fetchone("SELECT COUNT(*) FROM measurement_15m") == (2,) # unlimited + + +async def test_maintain_on_an_empty_database_and_size_report(store: TimeseriesStore) -> None: + report = await store.maintain(T0) + assert report.retention.total == 0 + assert report.vacuumed is False + size = await store.size_report() + assert size.file_bytes > 0 + assert size.total_bytes >= size.file_bytes + assert set(size.rows) == { + "measurement", + "measurement_1m", + "measurement_15m", + "forecast", + "price", + } + assert all(count == 0 for count in size.rows.values()) + + +async def test_vacuum_runs_only_when_worth_it( + store: TimeseriesStore, monkeypatch: pytest.MonkeyPatch +) -> None: + import suncore.storage.timeseries as module # noqa: PLC0415 + + monkeypatch.setattr(module, "VACUUM_MIN_FREE_BYTES", 4096) + await store.append_measurements( + [(f"key{i % 50}", T0 + timedelta(seconds=i), float(i)) for i in range(20_000)] + ) + assert (await store.maintain(T0 + timedelta(hours=1))).vacuumed is False + before = (await store.size_report()).total_bytes + report = await store.maintain(T0 + RAW_KEEP + timedelta(days=10)) + assert report.retention.raw == 20_000 + assert report.vacuumed is True + assert (await store.size_report()).total_bytes < before + + +# --- read_series ---------------------------------------------------------------------- + + +async def test_read_series_gaps_are_none_and_units_follow_the_key(store: TimeseriesStore) -> None: + await store.store_quarter_hours(SITE_GRID_W, _series([1.0, None, 3.0])) + series = await store.read_series(SITE_GRID_W, TimeRange(start=T0, end=T0 + 4 * Q)) + assert series.values == (1.0, None, 3.0, None) + assert series.unit is SeriesUnit.W + assert series.step == Q + soc = await store.read_series(SITE_SOC_PCT, TimeRange(start=T0, end=T0 + Q)) + assert soc.unit is SeriesUnit.PERCENT + + +async def test_read_series_includes_the_not_yet_aggregated_recent_data( + store: TimeseriesStore, +) -> None: + await store.append_measurements([("k", T0 + timedelta(seconds=30 * i), 2.0) for i in range(30)]) + window = TimeRange(start=T0, end=T0 + Q) + assert (await store.read_series("k", window, include_recent=False)).values == (None,) + assert (await store.read_series("k", window)).values == (2.0,) + await store.downsample(T0 + timedelta(hours=1)) + await store.apply_retention(T0 + timedelta(hours=1)) + assert (await store.read_series("k", window)).values == (2.0,) + + +async def test_read_series_one_minute_resolution(store: TimeseriesStore) -> None: + await store.append_measurements( + [("k", T0 + timedelta(seconds=10 * i), float(i)) for i in range(12)] + ) + await store.downsample(T0 + timedelta(hours=1)) + series = await store.read_series("k", TimeRange(start=T0, end=T0 + timedelta(minutes=2)), "1m") + assert series.step == timedelta(minutes=1) + assert series.values == (2.5, 8.5) + + +async def test_read_series_refuses_a_huge_window(store: TimeseriesStore) -> None: + with pytest.raises(ValueError, match="exceed"): + await store.read_series("k", TimeRange(start=T0, end=T0 + timedelta(days=30)), "1m") + + +async def test_derived_rows_never_overwrite_measured_ones(store: TimeseriesStore) -> None: + await store.store_quarter_hours("k", _series([10.0, None]), derived=False) + await store.store_quarter_hours("k", _series([99.0, 20.0, 30.0]), derived=True) + series = await store.read_series("k", TimeRange(start=T0, end=T0 + 3 * Q)) + assert series.values == (10.0, 20.0, 30.0) + await store.store_quarter_hours( + "k", _series([1.0, 2.0, 3.0]), derived=True + ) # derived on derived + series = await store.read_series("k", TimeRange(start=T0, end=T0 + 3 * Q)) + assert series.values == (10.0, 2.0, 3.0) + assert await store.database.fetchall("SELECT derived FROM measurement_15m ORDER BY ts") == [ + (0,), + (1,), + (1,), + ] + + +async def test_real_aggregates_replace_derived_rows(store: TimeseriesStore) -> None: + await store.store_quarter_hours("k", _series([99.0]), derived=True) + assert await store.has_quarter_history() + await store.append_measurements([("k", T0 + timedelta(seconds=10 * i), 4.0) for i in range(90)]) + await store.downsample(T0 + timedelta(hours=1)) + assert (await store.read_series("k", TimeRange(start=T0, end=T0 + Q))).values == (4.0,) + assert await store.database.fetchone("SELECT derived, n FROM measurement_15m") == (0, 90) + + +async def test_store_quarter_hours_needs_a_15_minute_series(store: TimeseriesStore) -> None: + with pytest.raises(ValueError, match="15-minute"): + await store.store_quarter_hours("k", _series([1.0], step=timedelta(hours=1))) + + +async def test_has_quarter_history(store: TimeseriesStore) -> None: + assert not await store.has_quarter_history() + await store.store_quarter_hours("k", _series([1.0])) + assert await store.has_quarter_history() + + +async def test_device_keys_are_plain_measurements(store: TimeseriesStore) -> None: + key = device_power_key("pool") + await store.append_measurements([(key, T0, 1500.0)]) + assert (await store.read_raw(key, TimeRange(start=T0, end=T0 + Q))) == [(T0, 1500.0)] + + +# --- forecasts ------------------------------------------------------------------------ + + +async def test_forecast_round_trip_with_gaps(store: TimeseriesStore) -> None: + issued = T0 - timedelta(hours=2) + forecast = _quantiles( + [None, 1.0, 2.0, None], [None, 2.0, 3.0, 4.0], [None, 3.0, 4.0, None], start=T0 + ) + assert await store.store_forecast("solcast", "pv", issued, forecast) == 4 + assert await store.read_forecast("solcast", "pv", issued) == forecast # leading None kept + assert await store.read_forecast("solcast", "load", issued) is None + assert await store.read_forecast("other", "pv", issued) is None + + +async def test_forecast_restore_replaces_the_same_issue(store: TimeseriesStore) -> None: + issued = T0 + await store.store_forecast("s", "pv", issued, QuantileSeries.deterministic(_series([1.0, 2.0]))) + await store.store_forecast("s", "pv", issued, QuantileSeries.deterministic(_series([5.0]))) + got = await store.read_forecast("s", "pv", issued) + assert got is not None + assert got.p50.values == (5.0,) + + +async def test_forecast_window_clip_and_plain_time_series(store: TimeseriesStore) -> None: + await store.store_forecast("s", "load", T0, _series([1.0, 2.0, 3.0, 4.0])) + got = await store.read_forecast("s", "load", T0, TimeRange(start=T0 + Q, end=T0 + 3 * Q)) + assert got is not None + assert got.p50.values == (2.0, 3.0) + assert got.p10.values == (None, None) # a plain series fills p50 only + assert got.start == T0 + Q + + +async def test_latest_forecast_before_semantics(store: TimeseriesStore) -> None: + # issue A (00:00) covers 00:00..00:45 with 1; issue B (00:30) covers 00:30..01:15 with 2 + await store.store_forecast("s", "pv", T0, QuantileSeries.deterministic(_series([1.0] * 4))) + await store.store_forecast( + "s", "pv", T0 + 2 * Q, QuantileSeries.deterministic(_series([2.0] * 4, start=T0 + 2 * Q)) + ) + before_b = await store.latest_forecast_before("s", "pv", T0 + Q) + assert before_b is not None + assert before_b.p50.values == (1.0, 1.0, 1.0, 1.0) + after_b = await store.latest_forecast_before("s", "pv", T0 + 2 * Q) + assert after_b is not None + assert after_b.p50.values == (1.0, 1.0, 2.0, 2.0, 2.0, 2.0) + assert await store.latest_forecast_before("s", "pv", T0 - Q) is None + assert await store.forecast_issues("s", "pv") == [T0, T0 + 2 * Q] + assert await store.forecast_issues("s", "pv", since=T0 + Q) == [T0 + 2 * Q] + + +async def test_forecast_retention_by_slot(store: TimeseriesStore) -> None: + await store.store_forecast("s", "pv", T0, QuantileSeries.deterministic(_series([1.0]))) + report = await store.apply_retention(T0 + timedelta(days=60)) + assert report.forecast == 1 + + +# --- prices --------------------------------------------------------------------------- + + +def _prices( + imp: list[float | None], + exp: list[float | None] | None, + start: datetime = T0, + step: timedelta = Q, +) -> PriceSeries: + def make(values: list[float | None]) -> TimeSeries: + return TimeSeries(start=start, step=step, values=tuple(values), unit=SeriesUnit.PRICE) + + return PriceSeries( + source_id="tibber", + price_includes="all_in", + import_price=make(imp), + export_price=None if exp is None else make(exp), + ) + + +async def test_prices_round_trip_with_negative_values_and_gaps(store: TimeseriesStore) -> None: + prices = _prices([0.30, -0.05, None, 0.28], [0.08, 0.0, None, 0.08]) + await store.store_prices(prices) + got = await store.read_prices("tibber", TimeRange(start=T0, end=T0 + 4 * Q)) + assert got == [prices] + + +async def test_prices_without_export_series(store: TimeseriesStore) -> None: + prices = _prices([0.3, 0.4], None) + await store.store_prices(prices) + got = await store.read_prices("tibber", TimeRange(start=T0, end=T0 + 2 * Q)) + assert got == [prices] + assert got[0].export_price is None + + +async def test_prices_mixed_resolution_segments_and_replacement(store: TimeseriesStore) -> None: + quarters = _prices([0.1, 0.2, 0.3, 0.4], None) + hourly = _prices([0.5, 0.6], None, start=T0 + timedelta(hours=1), step=timedelta(hours=1)) + await store.store_prices(quarters) + await store.store_prices(hourly) + got = await store.read_prices("tibber", TimeRange(start=T0, end=T0 + timedelta(hours=3))) + assert [s.step for s in got] == [Q, timedelta(hours=1)] + assert got[1].import_price.values == (0.5, 0.6) + # an hourly update of the first hour replaces the four quarter-hour rows it overlaps + await store.store_prices(_prices([0.9], None, step=timedelta(hours=1))) + got = await store.read_prices("tibber", TimeRange(start=T0, end=T0 + timedelta(hours=3))) + assert [s.step for s in got] == [timedelta(hours=1)] + assert got[0].import_price.values == (0.9, 0.5, 0.6) + + +async def test_prices_are_per_source_and_retained_long(store: TimeseriesStore) -> None: + await store.store_prices(_prices([0.3], None)) + assert await store.read_prices("nordpool", TimeRange(start=T0, end=T0 + Q)) == [] + assert (await store.apply_retention(T0 + timedelta(days=300))).price == 0 + assert (await store.apply_retention(T0 + timedelta(days=500))).price == 1 + + +async def test_price_gap_that_is_not_a_whole_slot_starts_a_new_segment( + store: TimeseriesStore, +) -> None: + await store.store_prices(_prices([0.1], None)) + await store.store_prices(_prices([0.2], None, start=T0 + timedelta(minutes=20))) + got = await store.read_prices("tibber", TimeRange(start=T0, end=T0 + timedelta(hours=1))) + assert len(got) == 2 + + +# --- DST ------------------------------------------------------------------------------ + + +@pytest.mark.parametrize( + ("day", "hours"), + [("2026-03-29", 23), ("2026-10-25", 25), ("2026-06-01", 24)], +) +async def test_series_round_trip_on_dst_days(store: TimeseriesStore, day: str, hours: int) -> None: + berlin = ZoneInfo("Europe/Berlin") + start = datetime.fromisoformat(day).replace(tzinfo=berlin).astimezone(UTC) + end = (datetime.fromisoformat(day) + timedelta(days=1)).replace(tzinfo=berlin).astimezone(UTC) + assert (end - start) == timedelta(hours=hours) + count = hours * 4 + values: list[float | None] = [float(i) if i % 7 else None for i in range(count)] + series = _series(values, start=start) + await store.store_quarter_hours("k", series) + got = await store.read_series("k", TimeRange(start=start, end=end)) + assert got.values == series.values + assert got.start == start + assert len(got) == count + # forecast issued the day before round-trips as well + forecast = QuantileSeries.deterministic(series) + await store.store_forecast("s", "pv", start - timedelta(hours=6), forecast) + assert await store.read_forecast("s", "pv", start - timedelta(hours=6)) == forecast + + +async def test_timestamps_are_stored_in_utc_whatever_the_input_zone(store: TimeseriesStore) -> None: + berlin = ZoneInfo("Europe/Berlin") + local = datetime(2026, 10, 25, 2, 30, tzinfo=berlin, fold=1) # the second 02:30 + await store.append_measurements([("k", local, 5.0)]) + utc = local.astimezone(UTC) # local arithmetic would reset `fold`: windows are UTC + rows = await store.read_raw("k", TimeRange(start=utc, end=utc + Q)) + assert rows == [(utc, 5.0)] + assert utc == datetime(2026, 10, 25, 1, 30, tzinfo=UTC) + assert floor_to_step(utc) == utc + + +@pytest.mark.parametrize("app", ["suncore", "suncore_beta"]) +def test_backup_excludes_the_database_its_wal_and_the_corrupt_copies(app: str) -> None: + config = (Path(__file__).resolve().parents[2] / "addon" / app / "config.yaml").read_text( + "utf-8" + ) + for entry in ( + "timeseries.db", + "timeseries.db-wal", + "timeseries.db-shm", + "timeseries.db.corrupt*", + ): + assert f" - {entry}\n" in config diff --git a/tests/storage/test_timeseries_properties.py b/tests/storage/test_timeseries_properties.py new file mode 100644 index 0000000..6757ef7 --- /dev/null +++ b/tests/storage/test_timeseries_properties.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Hypothesis properties of the downsampling: energy, idempotence, bucket boundaries.""" + +from __future__ import annotations + +import asyncio +import tempfile +from collections import defaultdict +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from suncore.storage.timeseries import TimeseriesStore, open_timeseries + +T0 = datetime(2026, 10, 1, tzinfo=UTC) +T0_S = int(T0.timestamp()) + +samples = st.lists( + st.tuples( + st.sampled_from(["a", "b"]), + st.integers(min_value=0, max_value=4 * 3600), # random gaps: sparse and clustered + st.integers(min_value=-500, max_value=3000), + ), + max_size=120, + unique_by=lambda s: (s[0], s[1]), +) +job_offsets = st.integers(min_value=0, max_value=6 * 3600) + + +async def _run(rows: list[tuple[str, int, int]], jobs: list[int]) -> tuple[list, list, list]: + with tempfile.TemporaryDirectory() as tmp: + async with await open_timeseries(Path(tmp) / "t.db") as db: + store = TimeseriesStore(db) + await store.append_measurements( + [(k, T0 + timedelta(seconds=o), float(v)) for k, o, v in rows] + ) + for job in jobs: + await store.downsample(T0 + timedelta(seconds=job)) + minute = await db.fetchall("SELECT key, ts, mean, min, max, n FROM measurement_1m") + quarter = await db.fetchall("SELECT key, ts, mean, min, max, n FROM measurement_15m") + snapshot = [await db.fetchall("SELECT * FROM measurement_15m ORDER BY key, ts")] + for job in jobs: # the same jobs again: nothing may change + await store.downsample(T0 + timedelta(seconds=job)) + again = await db.fetchall("SELECT * FROM measurement_15m ORDER BY key, ts") + return minute, quarter, [snapshot[0], again] + + +def _go(rows: list[tuple[str, int, int]], jobs: list[int]) -> tuple[list, list, list]: + return asyncio.run(_run(rows, jobs)) + + +FAST = settings( + max_examples=40, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture] +) + + +@FAST +@given(rows=samples, job=job_offsets) +def test_energy_is_preserved_per_complete_quarter(rows: list, job: int) -> None: + minute, quarter, _ = _go(rows, [job]) + horizon = (T0_S + job) // 900 * 900 # complete quarter hours end at or before this + by_quarter: dict[tuple[str, int], list[int]] = defaultdict(list) + for key, offset, value in rows: + bucket = (T0_S + offset) // 900 * 900 + if bucket < horizon: + by_quarter[(key, bucket)].append(value) + stored = {(str(r[0]), int(r[1])): r for r in quarter} + assert set(stored) == set(by_quarter) + for bucket, values in by_quarter.items(): + _key, _ts, mean, low, high, n = stored[bucket] + assert n == len(values) + assert abs(mean * n - sum(values)) < 1e-6 * max(1.0, abs(sum(values))) + assert (low, high) == (min(values), max(values)) + # the 1-minute tier adds up to the same energy + total = float(sum(sum(v) for v in by_quarter.values())) + minutes_total = sum(float(r[2]) * int(r[5]) for r in minute) + assert abs(minutes_total - total) < 1e-6 * max(1.0, abs(total)) + + +@FAST +@given(rows=samples, jobs=st.lists(job_offsets, min_size=1, max_size=3)) +def test_downsampling_is_idempotent_for_any_job_times(rows: list, jobs: list[int]) -> None: + _, _, (first, again) = _go(rows, jobs) + assert first == again + + +@FAST +@given(rows=samples, job=job_offsets) +def test_rows_sit_on_bucket_boundaries_and_never_in_the_future(rows: list, job: int) -> None: + minute, quarter, _ = _go(rows, [job]) + horizon = (T0_S + job) // 900 * 900 + assert all(int(r[1]) % 60 == 0 and int(r[1]) < horizon for r in minute) + assert all(int(r[1]) % 900 == 0 and int(r[1]) < horizon for r in quarter) + assert all(int(r[5]) > 0 for r in minute + quarter)