From 6c7a9579ddeb3211486b83cd514a53ae1acacdf0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 02:44:14 +0000 Subject: [PATCH 01/11] feat(storage): add the time-series database, statistics rebuild and plan snapshots (WP3) - timeseries.db as a separate SQLite file with its own migrations: raw measurements (48 h), 1-minute (30 days) and 15-minute aggregates (unlimited), forecasts versioned by issue time, prices; TimeseriesStore with batched appends, core-series round trips, latest forecast before t, idempotent nightly downsample/retention and VACUUM only when worth it. - Rebuild of the 15-minute history from recorder/statistics_during_period (hour period, explicit units, chunked), stored as derived rows; HAConnection.statistics_during_period plus a recorder fake in the simulator. - suncore.db migration 0004: plan_snapshot and learned_param with small repositories (schema 4; earlier versions refuse it). - Per-(domain, service) rate limiter with try-acquire/deadline for call_action_response (follow-up A3); boundary rule 7 admits suncore.app.adapters only. - Runtime wiring: batched measurement recorder (flush every 60 s), nightly job, rebuild after the first connect; errors are logged and never reach the control loop. - Fix: a cancelled writer could leave a transaction open on the shared connection. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- frontend/src/lib/i18n/locales/de.json | 1 + frontend/src/lib/i18n/locales/en.json | 1 + scripts/check_reasons.py | 4 +- suncore/app/adapters.py | 14 + suncore/app/control.py | 47 +- suncore/app/history.py | 172 +++++ suncore/app/main.py | 15 + suncore/app/recorder.py | 75 ++ suncore/app/runtime.py | 125 ++- suncore/core/measurements.py | 51 ++ suncore/ha/__init__.py | 4 + suncore/ha/connection.py | 117 +++ suncore/ha/statistics.py | 251 +++++++ suncore/ha/websocket.py | 63 +- suncore/sim/fake_ha.py | 36 +- suncore/sim/recorder.py | 99 +++ suncore/storage/__init__.py | 15 + suncore/storage/db.py | 46 +- suncore/storage/learned.py | 147 ++++ .../storage/migrations/0004_plans_learned.sql | 25 + suncore/storage/plans.py | 187 +++++ suncore/storage/timeseries.py | 711 ++++++++++++++++++ .../storage/ts_migrations/0001_initial.sql | 62 ++ suncore/storage/ts_migrations/__init__.py | 6 + tests/app/test_history.py | 160 ++++ tests/app/test_timeseries_runtime.py | 525 +++++++++++++ tests/ha/fakes.py | 8 +- tests/ha/test_action_rate_limit.py | 224 ++++++ tests/ha/test_statistics.py | 282 +++++++ tests/safety/test_boundary.py | 24 +- tests/storage/test_config_sections.py | 2 +- tests/storage/test_db_cancellation.py | 41 + tests/storage/test_plans_learned.py | 203 +++++ tests/storage/test_runtime_store.py | 6 +- tests/storage/test_timeseries.py | 504 +++++++++++++ 35 files changed, 4223 insertions(+), 30 deletions(-) create mode 100644 suncore/app/adapters.py create mode 100644 suncore/app/history.py create mode 100644 suncore/app/recorder.py create mode 100644 suncore/core/measurements.py create mode 100644 suncore/ha/statistics.py create mode 100644 suncore/sim/recorder.py create mode 100644 suncore/storage/learned.py create mode 100644 suncore/storage/migrations/0004_plans_learned.sql create mode 100644 suncore/storage/plans.py create mode 100644 suncore/storage/timeseries.py create mode 100644 suncore/storage/ts_migrations/0001_initial.sql create mode 100644 suncore/storage/ts_migrations/__init__.py create mode 100644 tests/app/test_history.py create mode 100644 tests/app/test_timeseries_runtime.py create mode 100644 tests/ha/test_action_rate_limit.py create mode 100644 tests/ha/test_statistics.py create mode 100644 tests/storage/test_db_cancellation.py create mode 100644 tests/storage/test_plans_learned.py create mode 100644 tests/storage/test_timeseries.py diff --git a/frontend/src/lib/i18n/locales/de.json b/frontend/src/lib/i18n/locales/de.json index 9f24c8f..0a28c0f 100644 --- a/frontend/src/lib/i18n/locales/de.json +++ b/frontend/src/lib/i18n/locales/de.json @@ -408,6 +408,7 @@ "grid.power": "Netzleistung (positiv = Bezug)", "grid.power_hint": "Ein Sensor, der positiv ist, wenn du Strom beziehst, und negativ bei Einspeisung.", "grid.sensors": "Netzsensoren", + "ha.error.action_rate_limited": "Eine Home-Assistant-Aktion wurde zu oft angefordert und übersprungen; SunCore fragt nach dem Mindestabstand erneut an.", "log.all_devices": "Alle Geräte", "log.count": "{count, plural, =0 {Keine Einträge} one {# Eintrag} other {# Einträge}}", "log.device": "Gerät", diff --git a/frontend/src/lib/i18n/locales/en.json b/frontend/src/lib/i18n/locales/en.json index bbc7daf..0bf9c58 100644 --- a/frontend/src/lib/i18n/locales/en.json +++ b/frontend/src/lib/i18n/locales/en.json @@ -408,6 +408,7 @@ "grid.power": "Grid power (positive = import)", "grid.power_hint": "One sensor that is positive when you draw from the grid and negative when you feed in.", "grid.sensors": "Grid sensors", + "ha.error.action_rate_limited": "A Home Assistant action was requested too often and skipped; SunCore asks again after the minimum interval.", "log.all_devices": "All devices", "log.count": "{count, plural, =0 {No entries} one {# entry} other {# entries}}", "log.device": "Device", diff --git a/scripts/check_reasons.py b/scripts/check_reasons.py index 3fcf235..0bbf5cc 100644 --- a/scripts/check_reasons.py +++ b/scripts/check_reasons.py @@ -8,7 +8,7 @@ * string constants starting with ``api.error.`` (the error ``detail`` keys of the API) or ``api.validation.`` in ``suncore/api`` (validation issue keys), * string constants that are exactly one key with the prefix ``wizard.`` (check and import - reasons built from a variable) or ``detect.criterion.`` anywhere + reasons built from a variable), ``detect.criterion.`` or ``ha.error.`` anywhere in ``suncore/``, and ``controller.`` in ``suncore/controller/reasons.py`` (the controller's key constants). @@ -28,7 +28,7 @@ SOURCE = ROOT / "suncore" LOCALES = ROOT / "frontend" / "src" / "lib" / "i18n" / "locales" ERROR_PREFIXES = ("api.error.", "api.validation.") -KEY_PREFIXES = ("wizard.", "detect.criterion.") +KEY_PREFIXES = ("wizard.", "detect.criterion.", "ha.error.") CONTROLLER_KEYS = Path("controller") / "reasons.py" """The controller defines its keys as constants in one module (log event names elsewhere may share the ``controller.`` prefix).""" diff --git a/suncore/app/adapters.py b/suncore/app/adapters.py new file mode 100644 index 0000000..1c7e2d5 --- /dev/null +++ b/suncore/app/adapters.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Composition of the tariff and forecast adapters (Phase 4 WP4/WP5). + +This module will build the price and forecast sources from the configured templates and hand +them the Home Assistant connection. It is the only module of ``suncore.app`` that may call +``HAConnection.call_action_response`` (``tests/safety/test_boundary.py``, rule 7): the call is a +read-only action with ``return_response`` and is rate limited per ``(domain, service)`` by the +connection (``ActionRateLimiter``). An adapter passes the polling interval of its template as +``min_interval_s`` and treats ``ActionRateLimitedError`` as "use the last value". + +Empty on purpose until the first adapter lands (PR 4-C). +""" + +from __future__ import annotations diff --git a/suncore/app/control.py b/suncore/app/control.py index d4e2421..8ab3a2d 100644 --- a/suncore/app/control.py +++ b/suncore/app/control.py @@ -49,7 +49,7 @@ DeviationEvent, ) from suncore.controller.loop import TickResult -from suncore.core.balance import compute_flows +from suncore.core.balance import FlowSnapshot, compute_flows from suncore.core.config import GoalsConfig from suncore.core.control import ( TEMPERATURE_ROLE, @@ -62,6 +62,14 @@ from suncore.core.decisions import Action, Decision, Reason from suncore.core.devices import DeviceConfig, control_roles from suncore.core.identity import identity_token +from suncore.core.measurements import ( + SITE_BATTERY_W, + SITE_GRID_W, + SITE_HOUSE_W, + SITE_PV_W, + SITE_SOC_PCT, + device_power_key, +) from suncore.core.signals import Quality, Sample from suncore.core.site import SiteConfig from suncore.devices.drivers import ClimateDriver, NumberDriver, create_driver @@ -101,6 +109,8 @@ """What ``create_driver`` returns, structurally: the gate's actuator plus ``device_state``.""" StateChangeListener = Callable[[], None] +MeasurementSink = Callable[[datetime, Mapping[str, float | None]], None] +"""Receives ``(tick time, {measurement key: value or None})`` after every tick.""" def probe_temperature_c(cache: EntityCache, device: DeviceConfig) -> float | None: @@ -169,6 +179,10 @@ class ControlStack: (``gate.recover(after_handover=True)``). predecessor: the gate of the replaced stack: devices it could not release (removed or re-mapped while Home Assistant was down) are released with its actuators. + on_measurements: called after every tick with the tick time and the site and device + measurements of that tick (``suncore.core.measurements`` keys; ``None`` = no valid + value). Synchronous and cheap by contract; an exception is logged and swallowed - + recording can never disturb the control loop. defer_recover: wait for :meth:`allow_recover` before ``gate.recover()`` (the runtime first learns Home Assistant's time zone, whose change would rebuild the stack in the middle of the recovery). @@ -199,8 +213,10 @@ def __init__( # noqa: PLR0915 - the composition root of the control layer after_handover: bool = False, predecessor: SafetyGate | None = None, defer_recover: bool = False, + on_measurements: MeasurementSink | None = None, ) -> None: self._connection = connection + self._on_measurements = on_measurements self._recover_allowed = not defer_recover self._after_handover = after_handover self._site = site @@ -687,11 +703,40 @@ async def tick(self) -> TickResult | None: ) self.estimates = estimates self.states = states + self._record_measurements(now, flows, estimates) self.last = await self.loop.tick(now, inputs) self._maybe_save_models(now) self._notify() return self.last + def _record_measurements( + self, + now: datetime, + flows: FlowSnapshot | None, + estimates: Mapping[str, PowerEstimate], + ) -> None: + """Hand this tick's values to the recorder; only ``OK`` data counts (a gap otherwise).""" + if self._on_measurements is None: + return + values: dict[str, float | None] = {} + if flows is not None: + for key, flow in ( + (SITE_GRID_W, flows.grid_w), + (SITE_PV_W, flows.pv_w), + (SITE_BATTERY_W, flows.battery_w), + (SITE_SOC_PCT, flows.battery_soc_pct), + (SITE_HOUSE_W, flows.house_w), + ): + values[key] = flow.value if flow.quality is Quality.OK else None + for device_id, estimate in estimates.items(): + values[device_power_key(device_id)] = ( + estimate.value_w if estimate.quality is Quality.OK else None + ) + try: + self._on_measurements(now, values) + except Exception: + log.exception("control.measurement_sink_failed") + async def drain(self) -> None: """Wait for the submissions and observations in flight (they wait on simulated time).""" await self.loop.drain() diff --git a/suncore/app/history.py b/suncore/app/history.py new file mode 100644 index 0000000..1773019 --- /dev/null +++ b/suncore/app/history.py @@ -0,0 +1,172 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Rebuild of the 15-minute site history from Home Assistant's statistics (Phase 4 WP3). + +Runs once after the first successful connect when ``measurement_15m`` is empty (a restore +without ``timeseries.db``, a fresh install). The configured site entities are asked for their +hourly statistics (``suncore.ha.statistics``); the answers are combined the way the live +balance does (``suncore.core.balance.compute_flows``): grid as ``import - export`` for a pair, +PV as the sum of the arrays, the house from its sensor or from ``pv + grid - battery``. The +rows are stored as *derived* quarter hours. + +Controlled devices are not rebuilt: their power comes from their own power models, which have +no statistic to read. ``UNVERIFIED``: whether the Supervisor proxy forwards ``recorder/*``; +an error is reported to the caller, which logs it and carries on without history. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta + +import structlog + +from suncore.core.measurements import ( + SITE_BATTERY_W, + SITE_GRID_W, + SITE_HOUSE_W, + SITE_PV_W, + SITE_SOC_PCT, +) +from suncore.core.series import Grid, SeriesUnit, TimeSeries, resample_power +from suncore.core.site import SiteConfig +from suncore.core.units import house_load_w +from suncore.ha.connection import HAConnection +from suncore.ha.statistics import StatisticSpec, fetch_quarter_history +from suncore.storage.timeseries import TimeseriesStore + +log = structlog.get_logger(__name__) + +REBUILD_DAYS = 28 +"""History that is rebuilt (the load forecast needs 14 days, the risk model starts at 30).""" + +_GRID, _GRID_IMPORT, _GRID_EXPORT = "grid", "grid_import", "grid_export" +_BATTERY, _SOC, _HOUSE = "battery", "soc", "house" + + +@dataclass(frozen=True, slots=True) +class RebuildResult: + """What a rebuild stored: quarter hours per measurement key, and the statistic ids + Home Assistant had no data for.""" + + stored: Mapping[str, int] + missing: tuple[str, ...] + + @property + def total(self) -> int: + return sum(self.stored.values()) + + +def site_statistic_specs(site: SiteConfig) -> list[StatisticSpec]: + """The statistics to ask for: every mapped power entity and the state of charge.""" + specs: list[StatisticSpec] = [] + grid = site.grid + if grid.power is not None: + specs.append(StatisticSpec(_GRID, grid.power.entity_id, invert=grid.power.invert_sign)) + else: + assert grid.import_power is not None + assert grid.export_power is not None + specs.append( + StatisticSpec( + _GRID_IMPORT, grid.import_power.entity_id, invert=grid.import_power.invert_sign + ) + ) + specs.append( + StatisticSpec( + _GRID_EXPORT, grid.export_power.entity_id, invert=grid.export_power.invert_sign + ) + ) + specs.extend( + StatisticSpec(f"pv{i}", ref.entity_id, invert=ref.invert_sign) + for i, ref in enumerate(site.pv_power_refs) + ) + if site.battery is not None: + power = site.battery.power + specs.append(StatisticSpec(_BATTERY, power.entity_id, invert=power.invert_sign)) + specs.append(StatisticSpec(_SOC, site.battery.soc.entity_id, unit=SeriesUnit.PERCENT)) + if site.house is not None: + specs.append(StatisticSpec(_HOUSE, site.house.entity_id, invert=site.house.invert_sign)) + return specs + + +def _common_grid(series: Sequence[TimeSeries]) -> Grid: + start = min(s.start for s in series) + end = max(s.end for s in series) + step = series[0].step + return Grid(start=start, step=step, count=int((end - start) / step)) + + +def _combine(parts: Sequence[TimeSeries | None], fn: Callable[..., float]) -> TimeSeries | None: + """Slot-wise ``fn`` over series on one grid; ``None`` if a part is missing in the slot.""" + present = [p for p in parts if p is not None] + if len(present) != len(parts) or not present: + return None + grid = _common_grid(present) + aligned = [resample_power(p, grid) for p in present] + values = tuple( + None if any(v is None for v in slot) else fn(*slot) + for slot in zip(*(a.values for a in aligned), strict=True) + ) + return TimeSeries(start=grid.start, step=grid.step, values=values, unit=aligned[0].unit) + + +def assemble_site_series( + site: SiteConfig, fetched: Mapping[str, TimeSeries] +) -> dict[str, TimeSeries]: + """Measurement key -> 15-minute series, combined like the live power balance.""" + get = fetched.get + if site.grid.power is not None: + grid = get(_GRID) + else: + grid = _combine([get(_GRID_IMPORT), get(_GRID_EXPORT)], lambda i, e: i - e) + pv_parts = [get(f"pv{i}") for i in range(len(site.pv_power_refs))] + pv = _combine(pv_parts, lambda *v: sum(v)) if pv_parts else None + if site.battery is None: + battery = soc = None + else: + battery, soc = get(_BATTERY), get(_SOC) + if site.house is not None: + house = get(_HOUSE) + else: + # pv + grid - battery; a site without PV or battery contributes 0 W for it. + has_pv, has_battery = bool(pv_parts), site.battery is not None + parts = [grid, *([pv] if has_pv else []), *([battery] if has_battery else [])] + + def balance(*v: float) -> float: + it = iter(v) + g = next(it) + return house_load_w(next(it) if has_pv else 0.0, g, next(it) if has_battery else 0.0) + + house = _combine(parts, balance) + named = { + SITE_GRID_W: grid, + SITE_PV_W: pv, + SITE_BATTERY_W: battery, + SITE_SOC_PCT: soc, + SITE_HOUSE_W: house, + } + return {key: series for key, series in named.items() if series is not None} + + +async def rebuild_history( + connection: HAConnection, + store: TimeseriesStore, + site: SiteConfig, + now: datetime, + *, + days: int = REBUILD_DAYS, +) -> RebuildResult: + """Fetch the site's hourly statistics and store them as derived quarter hours. + + Home Assistant errors propagate. Existing measured rows are never overwritten. + """ + end = now.astimezone(UTC).replace(minute=0, second=0, microsecond=0) + history = await fetch_quarter_history( + connection, site_statistic_specs(site), end - timedelta(days=days), end + ) + stored: dict[str, int] = {} + for key, series in assemble_site_series(site, history.series).items(): + stored[key] = await store.store_quarter_hours(key, series, derived=True) + result = RebuildResult(stored=stored, missing=history.missing) + log.info("history.rebuilt", quarter_hours=result.total, missing=list(result.missing)) + return result diff --git a/suncore/app/main.py b/suncore/app/main.py index 349641c..f6d1e57 100644 --- a/suncore/app/main.py +++ b/suncore/app/main.py @@ -24,6 +24,7 @@ from suncore.api import create_app from suncore.app.demo import DemoClock, demo_tick_s from suncore.app.logconfig import configure_logging +from suncore.app.recorder import MeasurementRecorder from suncore.app.runtime import DB_FILENAME, Runtime, utc_now from suncore.app.settings import ENV_SUPERVISOR_TOKEN, Settings, load_settings, under_supervisor from suncore.core.control import ControlSettings @@ -40,9 +41,14 @@ ConfigStore, Database, DecisionLog, + LearnedParamStore, + PlanSnapshotStore, PowerModelStateStore, SqliteRuntimeStore, + TimeseriesStore, + open_timeseries_or_recover, ) +from suncore.storage.timeseries import TS_FILENAME from suncore.templates import load_templates log = structlog.get_logger(__name__) @@ -120,11 +126,13 @@ async def build_runtime( data_dir = settings.data_dir data_dir.mkdir(parents=True, exist_ok=True) db = await Database.open(data_dir / DB_FILENAME) + timeseries_db = await open_timeseries_or_recover(data_dir / TS_FILENAME) try: config = ConfigStore(db) if connection is None: connection = WebSocketConnection(transport_from_env(env)) decision_log = DecisionLog(db) + timeseries = None if timeseries_db is None else TimeseriesStore(timeseries_db) cache = EntityCache(clock=clock) catalogue = EntityCatalogue(connection, wall_clock=clock) user_templates = None if demo else data_dir / "templates" @@ -150,6 +158,11 @@ async def build_runtime( decision_log=decision_log, runtime_store=SqliteRuntimeStore(db, clock=clock), power_store=PowerModelStateStore(db, clock=clock), + plans=PlanSnapshotStore(db, clock=clock), + learned=LearnedParamStore(db, clock=clock), + timeseries_db=timeseries_db, + timeseries=timeseries, + recorder=MeasurementRecorder(timeseries), connection=connection, cache=cache, catalogue=catalogue, @@ -166,6 +179,8 @@ async def build_runtime( runtime._tempdir = tempdir await runtime.prepare() except BaseException: + if timeseries_db is not None: + await timeseries_db.close() await db.close() if tempdir is not None: tempdir.cleanup() diff --git a/suncore/app/recorder.py b/suncore/app/recorder.py new file mode 100644 index 0000000..fe8258f --- /dev/null +++ b/suncore/app/recorder.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Batched writer of the controller's site measurements into ``timeseries.db`` (Phase 4 WP3). + +The controller calls :meth:`MeasurementRecorder.record` once per tick; it only appends to an +in-memory list. :meth:`flush` (every 60 s from the runtime, and once at shutdown) writes the +list in one transaction. A database error is logged and the batch is kept for the next flush +(bounded: the oldest rows are dropped first), so a broken disk can neither crash nor slow the +control loop and memory stays bounded. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Mapping +from datetime import datetime +from typing import Final + +import structlog + +from suncore.storage.timeseries import MeasurementRow, TimeseriesStore + +log = structlog.get_logger(__name__) + +FLUSH_INTERVAL_S: Final = 60.0 +MAX_BUFFERED_ROWS: Final = 50_000 +"""About 100 minutes of a 5 s tick with 10 keys: the cap for a database that stays broken.""" + + +class MeasurementRecorder: + """Collects ``(key, ts, value)`` rows and writes them in batches.""" + + def __init__( + self, store: TimeseriesStore | None, *, max_buffered: int = MAX_BUFFERED_ROWS + ) -> None: + self._store = store + self._max = max_buffered + self._buffer: list[MeasurementRow] = [] + self._lock = asyncio.Lock() + self.dropped = 0 + """Rows discarded because the buffer was full (a database that stays unavailable).""" + + @property + def enabled(self) -> bool: + return self._store is not None + + @property + def pending(self) -> int: + return len(self._buffer) + + def record(self, ts: datetime, values: Mapping[str, float | None]) -> None: + """Queue one tick's values; a ``None`` value is a gap and is not stored.""" + if self._store is None: + return + self._buffer.extend((key, ts, value) for key, value in values.items() if value is not None) + overflow = len(self._buffer) - self._max + if overflow > 0: + del self._buffer[:overflow] + self.dropped += overflow + + async def flush(self) -> int: + """Write the queued rows; returns how many were written (0 on error or nothing).""" + if self._store is None: + return 0 + async with self._lock: + batch, self._buffer = self._buffer, [] + if not batch: + return 0 + try: + return await self._store.append_measurements(batch) + except Exception: + log.exception("timeseries.flush_failed", rows=len(batch)) + combined = batch + self._buffer + self.dropped += max(0, len(combined) - self._max) + self._buffer = combined[-self._max :] + return 0 diff --git a/suncore/app/runtime.py b/suncore/app/runtime.py index 8859f9b..cd76160 100644 --- a/suncore/app/runtime.py +++ b/suncore/app/runtime.py @@ -8,7 +8,7 @@ import tempfile from collections.abc import Awaitable, Callable, Collection, Iterable, Mapping, Sequence from dataclasses import dataclass, field -from datetime import UTC, datetime, timedelta, tzinfo +from datetime import UTC, datetime, time, timedelta, tzinfo from pathlib import Path from typing import Any from zoneinfo import ZoneInfo, ZoneInfoNotFoundError @@ -28,7 +28,9 @@ ) from suncore.app.cadence import device_cadences, merge_cadences from suncore.app.control import ControlStack +from suncore.app.history import rebuild_history from suncore.app.modes import ConfigModes +from suncore.app.recorder import FLUSH_INTERVAL_S, MeasurementRecorder from suncore.app.settings import SUPERVISOR_INGRESS_PEER, Settings from suncore.core.balance import FlowSnapshot, compute_flows from suncore.core.config import DevicesConfig, GoalsConfig, OperationConfig, RegionConfig @@ -71,8 +73,11 @@ Database, DecisionLog, HistoryEntry, + LearnedParamStore, + PlanSnapshotStore, PowerModelStateStore, SqliteRuntimeStore, + TimeseriesStore, ) from suncore.templates import TemplateCatalog @@ -82,7 +87,9 @@ DB_FILENAME = "suncore.db" PRUNE_INTERVAL_S = 3600.0 -"""Decision-log retention runs hourly.""" +"""Decision-log and plan-snapshot retention run hourly.""" +MAINTENANCE_TIME = time(3, 30) +"""Local time of the nightly time-series job (downsample, retention, vacuum if worth it).""" def utc_now() -> datetime: @@ -116,6 +123,15 @@ class Runtime: devices: DevicesConfig | None = None region: RegionConfig | None = None goals: GoalsConfig | None = None + timeseries_db: Database | None = None + """``timeseries.db``; ``None`` = unavailable (too new, unopenable): control is unaffected.""" + timeseries: TimeseriesStore | None = None + plans: PlanSnapshotStore | None = None + learned: LearnedParamStore | None = None + recorder: MeasurementRecorder = field(default_factory=lambda: MeasurementRecorder(None)) + """Batched writer of the controller's measurements (disabled without ``timeseries``).""" + flush_interval_s: float = FLUSH_INTERVAL_S + maintenance_time: time = MAINTENANCE_TIME sleep: Callable[[float], Awaitable[None]] = asyncio.sleep prune_interval_s: float = PRUNE_INTERVAL_S real_clock: Clock = utc_now @@ -136,6 +152,11 @@ class Runtime: _unsubscribers: list[Callable[[], None]] = field(default_factory=list) _tempdir: tempfile.TemporaryDirectory[str] | None = None _prune_task: asyncio.Task[None] | None = None + _flush_task: asyncio.Task[None] | None = None + _maintenance_task: asyncio.Task[None] | None = None + _history_task: asyncio.Task[None] | None = None + _history_done: bool = False + """The statistics rebuild ran (or was not needed) in this process.""" _stack_lock: asyncio.Lock = field(default_factory=asyncio.Lock) _operation_lock: asyncio.Lock = field(default_factory=asyncio.Lock) _started: bool = False @@ -182,6 +203,11 @@ async def start(self) -> None: self.stack.start_ticking() self._connect_task = asyncio.create_task(self._connect(), name="suncore-ha-connect") self._prune_task = asyncio.create_task(self._prune_loop(), name="suncore-prune") + if self.timeseries is not None: + self._flush_task = asyncio.create_task(self._flush_loop(), name="suncore-ts-flush") + self._maintenance_task = asyncio.create_task( + self._maintenance_loop(), name="suncore-ts-maintenance" + ) async def _load_live_sections(self) -> None: self.site = await self.config.get_site() @@ -237,6 +263,7 @@ def _new_stack( reserve_soc_pct=None if battery is None else battery.min_soc_pct, cadences=merge_cadences(device_cadences(devices), self.extra_cadences), sink=self._record_decision, + on_measurements=self.recorder.record, cache=self.cache, power_store=self.power_store, attach=False, @@ -271,6 +298,7 @@ async def _rebuild_stack(self) -> None: await new.start() new.start_ticking() self._notify_control() + self._start_history_rebuild() # a site configured after the first connect def _unchanged_devices(self, previous: Mapping[str, DeviceConfig]) -> frozenset[str]: """Ids of the devices that are the same device, controlled the same way, after a change.""" @@ -294,11 +322,90 @@ async def prune_decisions(self) -> int: log.exception("decision_log.prune_failed") return 0 + async def prune_plans(self) -> int: + """One retention pass over the plan snapshots; failures are logged, never raised.""" + if self.plans is None: + return 0 + try: + return await self.plans.prune(self.clock()) + except Exception: + log.exception("plan_snapshot.prune_failed") + return 0 + async def _prune_loop(self) -> None: while True: await self.prune_decisions() + await self.prune_plans() await self.sleep(self.prune_interval_s) + # --- time series (timeseries.db) --------------------------------------------- + + async def _flush_loop(self) -> None: + while True: + await self.sleep(self.flush_interval_s) + await self.recorder.flush() # logs its own errors + + async def maintain_timeseries(self) -> bool: + """The nightly job once: flush, downsample, retention, vacuum if worth it. + + Failures are logged, never raised; returns whether the job ran through. + """ + if self.timeseries is None: + return False + try: + await self.recorder.flush() # the newest samples are aggregated too + await self.timeseries.maintain(self.clock()) + except Exception: + log.exception("timeseries.maintenance_failed") + return False + return True + + def seconds_until_maintenance(self) -> float: + """Seconds to the next ``maintenance_time`` in the site's time zone (DST safe).""" + tz = self._timezone() + now = self.clock().astimezone(tz) + target = datetime.combine(now.date(), self.maintenance_time, tzinfo=tz) + if target.astimezone(UTC) <= now.astimezone(UTC): + target = datetime.combine( + now.date() + timedelta(days=1), self.maintenance_time, tzinfo=tz + ) + return (target.astimezone(UTC) - now.astimezone(UTC)).total_seconds() + + async def _maintenance_loop(self) -> None: + while True: + await self.maintain_timeseries() # at start too: a night the app slept through + await self.sleep(self.seconds_until_maintenance()) + + def _start_history_rebuild(self) -> None: + """Rebuild the 15-minute history from HA's statistics, once, in the background.""" + if ( + self._history_done + or not self._started + or self.timeseries is None + or self.site is None + or self.connection.state is not ConnectionState.CONNECTED + or (self._history_task is not None and not self._history_task.done()) + ): + return + self._history_task = asyncio.create_task(self._rebuild_history(), name="suncore-history") + + async def _rebuild_history(self) -> None: + store, site = self.timeseries, self.site + if store is None or site is None: + return + try: + if not await store.has_quarter_history(): + await rebuild_history(self.connection, store, site, self.clock()) + except asyncio.CancelledError: + raise + except (HAError, TimeoutError) as exc: + # UNVERIFIED: the Supervisor proxy may not forward recorder/*. Not retried before + # the next start; the history then grows from the controller's own measurements. + log.warning("history.rebuild_failed", reason=type(exc).__name__, error=str(exc)) + except Exception: + log.exception("history.rebuild_crashed") + self._history_done = True + async def _connect(self) -> None: try: await self.connection.connect() @@ -308,6 +415,7 @@ async def _connect(self) -> None: log.error("ha.connect_failed", error_type=type(exc).__name__, error=str(exc)) return await self._prepare_control() + self._start_history_rebuild() def _time_zone_settled(self) -> bool: return self.region is not None or self._tz_learnt @@ -354,17 +462,28 @@ async def stop(self) -> None: for unsubscribe in self._unsubscribers: unsubscribe() self._unsubscribers.clear() - for task in (self._connect_task, self._prune_task, self._tz_task): + for task in ( + self._connect_task, + self._prune_task, + self._tz_task, + self._flush_task, + self._maintenance_task, + self._history_task, + ): if task is not None: task.cancel() with contextlib.suppress(asyncio.CancelledError, Exception): await task self._connect_task = self._prune_task = self._tz_task = None + self._flush_task = self._maintenance_task = self._history_task = None async with self._stack_lock: if self.stack is not None: await self.stack.shutdown() await self.catalogue.stop() await self.connection.close() + await self.recorder.flush() + if self.timeseries_db is not None: + await self.timeseries_db.close() await self.db.close() if self._tempdir is not None: self._tempdir.cleanup() diff --git a/suncore/core/measurements.py b/suncore/core/measurements.py new file mode 100644 index 0000000..42551dd --- /dev/null +++ b/suncore/core/measurements.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Keys of the recorded site measurements (Phase 4 WP3). + +The time-series store (``timeseries.db``) and its readers (load forecast, accuracy +statistics, the rebuild from Home Assistant's statistics) agree on these keys. Units and +signs are the core conventions: W for power, % for the state of charge; grid + = import, +battery + = charging, device + = consumption, PV + = generation. +""" + +from __future__ import annotations + +from typing import Final + +SITE_GRID_W: Final = "site.grid_w" +SITE_PV_W: Final = "site.pv_w" +SITE_BATTERY_W: Final = "site.battery_w" +SITE_SOC_PCT: Final = "site.soc_pct" +SITE_HOUSE_W: Final = "site.house_w" + +SITE_KEYS: Final[tuple[str, ...]] = ( + SITE_GRID_W, + SITE_PV_W, + SITE_BATTERY_W, + SITE_SOC_PCT, + SITE_HOUSE_W, +) + +_DEVICE_PREFIX: Final = "device." +_DEVICE_SUFFIX: Final = ".power_w" + + +def device_power_key(device_id: str) -> str: + """Key of a controlled device's estimated power (W).""" + return f"{_DEVICE_PREFIX}{device_id}{_DEVICE_SUFFIX}" + + +def is_percent_key(key: str) -> bool: + """Whether the measurement is a state in percent (not a power in W).""" + return key == SITE_SOC_PCT + + +__all__ = [ + "SITE_BATTERY_W", + "SITE_GRID_W", + "SITE_HOUSE_W", + "SITE_KEYS", + "SITE_PV_W", + "SITE_SOC_PCT", + "device_power_key", + "is_percent_key", +] diff --git a/suncore/ha/__init__.py b/suncore/ha/__init__.py index 824ac0e..bdc6b0e 100644 --- a/suncore/ha/__init__.py +++ b/suncore/ha/__init__.py @@ -9,6 +9,8 @@ from suncore.ha.connection import ( REGISTRY_EVENT_TYPES, ActionNotAllowedError, + ActionRateLimitedError, + ActionRateLimiter, AuthError, CommandError, CommandTimeoutError, @@ -64,6 +66,8 @@ "REGISTRY_EVENT_TYPES", "SITE_ROLES", "ActionNotAllowedError", + "ActionRateLimitedError", + "ActionRateLimiter", "AuthError", "Backoff", "CachedEntity", diff --git a/suncore/ha/connection.py b/suncore/ha/connection.py index c43efe3..0f5aea8 100644 --- a/suncore/ha/connection.py +++ b/suncore/ha/connection.py @@ -25,6 +25,7 @@ import time from collections.abc import Awaitable, Callable, Collection, Mapping, Sequence from dataclasses import dataclass, field +from datetime import datetime from enum import StrEnum from typing import Any, Final, Protocol, runtime_checkable @@ -72,6 +73,25 @@ class ActionNotAllowedError(HAError): """``call_action_response`` was asked for an action that is not on the read-only allowlist.""" +class ActionRateLimitedError(HAError): + """A read-only action was asked for again before its minimum interval elapsed. + + Nothing was sent to Home Assistant. ``retry_after_s`` is how long until the call would be + admitted. The user-visible text is the catalogue entry :data:`ERR_ACTION_RATE_LIMITED`. + """ + + code = "rate_limited" + + def __init__(self, action: str, retry_after_s: float) -> None: + super().__init__(f"{action} is rate limited, retry in {retry_after_s:.1f} s") + self.action = action + self.retry_after_s = retry_after_s + + +ERR_ACTION_RATE_LIMITED: Final = "ha.error.action_rate_limited" +"""Catalogue key of the refusal of a too frequent read-only action.""" + + # --- value types -------------------------------------------------------------- @@ -226,11 +246,41 @@ async def call_action_response( data: Mapping[str, Any] | None = None, *, target: Mapping[str, Any] | None = None, + min_interval_s: float | None = None, + max_wait_s: float = 0.0, ) -> dict[str, Any]: """Call a read-only action with ``return_response`` and return its response. Raises :class:`ActionNotAllowedError` unless ``domain.service`` is in :data:`suncore.core.actions.READ_ONLY_ACTIONS`. This is *not* a write path. + + Each ``(domain, service)`` is rate limited (:class:`ActionRateLimiter`): at most one + call per ``min_interval_s`` (default :data:`DEFAULT_ACTION_INTERVAL_S`; a caller passes + the interval of its configured polling). A call that is too early waits at most + ``max_wait_s`` (default: not at all) and otherwise raises + :class:`ActionRateLimitedError`; it never blocks indefinitely. + """ + ... + + async def statistics_during_period( + self, + statistic_ids: Sequence[str], + start: datetime, + end: datetime | None = None, + *, + period: str = "hour", + types: Sequence[str] | None = None, + units: Mapping[str, str] | None = None, + ) -> Mapping[str, Sequence[Mapping[str, Any]]]: + """``recorder/statistics_during_period`` (read only, not admin gated). + + ``start``/``end`` are sent as ISO 8601 with their UTC offset. The result maps each + statistic id *that has data* to its rows (``start``/``end`` in epoch milliseconds, + the requested ``types``); ids without data are absent. The response carries no unit: + pass ``units`` (e.g. ``{"power": "W", "energy": "Wh"}``) so the numbers do not change + when the user switches a sensor's display unit. Verified shapes: + ``docs/research/ha-websocket-api.md`` "Long-term statistics". + ``UNVERIFIED``: that the Supervisor proxy forwards ``recorder/*``. """ ... @@ -278,6 +328,73 @@ async def acquire(self) -> None: self._last = self._clock() +DEFAULT_ACTION_INTERVAL_S: Final = 30.0 +"""Default minimum spacing of two calls of the same read-only action (follow-up A3).""" + + +class ActionRateLimiter: + """Minimum spacing per key (``domain.service``) with try-acquire and a wait deadline. + + Slots are handed out in order: a caller that waits reserves the slot after the previous + one, so two concurrent callers do not both pass. Clock and sleep are injected. A key that + is too busy is refused instead of queued (:meth:`acquire` returns ``False``), so a slow + integration can never stack up calls. + """ + + def __init__( + self, + default_interval_s: float = DEFAULT_ACTION_INTERVAL_S, + *, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + ) -> None: + if default_interval_s < 0: + msg = "default_interval_s must be >= 0" + raise ValueError(msg) + self._default = default_interval_s + self._clock = clock + self._sleep = sleep + self._next_ok: dict[str, float] = {} + + def retry_after(self, key: str) -> float: + """Seconds until ``key`` would be admitted (0 = now).""" + return max(0.0, self._next_ok.get(key, 0.0) - self._clock()) + + def try_acquire(self, key: str, interval_s: float | None = None) -> bool: + """Take the slot if it is free now; otherwise change nothing and return ``False``.""" + interval = self._interval(interval_s) + now = self._clock() + if self._next_ok.get(key, 0.0) > now: + return False + self._next_ok[key] = now + interval + return True + + async def acquire( + self, key: str, interval_s: float | None = None, *, max_wait_s: float = 0.0 + ) -> bool: + """Take the next slot of ``key`` if it comes within ``max_wait_s`` (waiting for it). + + Returns ``False`` without waiting and without reserving when the slot is further away. + """ + interval = self._interval(interval_s) + now = self._clock() + slot = max(now, self._next_ok.get(key, 0.0)) + wait = slot - now + if wait > max(0.0, max_wait_s): + return False + self._next_ok[key] = slot + interval # reserved before sleeping: no await in between + if wait > 0: + await self._sleep(wait) + return True + + def _interval(self, interval_s: float | None) -> float: + interval = self._default if interval_s is None else interval_s + if interval < 0: + msg = "interval_s must be >= 0" + raise ValueError(msg) + return interval + + @dataclass(frozen=True, slots=True) class ServiceResult: """Outcome of a service call: HA's response, the state read back afterwards and the diff --git a/suncore/ha/statistics.py b/suncore/ha/statistics.py new file mode 100644 index 0000000..44dc5e0 --- /dev/null +++ b/suncore/ha/statistics.py @@ -0,0 +1,251 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Quarter-hour history from Home Assistant's long-term statistics (Phase 4 WP3). + +When ``timeseries.db`` is missing or empty (a restore without it, a fresh install) the +15-minute history is rebuilt from ``recorder/statistics_during_period``. The finest period +that outlives the recorder's purge is ``hour`` (``5minute`` is purged after ``purge_keep_days``, +default 10; verified in ``docs/research/ha-websocket-api.md`` "Long-term statistics"), so the +rebuilt history is a **step-hold of the hourly values**: + +* a power or state sensor (``mean``): each of the four quarter hours of an hour gets the hour's + mean, which preserves the energy of the hour (4 x W x 0.25 h = W x 1 h); +* an energy counter (``change`` = the hour's delta of ``sum``, in Wh): each quarter hour gets + ``change / 1 h`` as mean power, which preserves the energy as well. + +These values are *derived*: the intra-hour shape is lost. They are stored with +``derived = 1`` and are replaced as soon as the controller measured the same slot itself. + +Request details (all verified): ``start_time``/``end_time`` are ISO 8601 **with** a UTC offset +(a value without one is read in HA's time zone), the response has ``start``/``end`` in integer +epoch **milliseconds** and no unit, so ``units`` is always sent. A statistic id without data is +absent from the result; a value can be ``null``. ``UNVERIFIED``: that the Supervisor proxy +forwards ``recorder/*`` (first run on Home Assistant OS); an error is the caller's to log. +""" + +from __future__ import annotations + +import math +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any, Final, Literal + +import structlog + +from suncore.core.series import MAX_POINTS, PLANNING_STEP, SeriesUnit, TimeSeries +from suncore.ha.connection import HAConnection + +log = structlog.get_logger(__name__) + +STATISTICS_UNITS: Final[Mapping[str, str]] = {"power": "W", "energy": "Wh"} +"""Target units, always sent so a display-unit change in HA does not change the numbers.""" + +DEFAULT_CHUNK_DAYS: Final = 7 +DEFAULT_BATCH_SIZE: Final = 20 +HOUR_MS: Final = 3_600_000 +QUARTER_MS: Final = 900_000 + +StatKind = Literal["mean", "energy"] +"""``mean``: power or state sensor (column ``mean``); ``energy``: counter (``change``/``sum``).""" + +_TYPES: Final[Mapping[str, tuple[str, ...]]] = { + "mean": ("mean",), + "energy": ("change", "sum"), +} + + +@dataclass(frozen=True, slots=True) +class StatisticSpec: + """One statistic to fetch: ``key`` names it in the result, ``statistic_id`` is the + sensor's entity id. ``invert`` flips the sign (a source with opposite polarity).""" + + key: str + statistic_id: str + kind: StatKind = "mean" + unit: SeriesUnit = SeriesUnit.W + invert: bool = False + + +@dataclass(frozen=True, slots=True) +class StatisticsHistory: + """Result of a fetch: 15-minute series per spec key and the ids Home Assistant had no + data for in the whole range.""" + + series: Mapping[str, TimeSeries] + missing: tuple[str, ...] + + +def _hour_start(row: Mapping[str, Any]) -> int | None: + start = row.get("start") + if isinstance(start, bool) or not isinstance(start, int | float): + return None + start_ms = int(start) + return start_ms if start_ms % QUARTER_MS == 0 else None + + +def _number(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float): + return None + number = float(value) + return number if math.isfinite(number) else None + + +def hourly_values(rows: Iterable[Mapping[str, Any]], kind: StatKind) -> dict[int, float]: + """``{hour start (epoch ms): value}`` of the usable rows. + + ``mean`` rows give the mean as is; ``energy`` rows give the hour's energy in Wh from + ``change``, else from the difference of ``sum`` to the previous row *if that row is the + previous hour*. A row with a missing value, a start off the quarter-hour grid or a + negative energy (a counter reset HA did not compensate) is skipped - a gap, never a zero. + """ + values: dict[int, float] = {} + previous_sum: tuple[int, float] | None = None + for row in sorted(rows, key=lambda r: _hour_start(r) or 0): + start = _hour_start(row) + if start is None: + continue + end = row.get("end") + if ( + isinstance(end, int | float) + and not isinstance(end, bool) + and int(end) - start != HOUR_MS + ): + continue # not an hour row: the request asked for period "hour" + if kind == "mean": + mean = _number(row.get("mean")) + if mean is not None: + values[start] = mean + continue + total = _number(row.get("sum")) + change = _number(row.get("change")) + if ( + change is None + and total is not None + and previous_sum is not None + and previous_sum[0] + HOUR_MS == start + ): + change = total - previous_sum[1] + if total is not None: + previous_sum = (start, total) + if change is not None and change >= 0: + values[start] = change # Wh in the hour == mean W over one hour + return values + + +def quarter_hours( + hourly: Mapping[int, float], *, unit: SeriesUnit, invert: bool = False +) -> TimeSeries | None: + """Step-hold the hourly values onto a 15-minute grid (``None`` where an hour is missing). + + The grid runs from the first to the last known hour. ``None`` if there is no value. + """ + if not hourly: + return None + first = min(hourly) + last = max(hourly) + HOUR_MS + count = (last - first) // QUARTER_MS + if count > MAX_POINTS: + msg = f"statistics range too long: {count} points exceed {MAX_POINTS}" + raise ValueError(msg) + sign = -1.0 if invert else 1.0 + slots: list[float | None] = [None] * count + for start_ms, value in hourly.items(): + offset = (start_ms - first) // QUARTER_MS + for i in range(HOUR_MS // QUARTER_MS): + slots[offset + i] = sign * value + 0.0 # + 0.0: no negative zero + return TimeSeries( + start=datetime.fromtimestamp(first / 1000, UTC), + step=PLANNING_STEP, + values=tuple(slots), + unit=unit, + ) + + +def _hour_floor(moment: datetime) -> datetime: + utc = moment.astimezone(UTC) + return utc.replace(minute=0, second=0, microsecond=0) + + +def _hour_ceil(moment: datetime) -> datetime: + floor = _hour_floor(moment) + return floor if floor == moment.astimezone(UTC) else floor + timedelta(hours=1) + + +def _windows(start: datetime, end: datetime, chunk: timedelta) -> list[tuple[datetime, datetime]]: + out: list[tuple[datetime, datetime]] = [] + cursor = start + while cursor < end: + out.append((cursor, min(cursor + chunk, end))) + cursor += chunk + return out + + +async def fetch_quarter_history( + connection: HAConnection, + specs: Sequence[StatisticSpec], + start: datetime, + end: datetime, + *, + chunk_days: int = DEFAULT_CHUNK_DAYS, + batch_size: int = DEFAULT_BATCH_SIZE, +) -> StatisticsHistory: + """Fetch hourly statistics in chunks and return them as 15-minute series per spec. + + Requests are split by time (``chunk_days``) and by statistic id (``batch_size``) so one + answer stays small, and by kind because the requested ``types`` differ. Errors of + Home Assistant (:class:`~suncore.ha.connection.HAError`) propagate: a rebuild is + all-or-nothing and the caller decides whether to retry. + """ + if not start < end: + msg = "statistics range: start must be before end" + raise ValueError(msg) + if chunk_days < 1 or batch_size < 1: + msg = "chunk_days and batch_size must be positive" + raise ValueError(msg) + start, end = _hour_floor(start), _hour_ceil(end) + rows: dict[str, list[Mapping[str, Any]]] = {} + by_id = {s.statistic_id: s for s in specs} + for kind in ("mean", "energy"): + ids = sorted({s.statistic_id for s in specs if s.kind == kind}) + for begin, stop in _windows(start, end, timedelta(days=chunk_days)): + for i in range(0, len(ids), batch_size): + batch = ids[i : i + batch_size] + result = await connection.statistics_during_period( + batch, + begin, + stop, + period="hour", + types=_TYPES[kind], + units=STATISTICS_UNITS, + ) + for statistic_id, statistic_rows in result.items(): + if statistic_id in by_id and statistic_rows: + rows.setdefault(statistic_id, []).extend(statistic_rows) + series: dict[str, TimeSeries] = {} + missing: list[str] = [] + for spec in specs: + built = quarter_hours( + hourly_values(rows.get(spec.statistic_id, ()), spec.kind), + unit=spec.unit, + invert=spec.invert, + ) + if built is None: + missing.append(spec.statistic_id) + else: + series[spec.key] = built + if missing: + log.info("statistics.no_data", statistic_ids=sorted(set(missing))) + return StatisticsHistory(series=series, missing=tuple(dict.fromkeys(missing))) + + +__all__ = [ + "DEFAULT_BATCH_SIZE", + "DEFAULT_CHUNK_DAYS", + "STATISTICS_UNITS", + "StatKind", + "StatisticSpec", + "StatisticsHistory", + "fetch_quarter_history", + "hourly_values", + "quarter_hours", +] diff --git a/suncore/ha/websocket.py b/suncore/ha/websocket.py index 7a49628..9b0860b 100644 --- a/suncore/ha/websocket.py +++ b/suncore/ha/websocket.py @@ -77,6 +77,8 @@ DEFAULT_WRITE_INTERVAL_S, REGISTRY_EVENT_TYPES, ActionNotAllowedError, + ActionRateLimitedError, + ActionRateLimiter, AuthError, CommandError, CommandTimeoutError, @@ -106,6 +108,18 @@ """Period of the full ``get_states`` resync. It is also the real freshness of a sensor whose value does not change: the stream is silent, the resync sees its ``last_reported`` move.""" +STATISTIC_PERIODS: Final = frozenset({"5minute", "hour", "day", "week", "month", "year"}) +"""Periods of ``recorder/statistics_during_period`` (verified, research/ha-websocket-api.md).""" + + +def _iso_with_offset(moment: datetime) -> str: + """ISO 8601 with a UTC offset: a value without one is read in HA's time zone.""" + if moment.tzinfo is None or moment.utcoffset() is None: + msg = "statistics time range needs timezone-aware datetimes" + raise ValueError(msg) + return moment.isoformat() + + MAX_MESSAGE_BYTES: Final = 32 * 1024 * 1024 """A full ``get_states`` of a large installation exceeds aiohttp's 4 MiB default.""" @@ -312,6 +326,7 @@ def __init__( self._resync_interval = resync_interval self._stable_after = stable_after self._write_limiter = RateLimiter(write_interval, clock=clock, sleep=sleep) + self._action_limiter = ActionRateLimiter(clock=clock, sleep=sleep) secrets.register(transport.token) @@ -468,15 +483,21 @@ async def call_action_response( data: Mapping[str, Any] | None = None, *, target: Mapping[str, Any] | None = None, + min_interval_s: float | None = None, + max_wait_s: float = 0.0, ) -> dict[str, Any]: """Read-only action call with ``return_response``; see :class:`HAConnection`. - Deliberately not rate limited like writes (no device is touched) and not routed - through :class:`ServiceCaller`: the allowlist is the guard. + Not routed through :class:`ServiceCaller` (no device is touched): the allowlist is + the guard against writes and the per-action :class:`ActionRateLimiter` the guard + against hammering an integration (follow-up A3). """ if not is_read_only_action(domain, service): msg = f"{domain}.{service} is not a read-only action" raise ActionNotAllowedError(msg) + key = f"{domain}.{service}" + if not await self._action_limiter.acquire(key, min_interval_s, max_wait_s=max_wait_s): + raise ActionRateLimitedError(key, self._action_limiter.retry_after(key)) payload: dict[str, Any] = { "type": "call_service", "domain": domain, @@ -494,6 +515,44 @@ async def call_action_response( raise ProtocolError(msg) return dict(response) + async def statistics_during_period( + self, + statistic_ids: Sequence[str], + start: datetime, + end: datetime | None = None, + *, + period: str = "hour", + types: Sequence[str] | None = None, + units: Mapping[str, str] | None = None, + ) -> Mapping[str, Sequence[Mapping[str, Any]]]: + """``recorder/statistics_during_period``; see :class:`HAConnection` for the shape.""" + if period not in STATISTIC_PERIODS: + msg = f"unknown statistics period {period!r}" + raise ValueError(msg) + if not statistic_ids: + return {} + payload: dict[str, Any] = { + "type": "recorder/statistics_during_period", + "start_time": _iso_with_offset(start), + "statistic_ids": list(statistic_ids), + "period": period, + } + if end is not None: + payload["end_time"] = _iso_with_offset(end) + if types is not None: + payload["types"] = list(types) + if units is not None: + payload["units"] = dict(units) + result = await self._command(payload) + if not isinstance(result, Mapping): + msg = "recorder/statistics_during_period returned a non-object" + raise ProtocolError(msg) + return { + str(key): [row for row in rows if isinstance(row, Mapping)] + for key, rows in result.items() + if isinstance(rows, list) + } + # --- supervisor loop ------------------------------------------------------ async def _run(self) -> None: diff --git a/suncore/sim/fake_ha.py b/suncore/sim/fake_ha.py index 08fda53..45bb5a4 100644 --- a/suncore/sim/fake_ha.py +++ b/suncore/sim/fake_ha.py @@ -48,7 +48,7 @@ import heapq import itertools import logging -from collections.abc import Awaitable, Callable, Collection, Mapping +from collections.abc import Awaitable, Callable, Collection, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta from functools import partial @@ -90,6 +90,7 @@ validate_schema, ) from suncore.sim.house import House, HouseState +from suncore.sim.recorder import SimRecorder from suncore.sim.scenario import ManualEvent, Scenario, SimSignal _log = logging.getLogger(__name__) @@ -257,6 +258,19 @@ def __init__( self._unavailable: frozenset[str] = frozenset() self.call_log: list[CallRecord] = [] """Every service call seen, oldest first.""" + self.recorder = SimRecorder( + scenario, + { + GRID_ID: lambda s: s.grid_w, + PV_ID: lambda s: s.pv_w, + BATTERY_ID: lambda s: s.battery_w, + SOC_ID: lambda s: s.soc_pct, + HOUSE_ID: lambda s: s.load_w, + }, + ) + """Hourly statistics behind :meth:`statistics_during_period`.""" + self.statistics_error: HAError | None = None + """Make ``statistics_during_period`` fail (e.g. a proxy that blocks ``recorder/*``).""" # --- HAConnection --------------------------------------------------------- @@ -388,6 +402,24 @@ def fire_registry_event(self, event_type: str = REGISTRY_EVENT_TYPES[0]) -> None except Exception: _log.exception("registry listener failed") + async def statistics_during_period( + self, + statistic_ids: Sequence[str], + start: datetime, + end: datetime | None = None, + *, + period: str = "hour", + types: Sequence[str] | None = None, + units: Mapping[str, str] | None = None, # noqa: ARG002 - the fake always answers in W + ) -> Mapping[str, Sequence[Mapping[str, Any]]]: + """``recorder/statistics_during_period`` from :attr:`recorder` (hourly means in W).""" + self._require_connected() + if self.statistics_error is not None: + raise self.statistics_error + if not statistic_ids: + return {} + return self.recorder.query(statistic_ids, start, end, period, types) + async def call_action_response( self, domain: str, @@ -395,6 +427,8 @@ async def call_action_response( data: Mapping[str, Any] | None = None, # noqa: ARG002 *, target: Mapping[str, Any] | None = None, # noqa: ARG002 + min_interval_s: float | None = None, # noqa: ARG002 - the fake is not rate limited + max_wait_s: float = 0.0, # noqa: ARG002 ) -> dict[str, Any]: """Read-only action responses built from the demo data (allowlist like the real one).""" if not is_read_only_action(domain, service): diff --git a/suncore/sim/recorder.py b/suncore/sim/recorder.py new file mode 100644 index 0000000..49a3bea --- /dev/null +++ b/suncore/sim/recorder.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A minimal fake of Home Assistant's recorder statistics for the simulator (Phase 4 WP3). + +Answers ``recorder/statistics_during_period`` with ``period: "hour"`` in the verified shape +(``docs/research/ha-websocket-api.md``): rows of ``{start, end, mean}`` with epoch +**milliseconds**, no unit, ids without data absent from the result. The history is the +simulated house run *before* the scenario start at 15-minute steps (clear sky, no devices), +aggregated to hourly means - a plausible week for the rebuild, not a replay of the scenario. +Tests can replace the rows of any id with :meth:`set_rows`. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from datetime import UTC, datetime, timedelta +from typing import Any, Final + +from suncore.ha.connection import CommandError +from suncore.sim.house import House, HouseState +from suncore.sim.scenario import Scenario + +HISTORY_STEP_S: Final = 900 +HOUR_MS: Final = 3_600_000 + + +class SimRecorder: + """Hourly statistics of the simulated house. + + Args: + scenario: supplies the house parameters and the end of the history (its start). + series: statistic id -> reader of the value from a house state (the sim sensors). + history_hours: how far back the history reaches. + """ + + def __init__( + self, + scenario: Scenario, + series: Mapping[str, Callable[[HouseState], float]], + *, + history_hours: int = 72, + ) -> None: + self._scenario = scenario + self._series = dict(series) + self._history_hours = history_hours + self._rows: dict[str, list[dict[str, Any]]] | None = None + self._overrides: dict[str, list[dict[str, Any]]] = {} + self.calls: list[dict[str, Any]] = [] + """Every query seen (evidence for the chunking tests).""" + + def set_rows(self, statistic_id: str, rows: Sequence[Mapping[str, Any]]) -> None: + """Replace (or add) the rows of one statistic; an empty list removes its data.""" + self._overrides[statistic_id] = [dict(r) for r in rows] + + def _build(self) -> dict[str, list[dict[str, Any]]]: + end = self._scenario.start.astimezone(UTC).replace(minute=0, second=0, microsecond=0) + begin = end - timedelta(hours=self._history_hours) + house = House(self._scenario.house, begin, HISTORY_STEP_S) + steps_per_hour = 3600 // HISTORY_STEP_S + rows: dict[str, list[dict[str, Any]]] = {sid: [] for sid in self._series} + for hour in range(self._history_hours): + states = [house.step() for _ in range(steps_per_hour)] + start_ms = int((begin + timedelta(hours=hour)).timestamp() * 1000) + for statistic_id, read in self._series.items(): + mean = sum(read(s) for s in states) / len(states) + rows[statistic_id].append( + {"start": start_ms, "end": start_ms + HOUR_MS, "mean": mean} + ) + return rows + + def query( + self, + statistic_ids: Sequence[str], + start: datetime, + end: datetime | None, + period: str, + types: Sequence[str] | None, + ) -> dict[str, list[dict[str, Any]]]: + """The ``result`` object of ``recorder/statistics_during_period``.""" + self.calls.append( + {"ids": list(statistic_ids), "start": start, "end": end, "period": period} + ) + if period != "hour": + raise CommandError("not_found", f"period {period!r} is not simulated") + if self._rows is None: + self._rows = self._build() + lo = int(start.timestamp() * 1000) + hi = None if end is None else int(end.timestamp() * 1000) + wanted = tuple(types) if types is not None else ("mean",) + result: dict[str, list[dict[str, Any]]] = {} + for statistic_id in statistic_ids: + source = self._overrides.get(statistic_id, self._rows.get(statistic_id, [])) + picked = [ + {k: v for k, v in row.items() if k in ("start", "end", *wanted)} + for row in source + if row["start"] >= lo and (hi is None or row["start"] < hi) + ] + if picked: + result[statistic_id] = picked + return result diff --git a/suncore/storage/__init__.py b/suncore/storage/__init__.py index 772ea6a..9713a82 100644 --- a/suncore/storage/__init__.py +++ b/suncore/storage/__init__.py @@ -15,7 +15,14 @@ StorageError, UnknownSectionError, ) +from suncore.storage.learned import LearnedParam, LearnedParamStore +from suncore.storage.plans import PlanSnapshot, PlanSnapshotMeta, PlanSnapshotStore from suncore.storage.runtime import PowerModelStateStore, SqliteRuntimeStore +from suncore.storage.timeseries import ( + TimeseriesStore, + open_timeseries, + open_timeseries_or_recover, +) __all__ = [ "ConfigCorruptError", @@ -25,11 +32,19 @@ "HistoryEntry", "HistoryEntryNotFoundError", "ImportRejectedError", + "LearnedParam", + "LearnedParamStore", "MigrationError", + "PlanSnapshot", + "PlanSnapshotMeta", + "PlanSnapshotStore", "PowerModelStateStore", "SchemaTooNewError", "SectionSpec", "SqliteRuntimeStore", "StorageError", + "TimeseriesStore", "UnknownSectionError", + "open_timeseries", + "open_timeseries_or_recover", ] diff --git a/suncore/storage/db.py b/suncore/storage/db.py index e9da307..c4c5969 100644 --- a/suncore/storage/db.py +++ b/suncore/storage/db.py @@ -4,6 +4,8 @@ from __future__ import annotations import asyncio +import contextlib +import sqlite3 from collections.abc import AsyncIterator, Iterable, Sequence from contextlib import asynccontextmanager from pathlib import Path @@ -12,12 +14,26 @@ import aiosqlite -from suncore.storage.migrations import run_migrations +from suncore.storage.migrations import Migration, run_migrations SqlParam = str | int | float | bytes | None Row = tuple[object, ...] +async def _rollback(conn: aiosqlite.Connection) -> None: + """``ROLLBACK`` that completes even if the awaiting task is cancelled meanwhile. + + Statements of a cancelled caller are still executed by the connection's worker thread, so + the rollback is queued behind them and shielded; "no transaction is active" is fine. + """ + + async def run() -> None: + with contextlib.suppress(sqlite3.OperationalError): + await conn.execute("ROLLBACK") + + await asyncio.shield(asyncio.ensure_future(run())) + + class Transaction: """Statements inside one ``BEGIN IMMEDIATE`` block; obtained from ``Database.transaction``.""" @@ -32,6 +48,11 @@ async def execute(self, sql: str, params: Sequence[SqlParam] = ()) -> int: finally: await cur.close() + async def executemany(self, sql: str, rows: Iterable[Sequence[SqlParam]]) -> None: + """Run a statement for every parameter row (one round trip to the SQLite thread).""" + cur = await self._conn.executemany(sql, rows) + await cur.close() + async def fetchall(self, sql: str, params: Sequence[SqlParam] = ()) -> list[Row]: async with self._conn.execute(sql, params) as cur: return [tuple(r) for r in await cur.fetchall()] @@ -42,7 +63,7 @@ async def fetchone(self, sql: str, params: Sequence[SqlParam] = ()) -> Row | Non class Database: - """One connection to ``suncore.db``; all access is serialised by a lock.""" + """One connection to a SQLite file (``suncore.db``, ``timeseries.db``), serialised by a lock.""" def __init__(self, path: Path | str) -> None: self._path = str(path) @@ -50,12 +71,16 @@ def __init__(self, path: Path | str) -> None: self._lock = asyncio.Lock() @classmethod - async def open(cls, path: Path | str) -> Self: - """Open (creating if needed), configure, and migrate the database.""" + async def open(cls, path: Path | str, *, migrations: list[Migration] | None = None) -> Self: + """Open (creating if needed), configure, and migrate the database. + + ``migrations`` selects the migration set (default: ``suncore.db``'s; the time-series + database passes its own). A database newer than the set raises ``SchemaTooNewError``. + """ db = cls(path) await db._connect() try: - await run_migrations(db._connection()) + await run_migrations(db._connection(), migrations) except BaseException: await db.close() raise @@ -116,8 +141,7 @@ async def fetchone(self, sql: str, params: Sequence[SqlParam] = ()) -> Row | Non async def executemany(self, sql: str, rows: Iterable[Sequence[SqlParam]]) -> None: async with self.transaction() as tx: - for row in rows: - await tx.execute(sql, row) + await tx.executemany(sql, rows) @asynccontextmanager async def transaction(self) -> AsyncIterator[Transaction]: @@ -131,14 +155,14 @@ async def transaction(self) -> AsyncIterator[Transaction]: try: await conn.execute("BEGIN IMMEDIATE") except BaseException: - # A cancelled caller may leave the BEGIN running in the worker thread. - if conn.in_transaction: - await conn.execute("ROLLBACK") + # A cancelled caller may leave the BEGIN queued in the worker thread: the + # rollback queues behind it, so it always sees the transaction (if any). + await _rollback(conn) raise try: yield Transaction(conn) except BaseException: - await conn.execute("ROLLBACK") + await _rollback(conn) raise else: await conn.execute("COMMIT") diff --git a/suncore/storage/learned.py b/suncore/storage/learned.py new file mode 100644 index 0000000..b693c27 --- /dev/null +++ b/suncore/storage/learned.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Learned parameters (Phase 4 WP3): bias tables, error distributions, load profiles, ... + +One row per ``(namespace, key)`` holding a JSON object. The module that owns a namespace +defines the payload and migrates old payloads by ``schema_version``; this repository only +stores, lists and deletes them. A payload that does not parse is treated as missing (logged), +so a learned value can never stop the application - the owner then starts learning again. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any, Final + +import structlog + +from suncore.storage.db import Database + +log = structlog.get_logger(__name__) + +NS_FORECAST_BIAS: Final = "forecast_bias" +NS_NET_ERROR: Final = "net_error" +NS_LOAD_PROFILE: Final = "load_profile" +NS_BLEND_WEIGHTS: Final = "blend_weights" +NS_ML_STATE: Final = "ml_state" + +KNOWN_NAMESPACES: Final[frozenset[str]] = frozenset( + {NS_FORECAST_BIAS, NS_NET_ERROR, NS_LOAD_PROFILE, NS_BLEND_WEIGHTS, NS_ML_STATE} +) +"""The namespaces Phase 4 uses; other well-formed names are accepted (later phases).""" + +_NAME_RE: Final = re.compile(r"^[a-z][a-z0-9_]{0,63}$") +_KEY_MAX: Final = 200 + + +def _now() -> datetime: + return datetime.now(UTC) + + +@dataclass(frozen=True, slots=True) +class LearnedParam: + namespace: str + key: str + payload: dict[str, Any] + schema_version: int + updated_at: datetime + + +def _check(namespace: str, key: str | None = None) -> None: + if not _NAME_RE.match(namespace): + msg = f"invalid learned-parameter namespace {namespace!r}" + raise ValueError(msg) + if key is not None and not 0 < len(key) <= _KEY_MAX: + msg = "learned-parameter key must be 1..200 characters" + raise ValueError(msg) + + +class LearnedParamStore: + """``learned_param`` table behind typed methods.""" + + def __init__(self, db: Database, clock: Callable[[], datetime] = _now) -> None: + self._db = db + self._clock = clock + + async def put( + self, namespace: str, key: str, payload: Mapping[str, Any], *, schema_version: int = 1 + ) -> None: + """Insert or replace one parameter set.""" + _check(namespace, key) + await self._db.execute( + "INSERT INTO learned_param (namespace, key, payload, schema_version, updated_at) " + "VALUES (?, ?, ?, ?, ?) ON CONFLICT(namespace, key) DO UPDATE SET " + "payload = excluded.payload, schema_version = excluded.schema_version, " + "updated_at = excluded.updated_at", + ( + namespace, + key, + json.dumps(payload, allow_nan=False, sort_keys=True), + schema_version, + self._clock().astimezone(UTC).isoformat(), + ), + ) + + async def get(self, namespace: str, key: str) -> LearnedParam | None: + _check(namespace, key) + row = await self._db.fetchone( + "SELECT namespace, key, payload, schema_version, updated_at FROM learned_param " + "WHERE namespace = ? AND key = ?", + (namespace, key), + ) + return None if row is None else self._decode(row) + + async def in_namespace(self, namespace: str) -> list[LearnedParam]: + """All readable entries of a namespace, ordered by key.""" + _check(namespace) + rows = await self._db.fetchall( + "SELECT namespace, key, payload, schema_version, updated_at FROM learned_param " + "WHERE namespace = ? ORDER BY key", + (namespace,), + ) + decoded = (self._decode(r) for r in rows) + return [p for p in decoded if p is not None] + + async def delete(self, namespace: str, key: str) -> bool: + _check(namespace, key) + deleted = await self._db.execute( + "DELETE FROM learned_param WHERE namespace = ? AND key = ?", (namespace, key) + ) + return deleted > 0 + + async def delete_namespace(self, namespace: str) -> int: + """Forget everything learned in a namespace (e.g. a reset button); returns the count.""" + _check(namespace) + return await self._db.execute("DELETE FROM learned_param WHERE namespace = ?", (namespace,)) + + @staticmethod + def _decode(row: tuple[object, ...]) -> LearnedParam | None: + try: + data = json.loads(str(row[2])) + except json.JSONDecodeError: + data = None + if not isinstance(data, dict): + log.warning("learned_param.unreadable", namespace=row[0], key=row[1]) + return None + return LearnedParam( + namespace=str(row[0]), + key=str(row[1]), + payload=data, + schema_version=int(str(row[3])), + updated_at=datetime.fromisoformat(str(row[4])), + ) + + +__all__ = [ + "KNOWN_NAMESPACES", + "NS_BLEND_WEIGHTS", + "NS_FORECAST_BIAS", + "NS_LOAD_PROFILE", + "NS_ML_STATE", + "NS_NET_ERROR", + "LearnedParam", + "LearnedParamStore", +] diff --git a/suncore/storage/migrations/0004_plans_learned.sql b/suncore/storage/migrations/0004_plans_learned.sql new file mode 100644 index 0000000..225e17c --- /dev/null +++ b/suncore/storage/migrations/0004_plans_learned.sql @@ -0,0 +1,25 @@ +-- SPDX-License-Identifier: Apache-2.0 +-- Phase 4 (WP3): plan snapshots and learned parameters. +-- Schema 4 cannot be opened by earlier versions (they refuse newer databases): export before +-- updating. The `learned` table of 0001 stays untouched (unused since Phase 1). + +-- Compressed JSON of a computed plan (zlib); retention in suncore.storage.plans. +CREATE TABLE plan_snapshot ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL, + source TEXT NOT NULL, + schema_version INTEGER NOT NULL, + payload BLOB NOT NULL +); +CREATE INDEX plan_snapshot_created_idx ON plan_snapshot (created_at); + +-- Learned parameters: forecast_bias, net_error, load_profile, blend_weights, ml_state, ... +-- The owner of a namespace defines its payload and migrates it by schema_version. +CREATE TABLE learned_param ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + payload TEXT NOT NULL CHECK (json_valid(payload)), + schema_version INTEGER NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (namespace, key) +); diff --git a/suncore/storage/plans.py b/suncore/storage/plans.py new file mode 100644 index 0000000..eace1dc --- /dev/null +++ b/suncore/storage/plans.py @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Plan snapshots: compressed JSON payloads of computed plans (Phase 4 WP3). + +The planner (WP9) stores each plan it computes; the UI's plan history and the accuracy +bookkeeping read them back. The payload is opaque here - the plan model owns its shape and +passes its ``schema_version``. Retention keeps at most :data:`KEEP_LAST` snapshots and none +older than :data:`KEEP_DAYS` days. +""" + +from __future__ import annotations + +import json +import zlib +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any, Final + +import structlog + +from suncore.storage.db import Database + +log = structlog.get_logger(__name__) + +KEEP_LAST: Final = 200 +KEEP_DAYS: Final = 14 +MAX_LIMIT: Final = 500 +COMPRESSION_LEVEL: Final = 6 + + +def _now() -> datetime: + return datetime.now(UTC) + + +@dataclass(frozen=True, slots=True) +class PlanSnapshotMeta: + """Header of a stored snapshot (no payload).""" + + id: int + created_at: datetime + source: str + schema_version: int + + +@dataclass(frozen=True, slots=True) +class PlanSnapshot: + """A stored snapshot with its decoded payload.""" + + id: int + created_at: datetime + source: str + schema_version: int + payload: dict[str, Any] + + +def encode_payload(payload: Mapping[str, Any]) -> bytes: + """JSON (finite numbers only) compressed with zlib.""" + text = json.dumps(payload, allow_nan=False, separators=(",", ":"), sort_keys=True) + return zlib.compress(text.encode("utf-8"), COMPRESSION_LEVEL) + + +def decode_payload(blob: bytes) -> dict[str, Any]: + """Inverse of :func:`encode_payload`; ``ValueError`` if the blob is not a JSON object.""" + try: + data = json.loads(zlib.decompress(blob).decode("utf-8")) + except (zlib.error, UnicodeDecodeError, json.JSONDecodeError) as exc: + msg = f"plan snapshot payload unreadable: {exc}" + raise ValueError(msg) from exc + if not isinstance(data, dict): + msg = "plan snapshot payload is not an object" + raise ValueError(msg) + return data + + +class PlanSnapshotStore: + """``plan_snapshot`` table behind typed methods.""" + + def __init__(self, db: Database, clock: Callable[[], datetime] = _now) -> None: + self._db = db + self._clock = clock + + async def save( + self, + payload: Mapping[str, Any], + *, + source: str, + schema_version: int, + created_at: datetime | None = None, + ) -> int: + """Store a snapshot and return its id (ids grow with time).""" + when = (created_at or self._clock()).astimezone(UTC) + blob = encode_payload(payload) + async with self._db.transaction() as tx: + await tx.execute( + "INSERT INTO plan_snapshot (created_at, source, schema_version, payload) " + "VALUES (?, ?, ?, ?)", + (when.isoformat(), source, schema_version, blob), + ) + row = await tx.fetchone("SELECT last_insert_rowid()") + assert row is not None + return int(str(row[0])) + + async def get(self, snapshot_id: int) -> PlanSnapshot | None: + """The snapshot, or ``None`` if it does not exist or its payload is unreadable.""" + row = await self._db.fetchone( + "SELECT id, created_at, source, schema_version, payload FROM plan_snapshot " + "WHERE id = ?", + (snapshot_id,), + ) + return None if row is None else self._decode(row) + + async def latest(self) -> PlanSnapshot | None: + """The newest readable snapshot.""" + rows = await self._db.fetchall( + "SELECT id, created_at, source, schema_version, payload FROM plan_snapshot " + "ORDER BY id DESC LIMIT 5" + ) + for row in rows: + snapshot = self._decode(row) + if snapshot is not None: + return snapshot + return None + + async def headers(self, limit: int = 50) -> list[PlanSnapshotMeta]: + """Headers, newest first; ``limit`` is clamped to ``1..MAX_LIMIT``.""" + bounded = max(1, min(int(limit), MAX_LIMIT)) + rows = await self._db.fetchall( + "SELECT id, created_at, source, schema_version FROM plan_snapshot " + "ORDER BY id DESC LIMIT ?", + (bounded,), + ) + return [ + PlanSnapshotMeta( + id=int(str(r[0])), + created_at=datetime.fromisoformat(str(r[1])), + source=str(r[2]), + schema_version=int(str(r[3])), + ) + for r in rows + ] + + async def count(self) -> int: + row = await self._db.fetchone("SELECT COUNT(*) FROM plan_snapshot") + return int(str(row[0])) if row else 0 + + async def prune( + self, now: datetime, keep_last: int = KEEP_LAST, keep_days: int = KEEP_DAYS + ) -> int: + """Drop snapshots older than ``keep_days`` and all but the newest ``keep_last``.""" + cutoff = (now.astimezone(UTC) - timedelta(days=keep_days)).isoformat() + async with self._db.transaction() as tx: + by_age = await tx.execute("DELETE FROM plan_snapshot WHERE created_at < ?", (cutoff,)) + by_count = await tx.execute( + "DELETE FROM plan_snapshot WHERE id <= " + "COALESCE((SELECT id FROM plan_snapshot ORDER BY id DESC LIMIT 1 OFFSET ?), 0)", + (max(0, keep_last),), + ) + deleted = by_age + by_count + if deleted: + log.info("plan_snapshot.pruned", deleted=deleted) + return deleted + + @staticmethod + def _decode(row: tuple[object, ...]) -> PlanSnapshot | None: + try: + payload = decode_payload(bytes(row[4])) # type: ignore[call-overload] + except ValueError as exc: + log.warning("plan_snapshot.unreadable", snapshot_id=row[0], error=str(exc)[:200]) + return None + return PlanSnapshot( + id=int(str(row[0])), + created_at=datetime.fromisoformat(str(row[1])), + source=str(row[2]), + schema_version=int(str(row[3])), + payload=payload, + ) + + +__all__ = [ + "KEEP_DAYS", + "KEEP_LAST", + "PlanSnapshot", + "PlanSnapshotMeta", + "PlanSnapshotStore", + "decode_payload", + "encode_payload", +] diff --git a/suncore/storage/timeseries.py b/suncore/storage/timeseries.py new file mode 100644 index 0000000..8518952 --- /dev/null +++ b/suncore/storage/timeseries.py @@ -0,0 +1,711 @@ +# SPDX-License-Identifier: Apache-2.0 +"""``timeseries.db``: measurements, forecasts and prices (ADR-0004, Phase 4 WP3). + +A separate SQLite file next to ``suncore.db``, excluded from Home Assistant backups +(``backup_exclude``) and rebuildable: the 15-minute history comes back from HA's long-term +statistics (``suncore.app.history``). It has its own migrations (``ts_migrations``) and its own +``PRAGMA user_version``; a file that is newer than the code is refused (and left untouched). + +Tables and retention (all times injected, nothing reads the wall clock): + +======================== ====================================== ============= +table content kept +======================== ====================================== ============= +``measurement`` controller resolution, one row per tick 48 h +``measurement_1m`` mean/min/max/n per minute 30 days +``measurement_15m`` mean/min/max/n per quarter hour unlimited +``forecast`` P10/P50/P90 per slot, per issue time 45 days (slots) +``price`` import/export price per slot 400 days +======================== ====================================== ============= + +Conventions: timestamps are integer seconds since the UNIX epoch (UTC); a gap in a measurement +is a missing row (never a stored zero) and reads back as ``None``; the aggregates are +sample-weighted, so ``mean * n`` adds up exactly (a 15-minute mean is the sample mean over its +minutes) and the power-times-time energy of an interval survives the downsampling. +``downsample`` is idempotent: it recomputes every complete bucket from the finer table, and +the retention cut-offs are aligned to the bucket size, so a bucket is either fully present in +its source table or gone - a re-run never aggregates half a bucket. +""" + +from __future__ import annotations + +import math +import sqlite3 +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from importlib import resources +from pathlib import Path +from typing import Final, Literal + +import structlog + +from suncore.core.measurements import is_percent_key +from suncore.core.series import ( + MAX_POINTS, + PriceIncludes, + PriceSeries, + QuantileSeries, + SeriesUnit, + TimeRange, + TimeSeries, +) +from suncore.storage.db import Database, Row, SqlParam +from suncore.storage.errors import MigrationError, SchemaTooNewError +from suncore.storage.migrations import Migration, load_migrations + +log = structlog.get_logger(__name__) + +TS_FILENAME: Final = "timeseries.db" + +RAW_KEEP: Final = timedelta(hours=48) +MINUTE_KEEP: Final = timedelta(days=30) +FORECAST_KEEP: Final = timedelta(days=45) +PRICE_KEEP: Final = timedelta(days=400) + +VACUUM_MIN_FREE_BYTES: Final = 8 * 1024 * 1024 +VACUUM_MIN_FREE_FRACTION: Final = 0.25 +"""``VACUUM`` rewrites the file; it runs only if at least this much (and this share) is free.""" + +MINUTE_S: Final = 60 +QUARTER_S: Final = 900 + +Resolution = Literal["1m", "15m"] +ForecastKind = Literal["pv", "load", "price"] +_STEP_OF: Final[Mapping[str, int]] = {"1m": MINUTE_S, "15m": QUARTER_S} +_TABLE_OF: Final[Mapping[str, str]] = {"1m": "measurement_1m", "15m": "measurement_15m"} + +MeasurementRow = tuple[str, datetime, float | None] +"""``(key, timestamp, value)``; a ``None`` value is a gap and is not stored.""" + + +# --- opening ------------------------------------------------------------------------ + + +def timeseries_migrations() -> list[Migration]: + """The migrations of ``timeseries.db``.""" + return load_migrations(resources.files("suncore.storage.ts_migrations")) + + +async def open_timeseries(path: Path | str) -> Database: + """Open (creating, migrating) ``timeseries.db``; raises ``SchemaTooNewError`` if too new.""" + return await Database.open(path, migrations=timeseries_migrations()) + + +async def open_timeseries_or_recover(path: Path | str) -> Database | None: + """Open ``timeseries.db`` for the application; ``None`` = run without time series. + + The file is rebuildable, so a corrupt one (not a database, a migration that fails) is + moved aside as ``timeseries.db.corrupt`` and a fresh one is created. A file that is + *newer* than this code is never touched: the application runs without time series and + says so (the newer version may still need the data). Control never depends on this file. + """ + target = Path(path) + for attempt in (1, 2): + try: + return await open_timeseries(target) + except SchemaTooNewError as exc: + log.error("timeseries.schema_too_new", path=str(target), error=str(exc)) + return None + except (sqlite3.DatabaseError, MigrationError, OSError) as exc: + log.error("timeseries.open_failed", path=str(target), attempt=attempt, error=str(exc)) + if attempt == 2: + return None + try: + _move_aside(target) + except OSError: + log.exception("timeseries.move_aside_failed", path=str(target)) + return None + return None + + +def _move_aside(path: Path) -> None: + for suffix in ("-wal", "-shm"): + path.with_name(path.name + suffix).unlink(missing_ok=True) + if path.exists(): + path.replace(path.with_name(path.name + ".corrupt")) + + +# --- helpers ------------------------------------------------------------------------ + + +def _epoch(moment: datetime) -> int: + if moment.tzinfo is None or moment.utcoffset() is None: + msg = "timestamps must be timezone-aware" + raise ValueError(msg) + return math.floor(moment.timestamp()) + + +def _from_epoch(seconds: int) -> datetime: + return datetime.fromtimestamp(seconds, UTC) + + +def _floor(seconds: int, step: int) -> int: + return seconds - seconds % step + + +def _num(value: object) -> float: + assert isinstance(value, int | float) + return float(value) + + +def _int(value: object) -> int: + assert isinstance(value, int | float) + return int(value) + + +def _opt(value: float | None) -> float | None: + return None if value is None or not math.isfinite(value) else value + + +def _opt_num(value: object) -> float | None: + return None if value is None else _num(value) + + +def _default_unit(key: str) -> SeriesUnit: + return SeriesUnit.PERCENT if is_percent_key(key) else SeriesUnit.W + + +# --- reports ------------------------------------------------------------------------ + + +@dataclass(frozen=True, slots=True) +class DownsampleReport: + minute_rows: int + quarter_rows: int + + +@dataclass(frozen=True, slots=True) +class RetentionReport: + raw: int + minute: int + forecast: int + price: int + + @property + def total(self) -> int: + return self.raw + self.minute + self.forecast + self.price + + +@dataclass(frozen=True, slots=True) +class MaintenanceReport: + downsample: DownsampleReport + retention: RetentionReport + vacuumed: bool + size_bytes: int + + +@dataclass(frozen=True, slots=True) +class TimeseriesSize: + """Size of ``timeseries.db`` for the diagnostics page.""" + + file_bytes: int + wal_bytes: int + free_bytes: int + """Pages inside the file that hold no data (reclaimed by ``VACUUM``).""" + rows: Mapping[str, int] + + @property + def total_bytes(self) -> int: + return self.file_bytes + self.wal_bytes + + +# --- the repository ----------------------------------------------------------------- + + +class TimeseriesStore: + """Repository over ``timeseries.db`` with typed methods and core series as results.""" + + def __init__(self, db: Database) -> None: + self._db = db + + @property + def database(self) -> Database: + return self._db + + # --- measurements ----------------------------------------------------------- + + async def append_measurements(self, rows: Iterable[MeasurementRow]) -> int: + """Store measurements in one batch; gaps (``None``, NaN, inf) are skipped. + + A second value for the same ``(key, second)`` replaces the first. Returns the number + of rows written. + """ + batch: list[Sequence[SqlParam]] = [] + for key, moment, value in rows: + checked = _opt(value) + if checked is not None: + batch.append((key, _epoch(moment), checked)) + if batch: + await self._db.executemany( + "INSERT OR REPLACE INTO measurement (key, ts, value) VALUES (?, ?, ?)", batch + ) + return len(batch) + + async def read_raw(self, key: str, window: TimeRange) -> list[tuple[datetime, float]]: + """Controller-resolution samples in ``window`` (oldest first).""" + rows = await self._db.fetchall( + "SELECT ts, value FROM measurement WHERE key = ? AND ts >= ? AND ts < ? ORDER BY ts", + (key, _epoch(window.start), _ceil_epoch(window.end)), + ) + return [(_from_epoch(_int(r[0])), _num(r[1])) for r in rows] + + async def read_series( + self, + key: str, + window: TimeRange, + resolution: Resolution = "15m", + unit: SeriesUnit | None = None, + *, + include_recent: bool = True, + ) -> TimeSeries: + """The mean of ``key`` per minute or quarter hour as a regular series. + + The grid starts at the bucket containing ``window.start`` and ends at the bucket + containing the last instant of the window. A bucket without data is ``None``. + With ``include_recent`` (15 minutes only), buckets that the nightly job has not + aggregated yet are computed on the fly from the 1-minute and then the raw rows, so + yesterday's data is available at once; a bucket still in progress is included with + what has been measured so far (end the window at the last complete bucket if that + matters). + """ + step = _STEP_OF[resolution] + first = _floor(_epoch(window.start), step) + end = _ceil_epoch(window.end) + count = max(1, -(-(end - first) // step)) + if count > MAX_POINTS: + msg = f"read_series: {count} points exceed the limit of {MAX_POINTS}" + raise ValueError(msg) + last = first + count * step + means: dict[int, float] = {} + if include_recent and resolution == "15m": + means.update(await self._recent_quarters(key, first, last)) + table = _TABLE_OF[resolution] + rows = await self._db.fetchall( + f"SELECT ts, mean FROM {table} WHERE key = ? AND ts >= ? AND ts < ?", + (key, first, last), + ) + means.update({_int(r[0]): _num(r[1]) for r in rows}) + values = tuple(means.get(first + i * step) for i in range(count)) + return TimeSeries( + start=_from_epoch(first), + step=timedelta(seconds=step), + values=values, + unit=unit or _default_unit(key), + ) + + async def _recent_quarters(self, key: str, first: int, last: int) -> dict[int, float]: + """Quarter-hour means from the finer tables (1 minute wins over raw per bucket).""" + out: dict[int, float] = {} + raw = await self._db.fetchall( + "SELECT (ts / 900) * 900 AS b, AVG(value) FROM measurement " + "WHERE key = ? AND ts >= ? AND ts < ? GROUP BY b", + (key, first, last), + ) + out.update({_int(r[0]): _num(r[1]) for r in raw}) + minutes = await self._db.fetchall( + "SELECT (ts / 900) * 900 AS b, SUM(mean * n) / SUM(n) FROM measurement_1m " + "WHERE key = ? AND ts >= ? AND ts < ? GROUP BY b", + (key, first, last), + ) + out.update({_int(r[0]): _num(r[1]) for r in minutes}) + return out + + async def store_quarter_hours( + self, key: str, series: TimeSeries, *, derived: bool = False + ) -> int: + """Write a 15-minute series into ``measurement_15m`` (``None`` slots are skipped). + + ``derived=True`` marks values rebuilt from Home Assistant's statistics: they fill + free slots and replace earlier derived rows but never overwrite measured ones. + Returns the number of slots offered. + """ + if series.step != timedelta(seconds=QUARTER_S): + msg = "store_quarter_hours: needs a 15-minute series" + raise ValueError(msg) + batch: list[Sequence[SqlParam]] = [] + for begin, _end, value in series.slots(): + checked = _opt(value) + if checked is not None: + batch.append((key, _epoch(begin), checked, checked, checked, 1 if derived else 0)) + if not batch: + return 0 + guard = "WHERE measurement_15m.derived = 1" if derived else "" + await self._db.executemany( + "INSERT INTO measurement_15m (key, ts, mean, min, max, n, derived) " + "VALUES (?, ?, ?, ?, ?, 1, ?) ON CONFLICT(key, ts) DO UPDATE SET " + "mean = excluded.mean, min = excluded.min, max = excluded.max, " + f"n = excluded.n, derived = excluded.derived {guard}", + batch, + ) + return len(batch) + + async def has_quarter_history(self) -> bool: + """Whether ``measurement_15m`` holds any row (the rebuild runs when it does not).""" + return await self._db.fetchone("SELECT 1 FROM measurement_15m LIMIT 1") is not None + + # --- forecasts ---------------------------------------------------------------- + + async def store_forecast( + self, + source_id: str, + kind: ForecastKind, + issued_at: datetime, + forecast: QuantileSeries | TimeSeries, + ) -> int: + """Store one issue of a forecast; an earlier store of the same issue is replaced. + + Every slot is stored, also an unknown one (all NULL), so the series reads back with + the same start and length. A plain ``TimeSeries`` fills ``p50`` only. + """ + p10: TimeSeries | None + p90: TimeSeries | None + if isinstance(forecast, QuantileSeries): + p10, p50, p90 = forecast.p10, forecast.p50, forecast.p90 + else: + p10 = p90 = None + p50 = forecast + step_s = int(p50.step.total_seconds()) + issued = _epoch(issued_at) + rows: list[Sequence[SqlParam]] = [] + for i, begin in enumerate(p50.slot_start(j) for j in range(len(p50))): + rows.append( + ( + source_id, + kind, + issued, + _epoch(begin), + step_s, + p50.unit.value, + None if p10 is None else p10.values[i], + p50.values[i], + None if p90 is None else p90.values[i], + ) + ) + async with self._db.transaction() as tx: + await tx.execute( + "DELETE FROM forecast WHERE source_id = ? AND kind = ? AND issued_at = ?", + (source_id, kind, issued), + ) + await tx.executemany( + "INSERT INTO forecast (source_id, kind, issued_at, slot_start, step_s, unit, " + "p10, p50, p90) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + rows, + ) + return len(rows) + + async def read_forecast( + self, + source_id: str, + kind: ForecastKind, + issued_at: datetime, + window: TimeRange | None = None, + ) -> QuantileSeries | None: + """One issue of a forecast (optionally clipped to ``window``); ``None`` if absent.""" + lo, hi = _bounds(window) + rows = await self._db.fetchall( + "SELECT slot_start, step_s, unit, p10, p50, p90 FROM forecast " + "WHERE source_id = ? AND kind = ? AND issued_at = ? AND slot_start >= ? " + "AND slot_start < ? ORDER BY slot_start", + (source_id, kind, _epoch(issued_at), lo, hi), + ) + return _quantile_series(rows) + + async def latest_forecast_before( + self, + source_id: str, + kind: ForecastKind, + before: datetime, + window: TimeRange | None = None, + ) -> QuantileSeries | None: + """Per slot the newest value issued at or before ``before`` (for the accuracy). + + To judge a forecast at lead time L, ask for ``before = slot_start - L``: every slot + then carries what was known L earlier, whichever issue that was. The result spans + the first to the last slot found; slots no issue covered are ``None``. + """ + lo, hi = _bounds(window) + limit = _epoch(before) + rows = await self._db.fetchall( + "SELECT f.slot_start, f.step_s, f.unit, f.p10, f.p50, f.p90 FROM forecast f " + "JOIN (SELECT slot_start, MAX(issued_at) AS issued FROM forecast " + "WHERE source_id = ? AND kind = ? AND issued_at <= ? AND slot_start >= ? " + "AND slot_start < ? GROUP BY slot_start) m " + "ON f.slot_start = m.slot_start AND f.issued_at = m.issued " + "WHERE f.source_id = ? AND f.kind = ? ORDER BY f.slot_start", + (source_id, kind, limit, lo, hi, source_id, kind), + ) + return _quantile_series(rows) + + async def forecast_issues( + self, source_id: str, kind: ForecastKind, since: datetime | None = None + ) -> list[datetime]: + """Issue times stored for a source, oldest first.""" + rows = await self._db.fetchall( + "SELECT DISTINCT issued_at FROM forecast WHERE source_id = ? AND kind = ? " + "AND issued_at >= ? ORDER BY issued_at", + (source_id, kind, 0 if since is None else _epoch(since)), + ) + return [_from_epoch(_int(r[0])) for r in rows] + + # --- prices ------------------------------------------------------------------- + + async def store_prices(self, prices: PriceSeries) -> int: + """Store prices; rows of the same source that overlap their time span are replaced.""" + step_s = int(prices.step.total_seconds()) + start = _epoch(prices.start) + end = start + step_s * len(prices) + export = prices.export_price + rows: list[Sequence[SqlParam]] = [ + ( + prices.source_id, + _epoch(prices.import_price.slot_start(i)), + step_s, + prices.price_includes, + prices.import_price.values[i], + None if export is None else export.values[i], + ) + for i in range(len(prices)) + ] + async with self._db.transaction() as tx: + await tx.execute( + "DELETE FROM price WHERE source_id = ? AND slot_start < ? " + "AND slot_start + step_s > ?", + (prices.source_id, end, start), + ) + await tx.executemany( + "INSERT INTO price (source_id, slot_start, step_s, includes, import_price, " + "export_price) VALUES (?, ?, ?, ?, ?, ?)", + rows, + ) + return len(rows) + + async def read_prices(self, source_id: str, window: TimeRange) -> list[PriceSeries]: + """Stored prices overlapping ``window`` as regular segments (a change of step, + ``price_includes`` or a gap that is not a whole number of slots starts a new one). + + A segment whose export prices are all unknown has ``export_price = None``. + """ + rows = await self._db.fetchall( + "SELECT slot_start, step_s, includes, import_price, export_price FROM price " + "WHERE source_id = ? AND slot_start < ? AND slot_start + step_s > ? " + "ORDER BY slot_start", + (source_id, _ceil_epoch(window.end), _epoch(window.start)), + ) + segments: list[list[Row]] = [] + for row in rows: + if segments and _continues(segments[-1], row): + segments[-1].append(row) + else: + segments.append([row]) + return [_price_segment(source_id, segment) for segment in segments] + + # --- maintenance -------------------------------------------------------------- + + async def downsample(self, now: datetime) -> DownsampleReport: + """Aggregate every complete minute and quarter hour (idempotent, see module doc).""" + now_s = _epoch(now) + async with self._db.transaction() as tx: + minutes = await tx.execute( + "INSERT INTO measurement_1m (key, ts, mean, min, max, n) " + "SELECT key, (ts / 60) * 60 AS b, AVG(value), MIN(value), MAX(value), COUNT(*) " + "FROM measurement WHERE ts < ? GROUP BY key, b " + "ON CONFLICT(key, ts) DO UPDATE SET mean = excluded.mean, min = excluded.min, " + "max = excluded.max, n = excluded.n", + (_floor(now_s, MINUTE_S),), + ) + quarters = await tx.execute( + "INSERT INTO measurement_15m (key, ts, mean, min, max, n, derived) " + "SELECT key, (ts / 900) * 900 AS b, SUM(mean * n) / SUM(n), MIN(min), MAX(max), " + "SUM(n), 0 FROM measurement_1m WHERE ts < ? GROUP BY key, b " + "ON CONFLICT(key, ts) DO UPDATE SET mean = excluded.mean, min = excluded.min, " + "max = excluded.max, n = excluded.n, derived = 0", + (_floor(now_s, QUARTER_S),), + ) + return DownsampleReport(minute_rows=minutes, quarter_rows=quarters) + + async def apply_retention(self, now: datetime) -> RetentionReport: + """Delete what is older than the retention of its table (cut-offs bucket-aligned).""" + now_s = _epoch(now) + raw_cut = _floor(now_s - int(RAW_KEEP.total_seconds()), MINUTE_S) + minute_cut = _floor(now_s - int(MINUTE_KEEP.total_seconds()), QUARTER_S) + forecast_cut = now_s - int(FORECAST_KEEP.total_seconds()) + price_cut = now_s - int(PRICE_KEEP.total_seconds()) + async with self._db.transaction() as tx: + raw = await tx.execute("DELETE FROM measurement WHERE ts < ?", (raw_cut,)) + minute = await tx.execute("DELETE FROM measurement_1m WHERE ts < ?", (minute_cut,)) + forecast = await tx.execute( + "DELETE FROM forecast WHERE slot_start < ?", (forecast_cut,) + ) + price = await tx.execute("DELETE FROM price WHERE slot_start < ?", (price_cut,)) + return RetentionReport(raw=raw, minute=minute, forecast=forecast, price=price) + + async def maintain(self, now: datetime) -> MaintenanceReport: + """The nightly job: downsample first (nothing is deleted before it is aggregated), + then retention, then ``VACUUM`` if enough of the file is free.""" + downsample = await self.downsample(now) + retention = await self.apply_retention(now) + vacuumed = await self._vacuum_if_worth_it() + size = await self.size_report() + report = MaintenanceReport( + downsample=downsample, + retention=retention, + vacuumed=vacuumed, + size_bytes=size.total_bytes, + ) + log.info( + "timeseries.maintained", + minute_rows=downsample.minute_rows, + quarter_rows=downsample.quarter_rows, + deleted=retention.total, + vacuumed=vacuumed, + size_bytes=report.size_bytes, + ) + return report + + async def _free_and_total(self) -> tuple[int, int]: + page_size = await self._pragma("page_size") + return ( + await self._pragma("freelist_count") * page_size, + await self._pragma("page_count") * page_size, + ) + + async def _pragma(self, name: str) -> int: + row = await self._db.fetchone(f"PRAGMA {name}") + return _int(row[0]) if row else 0 + + async def _vacuum_if_worth_it(self) -> bool: + free, total = await self._free_and_total() + if free < VACUUM_MIN_FREE_BYTES or total == 0 or free / total < VACUUM_MIN_FREE_FRACTION: + return False + await self._db.execute("VACUUM") + await self._db.fetchone("PRAGMA wal_checkpoint(TRUNCATE)") + return True + + async def size_report(self) -> TimeseriesSize: + """File, WAL and free bytes plus the row count per table.""" + path = Path(self._db.path) + free, _total = await self._free_and_total() + rows: dict[str, int] = {} + for table in ("measurement", "measurement_1m", "measurement_15m", "forecast", "price"): + row = await self._db.fetchone(f"SELECT COUNT(*) FROM {table}") + rows[table] = _int(row[0]) if row else 0 + return TimeseriesSize( + file_bytes=_size(path), + wal_bytes=_size(path.with_name(path.name + "-wal")), + free_bytes=free, + rows=rows, + ) + + +# --- row conversion ----------------------------------------------------------------- + + +def _size(path: Path) -> int: + try: + return path.stat().st_size + except OSError: + return 0 + + +def _ceil_epoch(moment: datetime) -> int: + seconds = moment.timestamp() + _epoch(moment) # validates awareness + return math.ceil(seconds) + + +def _bounds(window: TimeRange | None) -> tuple[int, int]: + if window is None: + return 0, 2**62 + return _epoch(window.start), _ceil_epoch(window.end) + + +def _quantile_series(rows: Sequence[Row]) -> QuantileSeries | None: + """Rebuild a series from slot rows; the most frequent step wins, gaps become ``None``.""" + if not rows: + return None + steps = Counter(_int(r[1]) for r in rows) + step_s = steps.most_common(1)[0][0] + if len(steps) > 1: + log.warning("timeseries.mixed_forecast_steps", steps=sorted(steps), used=step_s) + kept = [r for r in rows if _int(r[1]) == step_s] + units = Counter(str(r[2]) for r in kept) + unit = SeriesUnit(units.most_common(1)[0][0]) + kept = [r for r in kept if SeriesUnit(str(r[2])) is unit] + first = _int(kept[0][0]) + count = (_int(kept[-1][0]) - first) // step_s + 1 + grids: tuple[list[float | None], list[float | None], list[float | None]] = ( + [None] * count, + [None] * count, + [None] * count, + ) + for row in kept: + offset = _int(row[0]) - first + if offset % step_s: # off the grid of the chosen step: ignore the stray row + continue + index = offset // step_s + for column in range(3): + grids[column][index] = _opt_num(row[3 + column]) + start = _from_epoch(first) + step = timedelta(seconds=step_s) + p10, p50, p90 = (TimeSeries(start=start, step=step, values=tuple(g), unit=unit) for g in grids) + return QuantileSeries(p10=p10, p50=p50, p90=p90) + + +def _continues(segment: Sequence[Row], row: Row) -> bool: + last = segment[-1] + if _int(row[1]) != _int(last[1]) or str(row[2]) != str(last[2]): + return False + gap = _int(row[0]) - (_int(last[0]) + _int(last[1])) + return gap >= 0 and gap % _int(last[1]) == 0 + + +def _price_segment(source_id: str, segment: Sequence[Row]) -> PriceSeries: + step_s = _int(segment[0][1]) + first = _int(segment[0][0]) + count = (_int(segment[-1][0]) - first) // step_s + 1 + imports: list[float | None] = [None] * count + exports: list[float | None] = [None] * count + for row in segment: + index = (_int(row[0]) - first) // step_s + imports[index] = _opt_num(row[3]) + exports[index] = _opt_num(row[4]) + start = _from_epoch(first) + step = timedelta(seconds=step_s) + includes: PriceIncludes = "all_in" if str(segment[0][2]) == "all_in" else "spot_only" + export = ( + None + if all(v is None for v in exports) + else TimeSeries(start=start, step=step, values=tuple(exports), unit=SeriesUnit.PRICE) + ) + return PriceSeries( + source_id=source_id, + price_includes=includes, + import_price=TimeSeries( + start=start, step=step, values=tuple(imports), unit=SeriesUnit.PRICE + ), + export_price=export, + ) + + +__all__ = [ + "FORECAST_KEEP", + "MINUTE_KEEP", + "PRICE_KEEP", + "RAW_KEEP", + "TS_FILENAME", + "VACUUM_MIN_FREE_BYTES", + "VACUUM_MIN_FREE_FRACTION", + "DownsampleReport", + "ForecastKind", + "MaintenanceReport", + "MeasurementRow", + "Resolution", + "RetentionReport", + "TimeseriesSize", + "TimeseriesStore", + "open_timeseries", + "open_timeseries_or_recover", + "timeseries_migrations", +] diff --git a/suncore/storage/ts_migrations/0001_initial.sql b/suncore/storage/ts_migrations/0001_initial.sql new file mode 100644 index 0000000..f7dde6d --- /dev/null +++ b/suncore/storage/ts_migrations/0001_initial.sql @@ -0,0 +1,62 @@ +-- SPDX-License-Identifier: Apache-2.0 +-- timeseries.db (ADR-0004, Phase 4 WP3). All timestamps are integer seconds since the UNIX +-- epoch (UTC); a gap is the absence of a row, never a stored zero. + +-- Controller resolution, kept 48 h. +CREATE TABLE measurement ( + key TEXT NOT NULL, + ts INTEGER NOT NULL, + value REAL NOT NULL, + PRIMARY KEY (key, ts) +) WITHOUT ROWID; + +-- 1-minute aggregates (ts = start of the minute), kept 30 days. +CREATE TABLE measurement_1m ( + key TEXT NOT NULL, + ts INTEGER NOT NULL, + mean REAL NOT NULL, + min REAL NOT NULL, + max REAL NOT NULL, + n INTEGER NOT NULL CHECK (n > 0), + PRIMARY KEY (key, ts) +) WITHOUT ROWID; + +-- 15-minute aggregates (ts = start of the quarter hour), kept for ever. `derived` = 1 marks +-- rows rebuilt from Home Assistant's hourly statistics (a step-hold of the hourly mean). +CREATE TABLE measurement_15m ( + key TEXT NOT NULL, + ts INTEGER NOT NULL, + mean REAL NOT NULL, + min REAL NOT NULL, + max REAL NOT NULL, + n INTEGER NOT NULL CHECK (n > 0), + derived INTEGER NOT NULL DEFAULT 0 CHECK (derived IN (0, 1)), + PRIMARY KEY (key, ts) +) WITHOUT ROWID; + +-- Forecasts versioned by issue time. p10/p50/p90 are NULL where the value is unknown; +-- a price-like single value is stored in p50 (kind = 'price'). +CREATE TABLE forecast ( + source_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('pv', 'load', 'price')), + issued_at INTEGER NOT NULL, + slot_start INTEGER NOT NULL, + step_s INTEGER NOT NULL CHECK (step_s > 0), + unit TEXT NOT NULL, + p10 REAL, + p50 REAL, + p90 REAL, + PRIMARY KEY (source_id, kind, issued_at, slot_start) +) WITHOUT ROWID; +CREATE INDEX forecast_slot_idx ON forecast (source_id, kind, slot_start); + +-- Prices by slot; a later store of the same time span replaces the earlier rows. +CREATE TABLE price ( + source_id TEXT NOT NULL, + slot_start INTEGER NOT NULL, + step_s INTEGER NOT NULL CHECK (step_s > 0), + includes TEXT NOT NULL CHECK (includes IN ('spot_only', 'all_in')), + import_price REAL, + export_price REAL, + PRIMARY KEY (source_id, slot_start) +) WITHOUT ROWID; diff --git a/suncore/storage/ts_migrations/__init__.py b/suncore/storage/ts_migrations/__init__.py new file mode 100644 index 0000000..5044629 --- /dev/null +++ b/suncore/storage/ts_migrations/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Ordered SQL migrations of ``timeseries.db`` (same file format as ``storage.migrations``). + +``timeseries.db`` has its own ``PRAGMA user_version``; it is excluded from backups and can be +rebuilt, so a database that is newer than the code is left untouched and not opened. +""" diff --git a/tests/app/test_history.py b/tests/app/test_history.py new file mode 100644 index 0000000..937743f --- /dev/null +++ b/tests/app/test_history.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Combination of the site's statistics into measurement series (rebuild, Phase 4 WP3).""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from suncore.app.history import assemble_site_series, rebuild_history, site_statistic_specs +from suncore.core.measurements import ( + SITE_BATTERY_W, + SITE_GRID_W, + SITE_HOUSE_W, + SITE_PV_W, + SITE_SOC_PCT, +) +from suncore.core.series import SeriesUnit, TimeRange, TimeSeries +from suncore.core.site import BatteryConfig, EntityRef, GridConfig, PvArray, SiteConfig +from suncore.storage import open_timeseries +from suncore.storage.timeseries import TimeseriesStore + +T0 = datetime(2026, 10, 10, tzinfo=UTC) +Q = timedelta(minutes=15) + + +def ts(values: list[float | None], start: datetime = T0) -> TimeSeries: + return TimeSeries(start=start, step=Q, values=tuple(values), unit=SeriesUnit.W) + + +def ref(name: str, *, invert: bool = False) -> EntityRef: + return EntityRef(entity_id=f"sensor.{name}", invert_sign=invert) + + +def full_site(*, house: bool = False) -> SiteConfig: + return SiteConfig( + grid=GridConfig(power=ref("grid", invert=True)), + pv=[ + PvArray(name="A", power=ref("pv_a")), + PvArray(name="B", power=ref("pv_b")), + PvArray(name="C"), + ], + battery=BatteryConfig(power=ref("bat"), soc=ref("soc")), + house=ref("house") if house else None, + ) + + +def test_specs_cover_every_mapped_power_entity_and_the_soc() -> None: + specs = site_statistic_specs(full_site(house=True)) + by_key = {s.key: s for s in specs} + assert set(by_key) == {"grid", "pv0", "pv1", "battery", "soc", "house"} # array C has no sensor + assert by_key["grid"].invert is True + assert by_key["soc"].unit is SeriesUnit.PERCENT + assert all(s.kind == "mean" for s in specs) + + +def test_pair_grid_uses_two_statistics_and_subtracts() -> None: + site = SiteConfig( + grid=GridConfig(import_power=ref("imp"), export_power=ref("exp")), + pv=[PvArray(name="A", power=ref("pv"))], + ) + assert {s.key for s in site_statistic_specs(site)} == {"grid_import", "grid_export", "pv0"} + out = assemble_site_series( + site, + { + "grid_import": ts([500.0, 0.0]), + "grid_export": ts([0.0, 300.0]), + "pv0": ts([100.0, 900.0]), + }, + ) + assert out[SITE_GRID_W].values == (500.0, -300.0) + # no battery: house = pv + grid + assert out[SITE_HOUSE_W].values == (600.0, 600.0) + assert SITE_BATTERY_W not in out + assert SITE_SOC_PCT not in out + + +def test_house_is_derived_like_the_live_balance() -> None: + fetched = { + "grid": ts([200.0, -100.0]), + "pv0": ts([1000.0, 800.0]), + "pv1": ts([500.0, 200.0]), + "battery": ts([300.0, -50.0]), + "soc": ts([40.0, 41.0]), + } + out = assemble_site_series(full_site(), fetched) + assert out[SITE_PV_W].values == (1500.0, 1000.0) + # pv + grid - battery + assert out[SITE_HOUSE_W].values == (1400.0, 950.0) + assert out[SITE_SOC_PCT].values == (40.0, 41.0) + + +def test_a_configured_house_sensor_wins() -> None: + out = assemble_site_series( + full_site(house=True), + { + "grid": ts([1.0]), + "pv0": ts([1.0]), + "pv1": ts([1.0]), + "battery": ts([0.0]), + "house": ts([42.0]), + }, + ) + assert out[SITE_HOUSE_W].values == (42.0,) + + +def test_missing_parts_become_gaps_not_zeros() -> None: + fetched = { + "grid": ts([200.0, 300.0, 400.0]), + "pv0": ts([1000.0, None, 1000.0]), + "pv1": ts([500.0, 500.0, 500.0]), + "battery": ts([0.0, 0.0], start=T0 + Q), # shorter and later: only slots 1 and 2 overlap + } + out = assemble_site_series(full_site(), fetched) + assert out[SITE_PV_W].values == (1500.0, None, 1500.0) + assert out[SITE_HOUSE_W].values == (None, None, 1900.0) + # a missing array sensor makes the whole PV sum unknown + partial = assemble_site_series( + full_site(), {"grid": ts([1.0]), "pv0": ts([1.0]), "battery": ts([0.0])} + ) + assert SITE_PV_W not in partial + assert SITE_HOUSE_W not in partial + + +def test_no_pv_and_no_battery_house_is_the_grid() -> None: + site = SiteConfig(grid=GridConfig(power=ref("grid"))) + out = assemble_site_series(site, {"grid": ts([700.0])}) + assert out == {SITE_GRID_W: ts([700.0]), SITE_HOUSE_W: ts([700.0])} + + +async def test_rebuild_stores_derived_rows_and_reports_missing_ids(tmp_path) -> None: # type: ignore[no-untyped-def] + from suncore.ha.connection import HAConnection # noqa: PLC0415 + + class Stats: + async def statistics_during_period(self, ids, *_args, **_kw): # type: ignore[no-untyped-def] + hour = int(T0.timestamp() * 1000) + rows = { + "sensor.grid": [{"start": hour, "end": hour + 3_600_000, "mean": -400.0}], + "sensor.house": [{"start": hour, "end": hour + 3_600_000, "mean": 900.0}], + } + return {i: rows[i] for i in ids if i in rows} + + site = SiteConfig(grid=GridConfig(power=ref("grid", invert=True)), house=ref("house")) + async with await open_timeseries(tmp_path / "t.db") as db: + store = TimeseriesStore(db) + connection: HAConnection = Stats() # type: ignore[assignment] + result = await rebuild_history(connection, store, site, T0 + timedelta(hours=1), days=1) + assert result.stored == {SITE_GRID_W: 4, SITE_HOUSE_W: 4} + assert result.total == 8 + assert result.missing == () + grid = await store.read_series( + SITE_GRID_W, TimeRange(start=T0, end=T0 + timedelta(hours=1)) + ) + assert grid.values == (400.0,) * 4 # invert_sign applied + assert await db.fetchone("SELECT MIN(derived), MAX(derived) FROM measurement_15m") == (1, 1) + # the energy of the hour is kept: 900 W x 1 h + house = await store.read_series( + SITE_HOUSE_W, TimeRange(start=T0, end=T0 + timedelta(hours=1)) + ) + assert house.total() == pytest.approx(900.0) diff --git a/tests/app/test_timeseries_runtime.py b/tests/app/test_timeseries_runtime.py new file mode 100644 index 0000000..c25f024 --- /dev/null +++ b/tests/app/test_timeseries_runtime.py @@ -0,0 +1,525 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Wiring of ``timeseries.db`` into the runtime: recording, flushing, nightly job, rebuild.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator, Callable +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import pytest +import structlog.testing + +from suncore.app.control import ControlStack +from suncore.app.history import site_statistic_specs +from suncore.app.main import build_runtime +from suncore.app.recorder import MeasurementRecorder +from suncore.app.runtime import Runtime +from suncore.app.settings import Settings +from suncore.core.measurements import ( + SITE_HOUSE_W, + SITE_KEYS, + SITE_SOC_PCT, + device_power_key, +) +from suncore.core.series import TimeRange +from suncore.ha.connection import CommandError +from suncore.storage.timeseries import TS_FILENAME, TimeseriesStore +from tests.app.conftest import RecordingConnection, demo_settings + +T0 = datetime(2026, 6, 21, 0, 0, tzinfo=UTC) + + +def prod_settings(tmp_path: Path) -> Settings: + return Settings(demo_mode=False, data_dir=tmp_path / "data", static_dir=tmp_path / "no-dist") + + +class GatedSleep: + """Sleep that records the requested delays and returns only when released.""" + + def __init__(self) -> None: + self.requested: list[float] = [] + self._events: list[asyncio.Event] = [] + + async def __call__(self, delay: float) -> None: + self.requested.append(delay) + event = asyncio.Event() + self._events.append(event) + await event.wait() + + async def release(self, delay: float) -> None: + """Let the oldest pending sleep of ``delay`` return.""" + for event, requested in zip(self._events, self.requested, strict=False): + if requested == delay and not event.is_set(): + event.set() + break + for _ in range(10): + await asyncio.sleep(0) + + +async def until(predicate: Callable[[], bool], timeout_s: float = 5.0) -> None: + async with asyncio.timeout(timeout_s): + while not predicate(): # noqa: ASYNC110 - polling a plain predicate + await asyncio.sleep(0.01) + + +async def idle(runtime: Runtime) -> ControlStack: + """Stop the background tick of the current stack (after a possible rebuild settled).""" + await until(lambda: runtime.stack is not None and runtime._tz_learnt) + stack = runtime.stack + assert stack is not None + await stack._stop_tasks() + return stack + + +def store_of(runtime: Runtime) -> TimeseriesStore: + assert runtime.timeseries is not None + return runtime.timeseries + + +@pytest.fixture +async def gated(tmp_path: Path) -> AsyncIterator[tuple[Runtime, GatedSleep, RecordingConnection]]: + conn = RecordingConnection(speed=6000.0) + runtime = await build_runtime(demo_settings(tmp_path), env={}, connection=conn) + sleep = GatedSleep() + runtime.sleep = sleep + runtime.clock = lambda: T0 + timedelta(hours=10) + await runtime.start() + try: + yield runtime, sleep, conn + finally: + await runtime.stop() + + +# --- opening and closing -------------------------------------------------------------------- + + +async def test_production_path_creates_timeseries_next_to_suncore_db(tmp_path: Path) -> None: + settings = prod_settings(tmp_path) + runtime = await build_runtime(settings, env={}, connection=RecordingConnection(6000.0)) + try: + assert (settings.data_dir / "suncore.db").is_file() + assert (settings.data_dir / TS_FILENAME).is_file() + assert runtime.timeseries is not None + assert runtime.timeseries_db is not None + assert await runtime.timeseries_db.user_version() >= 1 + assert runtime.plans is not None + assert runtime.learned is not None + assert runtime.recorder.enabled + finally: + await runtime.stop() + assert not await runtime.timeseries_db.check() # closed with the runtime + assert not await runtime.db.check() + + +async def test_demo_keeps_timeseries_in_its_throw_away_dir(tmp_path: Path) -> None: + settings = demo_settings(tmp_path) + runtime = await build_runtime(settings, env={}, connection=RecordingConnection(6000.0)) + try: + assert runtime.timeseries_db is not None + assert not Path(runtime.timeseries_db.path).is_relative_to(settings.data_dir) + assert Path(runtime.timeseries_db.path).name == TS_FILENAME + finally: + await runtime.stop() + assert not settings.data_dir.exists() + + +async def test_too_new_timeseries_db_disables_recording_but_not_the_app(tmp_path: Path) -> None: + import sqlite3 # noqa: PLC0415 + + settings = prod_settings(tmp_path) + settings.data_dir.mkdir(parents=True) + con = sqlite3.connect(settings.data_dir / TS_FILENAME) + con.execute("PRAGMA user_version = 99") + con.close() + runtime = await build_runtime(settings, env={}, connection=RecordingConnection(6000.0)) + try: + assert runtime.timeseries is None + assert not runtime.recorder.enabled + await runtime.start() # control and API come up without it + stack = await idle(runtime) + assert await stack.tick() is not None + assert runtime.recorder.pending == 0 + assert await runtime.maintain_timeseries() is False + finally: + await runtime.stop() + con = sqlite3.connect(settings.data_dir / TS_FILENAME) + assert con.execute("PRAGMA user_version").fetchone() == (99,) + con.close() + + +# --- recording --------------------------------------------------------------------------------- + + +async def test_ticks_are_buffered_and_not_written_per_tick( + gated: tuple[Runtime, GatedSleep, RecordingConnection], monkeypatch: pytest.MonkeyPatch +) -> None: + runtime, _sleep, _conn = gated + store = store_of(runtime) + writes: list[int] = [] + original = store.append_measurements + + async def spy(rows: Any) -> int: + batch = list(rows) + writes.append(len(batch)) + return await original(batch) + + monkeypatch.setattr(store, "append_measurements", spy) + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + for _ in range(5): + await stack.tick() + assert writes == [] # nothing written per tick + pending = runtime.recorder.pending + assert pending >= 5 * len(SITE_KEYS) # five site values per tick (+ the devices) + assert await runtime.recorder.flush() == pending + assert writes == [pending] # one batch + assert runtime.recorder.pending == 0 + assert await runtime.recorder.flush() == 0 + assert writes == [pending] + + +async def test_recorded_keys_cover_site_and_devices( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, _sleep, _conn = gated + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + await stack.tick() + await runtime.recorder.flush() + rows = await store_of(runtime).database.fetchall("SELECT DISTINCT key FROM measurement") + keys = {str(r[0]) for r in rows} + assert set(SITE_KEYS) <= keys + assert {device_power_key(d) for d in stack.models} <= keys + soc = await store_of(runtime).database.fetchone( + "SELECT value FROM measurement WHERE key = ?", (SITE_SOC_PCT,) + ) + assert soc is not None + assert 0.0 <= float(str(soc[0])) <= 100.0 + + +async def test_flush_cadence_is_60_seconds_and_a_stop_flushes_the_rest( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, sleep, _conn = gated + assert runtime.flush_interval_s == 60.0 + await until(lambda: 60.0 in sleep.requested) # the flush loop is waiting + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + await stack.tick() + assert runtime.recorder.pending > 0 + await sleep.release(60.0) # one flush interval elapses + await until(lambda: runtime.recorder.pending == 0) + count = await store_of(runtime).database.fetchone("SELECT COUNT(*) FROM measurement") + assert count is not None + assert int(str(count[0])) > 0 + await stack.tick() + assert runtime.recorder.pending > 0 + pending = runtime.recorder.pending + written: list[int] = [] + store = store_of(runtime) + original = store.append_measurements + + async def spy(rows: Any) -> int: + batch = list(rows) + written.append(len(batch)) + return await original(batch) + + store.append_measurements = spy # type: ignore[method-assign] + await runtime.stop() # the shutdown writes what is left, before the database is closed + assert written == [pending] + assert runtime.recorder.pending == 0 + + +async def test_a_database_error_never_reaches_the_controller( + gated: tuple[Runtime, GatedSleep, RecordingConnection], monkeypatch: pytest.MonkeyPatch +) -> None: + runtime, _sleep, _conn = gated + store = store_of(runtime) + + async def broken(_rows: Any) -> int: + msg = "disk full" + raise OSError(msg) + + monkeypatch.setattr(store, "append_measurements", broken) + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + assert await stack.tick() is not None + pending = runtime.recorder.pending + with structlog.testing.capture_logs() as logs: + assert await runtime.recorder.flush() == 0 # logged, not raised + assert any(e["event"] == "timeseries.flush_failed" for e in logs) + assert runtime.recorder.pending == pending # kept for the next flush + assert await stack.tick() is not None # control is unaffected + monkeypatch.undo() + assert await runtime.recorder.flush() > 0 # the database is back: the kept rows are written + + +async def test_a_failing_sink_does_not_break_the_tick( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, _sleep, _conn = gated + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + + def boom(_ts: datetime, _values: Any) -> None: + msg = "sink exploded" + raise RuntimeError(msg) + + stack._on_measurements = boom + with structlog.testing.capture_logs() as logs: + assert await stack.tick() is not None + assert any(e["event"] == "control.measurement_sink_failed" for e in logs) + + +async def test_only_ok_quality_values_are_recorded( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, _sleep, conn = gated + stack = await idle(runtime) + seen: list[dict[str, float | None]] = [] + stack._on_measurements = lambda _ts, values: seen.append(dict(values)) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + await stack.tick() + assert seen[0][SITE_HOUSE_W] is not None + # a dead house sensor is a gap (None), never a zero + from suncore.ha.connection import EntityState, EntityUpdate # noqa: PLC0415 + + stack.cache.ingest( + EntityUpdate( + changed={ + "sensor.sim_house_power": EntityState( + "sensor.sim_house_power", + "unavailable", + {"unit_of_measurement": "W"}, + last_changed=1.0, + last_updated=1.0, + ) + } + ) + ) + await stack.tick() + assert seen[1][SITE_HOUSE_W] is None + assert conn is not None + + +# --- the recorder itself ----------------------------------------------------------------------- + + +async def test_recorder_skips_gaps_and_is_a_noop_without_store() -> None: + off = MeasurementRecorder(None) + off.record(T0, {"k": 1.0}) + assert off.pending == 0 + assert await off.flush() == 0 + + +async def test_recorder_buffer_is_bounded_and_drops_the_oldest(tmp_path: Path) -> None: + from suncore.storage import open_timeseries # noqa: PLC0415 + + async with await open_timeseries(tmp_path / "t.db") as db: + recorder = MeasurementRecorder(TimeseriesStore(db), max_buffered=5) + for i in range(8): + recorder.record(T0 + timedelta(seconds=i), {"k": float(i), "gap": None}) + assert recorder.pending == 5 + assert recorder.dropped == 3 + assert await recorder.flush() == 5 + store = TimeseriesStore(db) + rows = await store.read_raw("k", TimeRange(start=T0, end=T0 + timedelta(minutes=1))) + assert [v for _, v in rows] == [3.0, 4.0, 5.0, 6.0, 7.0] # the oldest three are gone + + +async def test_recorder_retries_a_failed_batch_within_the_bound(tmp_path: Path) -> None: + from suncore.storage import open_timeseries # noqa: PLC0415 + + async with await open_timeseries(tmp_path / "t.db") as db: + store = TimeseriesStore(db) + recorder = MeasurementRecorder(store, max_buffered=4) + recorder.record(T0, {"k": 1.0}) + recorder.record(T0 + timedelta(seconds=1), {"k": 2.0}) + original = store.append_measurements + fail = True + + async def flaky(rows: Any) -> int: + if fail: + msg = "locked" + raise OSError(msg) + return await original(rows) + + store.append_measurements = flaky # type: ignore[method-assign] + assert await recorder.flush() == 0 + recorder.record(T0 + timedelta(seconds=2), {"k": 3.0}) + recorder.record(T0 + timedelta(seconds=3), {"k": 4.0}) + recorder.record(T0 + timedelta(seconds=4), {"k": 5.0}) + assert recorder.pending == 4 + assert recorder.dropped == 1 + fail = False + assert await recorder.flush() == 4 + rows = await original.__self__.read_raw( # type: ignore[attr-defined] + "k", TimeRange(start=T0, end=T0 + timedelta(minutes=1)) + ) + assert [v for _, v in rows] == [2.0, 3.0, 4.0, 5.0] + + +# --- the nightly job --------------------------------------------------------------------------- + + +async def test_nightly_job_runs_at_start_then_waits_for_the_next_0330_local( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, sleep, _conn = gated + # the clock is 10:00 UTC = 12:00 CEST (Europe/Berlin, the demo region); next 03:30 CEST + # is 15.5 h away + expected = 15.5 * 3600 + await until(lambda: expected in sleep.requested) + assert runtime.seconds_until_maintenance() == expected + + +@pytest.mark.parametrize( + ("now", "expected_h"), + [ + (datetime(2026, 6, 21, 1, 29, tzinfo=UTC), 0.0 + 1 / 60), # 03:29 CEST -> 1 min + (datetime(2026, 6, 21, 1, 30, tzinfo=UTC), 24.0), # exactly 03:30: tomorrow's + (datetime(2026, 3, 28, 12, 0, tzinfo=UTC), 13.5), # DST night: 23 h day (03:30 CEST) + (datetime(2026, 10, 24, 12, 0, tzinfo=UTC), 14.5), # DST night: 25 h day (03:30 CET) + ], +) +async def test_seconds_until_maintenance_is_dst_safe( + gated: tuple[Runtime, GatedSleep, RecordingConnection], now: datetime, expected_h: float +) -> None: + runtime, _sleep, _conn = gated + runtime.clock = lambda: now + assert runtime.seconds_until_maintenance() == pytest.approx(expected_h * 3600, abs=1) + + +async def test_maintenance_downsamples_and_survives_errors( + gated: tuple[Runtime, GatedSleep, RecordingConnection], monkeypatch: pytest.MonkeyPatch +) -> None: + runtime, _sleep, _conn = gated + store = store_of(runtime) + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + await stack.tick() + later = datetime.now(UTC) + timedelta(days=5) # the stack stamps with the real clock + runtime.clock = lambda: later + assert await runtime.maintain_timeseries() is True + assert await store.database.fetchone("SELECT COUNT(*) FROM measurement") == (0,) + assert await store.database.fetchone("SELECT COUNT(*) FROM measurement_1m") != (0,) + + async def broken(_now: datetime) -> Any: + msg = "boom" + raise OSError(msg) + + monkeypatch.setattr(store, "maintain", broken) + with structlog.testing.capture_logs() as logs: + assert await runtime.maintain_timeseries() is False + assert any(e["event"] == "timeseries.maintenance_failed" for e in logs) + + +async def test_plan_snapshots_are_pruned_hourly( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, _sleep, _conn = gated + assert runtime.plans is not None + await runtime.plans.save( + {"x": 1}, source="t", schema_version=1, created_at=runtime.clock() - timedelta(days=30) + ) + await runtime.prune_plans() # the hourly loop may have been first: either way it is gone + assert await runtime.plans.count() == 0 + + +# --- the statistics rebuild --------------------------------------------------------------------- + + +@pytest.fixture +async def rebuild_env(tmp_path: Path) -> AsyncIterator[tuple[Runtime, RecordingConnection]]: + conn = RecordingConnection(speed=6000.0) + runtime = await build_runtime(demo_settings(tmp_path), env={}, connection=conn) + # the fake house's recorder history ends at the scenario start + runtime.clock = lambda: conn._scenario.start + try: + yield runtime, conn + finally: + await runtime.stop() + + +async def test_rebuild_runs_once_after_connect_when_the_15_minute_table_is_empty( + rebuild_env: tuple[Runtime, RecordingConnection], +) -> None: + runtime, conn = rebuild_env + store = store_of(runtime) + assert not await store.has_quarter_history() + await runtime.start() + await until(lambda: runtime._history_done) + assert await store.has_quarter_history() + keys = { + str(r[0]) for r in await store.database.fetchall("SELECT DISTINCT key FROM measurement_15m") + } + assert keys == set(SITE_KEYS) + derived = await store.database.fetchone("SELECT MIN(derived) FROM measurement_15m") + assert derived == (1,) + # 72 h of simulated history, all hours present + count = await store.database.fetchone( + "SELECT COUNT(*) FROM measurement_15m WHERE key = ?", (SITE_HOUSE_W,) + ) + assert count == (72 * 4,) + assert all(c["period"] == "hour" for c in conn.recorder.calls) + asked = {i for c in conn.recorder.calls for i in c["ids"]} + assert asked == {s.statistic_id for s in site_statistic_specs(runtime.site)} # type: ignore[arg-type] + calls = len(conn.recorder.calls) + runtime._start_history_rebuild() # a reconnect or a site edit does not rebuild again + await asyncio.sleep(0.05) + assert len(conn.recorder.calls) == calls + + +async def test_rebuild_is_skipped_when_history_exists( + rebuild_env: tuple[Runtime, RecordingConnection], +) -> None: + runtime, conn = rebuild_env + from suncore.core.series import SeriesUnit, TimeSeries # noqa: PLC0415 + + await store_of(runtime).store_quarter_hours( + SITE_HOUSE_W, + TimeSeries(start=T0, step=timedelta(minutes=15), values=(1.0,), unit=SeriesUnit.W), + ) + await runtime.start() + await until(lambda: runtime._history_done) + assert conn.recorder.calls == [] + + +async def test_rebuild_failure_is_logged_and_does_not_stop_the_app( + rebuild_env: tuple[Runtime, RecordingConnection], +) -> None: + runtime, conn = rebuild_env + conn.statistics_error = CommandError("unknown_command", "Unknown command.") + with structlog.testing.capture_logs() as logs: + await runtime.start() + await until(lambda: runtime._history_done) + warning = next(e for e in logs if e["event"] == "history.rebuild_failed") + assert warning["reason"] == "CommandError" + assert not await store_of(runtime).has_quarter_history() + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + assert await stack.tick() is not None # control is unaffected + + +async def test_rebuild_waits_for_the_site( + tmp_path: Path, +) -> None: + conn = RecordingConnection(speed=6000.0) + runtime = await build_runtime(demo_settings(tmp_path), env={}, connection=conn) + runtime.clock = lambda: conn._scenario.start + site = runtime.site + runtime.site = None + try: + await runtime.start() + await asyncio.sleep(0.1) + assert not runtime._history_done + assert conn.recorder.calls == [] + runtime.site = site + runtime._start_history_rebuild() + await until(lambda: runtime._history_done) + assert conn.recorder.calls + finally: + await runtime.stop() diff --git a/tests/ha/fakes.py b/tests/ha/fakes.py index d272b8e..114e758 100644 --- a/tests/ha/fakes.py +++ b/tests/ha/fakes.py @@ -112,6 +112,8 @@ def __init__(self, token: str = TOKEN) -> None: self.device_registry: list[dict[str, Any]] = [] self.area_registry: list[dict[str, Any]] = [] self.energy_prefs: dict[str, Any] | None = None # None = "No prefs" (not_found) + # recorder/statistics_during_period: statistic_id -> rows (ms epoch, as HA answers). + self.statistics: dict[str, list[dict[str, Any]]] = {} self.service_response: Callable[[dict[str, Any]], Any] = lambda msg: {"echo": msg} self.event_subscriptions: dict[web.WebSocketResponse, dict[int, str]] = {} self._server: TestServer | None = None @@ -228,7 +230,7 @@ async def _reply(self, ws: web.WebSocketResponse, *messages: dict[str, Any]) -> for message in messages: await ws.send_str(json.dumps(message)) - async def _on_message( # noqa: PLR0912 + async def _on_message( # noqa: PLR0912, PLR0915 self, ws: web.WebSocketResponse, msg: dict[str, Any] ) -> None: self.received.append(msg) @@ -273,6 +275,10 @@ def ok(result: Any = None) -> dict[str, Any]: ) else: await self._reply(ws, ok(self.energy_prefs)) + elif kind == "recorder/statistics_during_period": + wanted_ids = msg["statistic_ids"] + result = {i: self.statistics[i] for i in wanted_ids if self.statistics.get(i)} + await self._reply(ws, ok(result)) elif kind == "subscribe_events": self.event_subscriptions.setdefault(ws, {})[mid] = msg["event_type"] await self._reply(ws, ok()) diff --git a/tests/ha/test_action_rate_limit.py b/tests/ha/test_action_rate_limit.py new file mode 100644 index 0000000..10a3538 --- /dev/null +++ b/tests/ha/test_action_rate_limit.py @@ -0,0 +1,224 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Rate limit of ``call_action_response`` per (domain, service) (Phase 4 follow-up A3).""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator + +import pytest + +from suncore.ha.connection import ( + DEFAULT_ACTION_INTERVAL_S, + ERR_ACTION_RATE_LIMITED, + ActionNotAllowedError, + ActionRateLimitedError, + ActionRateLimiter, +) +from suncore.ha.transport import DirectTransport +from suncore.ha.websocket import WebSocketConnection +from tests.ha.fakes import TOKEN, FakeClock, FakeHA + + +def _limiter(clock: FakeClock, interval: float = 30.0) -> ActionRateLimiter: + return ActionRateLimiter(interval, clock=clock.monotonic, sleep=clock.sleep) + + +# --- the limiter --------------------------------------------------------------------- + + +def test_default_interval_is_30_seconds() -> None: + assert DEFAULT_ACTION_INTERVAL_S == 30.0 + + +def test_try_acquire_admits_once_per_interval() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert limiter.try_acquire("tibber.get_prices") + assert not limiter.try_acquire("tibber.get_prices") + assert limiter.retry_after("tibber.get_prices") == 30.0 + clock.now = 29.9 + assert not limiter.try_acquire("tibber.get_prices") + clock.now = 30.0 + assert limiter.try_acquire("tibber.get_prices") + assert limiter.retry_after("tibber.get_prices") == 30.0 + + +def test_a_refused_try_does_not_move_the_slot() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert limiter.try_acquire("a.b") + clock.now = 20.0 + assert not limiter.try_acquire("a.b") + assert limiter.retry_after("a.b") == 10.0 + + +def test_keys_are_independent() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert limiter.try_acquire("tibber.get_prices") + assert limiter.try_acquire("weather.get_forecasts") + assert not limiter.try_acquire("tibber.get_prices") + assert limiter.retry_after("never.used") == 0.0 + + +def test_a_caller_supplied_interval_applies_to_the_next_slot() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert limiter.try_acquire("a.b", 5.0) + assert limiter.retry_after("a.b") == 5.0 + clock.now = 5.0 + assert limiter.try_acquire("a.b", 0.0) + assert limiter.try_acquire("a.b", 0.0) # interval 0: never limited + + +def test_negative_intervals_are_rejected() -> None: + clock = FakeClock() + with pytest.raises(ValueError, match="default_interval_s"): + ActionRateLimiter(-1.0, clock=clock.monotonic, sleep=clock.sleep) + with pytest.raises(ValueError, match="interval_s"): + _limiter(clock).try_acquire("a.b", -1.0) + + +async def test_acquire_without_deadline_is_a_try() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert await limiter.acquire("a.b") + assert not await limiter.acquire("a.b") + assert clock.requested == [] # never slept + + +async def test_acquire_waits_for_a_slot_within_the_deadline() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert await limiter.acquire("a.b") + clock.now = 10.0 + task = asyncio.create_task(limiter.acquire("a.b", max_wait_s=25.0)) + await clock.settle() + assert clock.requested == [20.0] + assert not task.done() + await clock.advance(20.0) + assert await task is True + assert limiter.retry_after("a.b") == 30.0 + + +async def test_acquire_refuses_when_the_slot_is_beyond_the_deadline() -> None: + clock = FakeClock() + limiter = _limiter(clock) + assert await limiter.acquire("a.b") + assert not await limiter.acquire("a.b", max_wait_s=10.0) + assert clock.requested == [] + assert limiter.retry_after("a.b") == 30.0 # nothing was reserved by the refusal + + +async def test_concurrent_waiters_get_consecutive_slots() -> None: + clock = FakeClock() + limiter = _limiter(clock, 10.0) + assert await limiter.acquire("a.b") + first = asyncio.create_task(limiter.acquire("a.b", max_wait_s=100.0)) + second = asyncio.create_task(limiter.acquire("a.b", max_wait_s=100.0)) + third = asyncio.create_task(limiter.acquire("a.b", max_wait_s=15.0)) + await clock.settle() + assert sorted(clock.requested) == [10.0, 20.0] # two slots reserved in order + assert await third is False # its slot (30 s) is beyond its 15 s deadline + await clock.advance(20.0) + assert await first is True + assert await second is True + + +# --- on the connection --------------------------------------------------------------- + + +@pytest.fixture +async def ha() -> AsyncIterator[FakeHA]: + server = FakeHA() + await server.start() + yield server + await server.stop() + + +@pytest.fixture +def clock() -> FakeClock: + return FakeClock() + + +@pytest.fixture +async def conn(ha: FakeHA, clock: FakeClock) -> AsyncIterator[WebSocketConnection]: + connection = WebSocketConnection( + DirectTransport.from_base_url(ha.base_url, TOKEN), + clock=clock.monotonic, + sleep=clock.sleep, + rng=lambda: 0.5, + heartbeat_interval=777.0, + heartbeat_timeout=666.0, + resync_interval=555.0, + command_timeout=888.0, + connect_timeout=999.0, + ) + await asyncio.wait_for(connection.connect(), 5) + yield connection + await connection.close() + + +def _calls(ha: FakeHA) -> int: + return len([m for m in ha.received if m["type"] == "call_service"]) + + +async def test_second_call_within_the_interval_is_refused_not_sent( + conn: WebSocketConnection, ha: FakeHA, clock: FakeClock +) -> None: + ha.service_response = lambda _m: {"ok": True} + assert await conn.call_action_response("tibber", "get_prices") == {"ok": True} + with pytest.raises(ActionRateLimitedError) as info: + await conn.call_action_response("tibber", "get_prices") + assert info.value.code == "rate_limited" + assert info.value.action == "tibber.get_prices" + assert info.value.retry_after_s == pytest.approx(30.0) + assert _calls(ha) == 1 + clock.now += 30.0 + assert await conn.call_action_response("tibber", "get_prices") == {"ok": True} + assert _calls(ha) == 2 + + +async def test_other_actions_are_not_affected(conn: WebSocketConnection, ha: FakeHA) -> None: + ha.service_response = lambda _m: {} + await conn.call_action_response("tibber", "get_prices") + await conn.call_action_response("weather", "get_forecasts") + assert _calls(ha) == 2 + + +async def test_configured_interval_and_deadline( + conn: WebSocketConnection, ha: FakeHA, clock: FakeClock +) -> None: + ha.service_response = lambda _m: {} + await conn.call_action_response("tibber", "get_prices", min_interval_s=5.0) + with pytest.raises(ActionRateLimitedError) as info: + await conn.call_action_response("tibber", "get_prices", min_interval_s=5.0) + assert info.value.retry_after_s == pytest.approx(5.0) + waiter = asyncio.create_task( + conn.call_action_response("tibber", "get_prices", min_interval_s=5.0, max_wait_s=6.0) + ) + await clock.settle() + await clock.advance(5.0) + assert await waiter == {} + assert _calls(ha) == 2 + + +async def test_not_allowed_is_checked_before_the_limiter( + conn: WebSocketConnection, ha: FakeHA +) -> None: + with pytest.raises(ActionNotAllowedError): + await conn.call_action_response("switch", "turn_on") + with pytest.raises(ActionNotAllowedError): + await conn.call_action_response("switch", "turn_on") # not "rate limited" either + assert _calls(ha) == 0 + + +def test_the_refusal_has_a_catalogue_key() -> None: + import json # noqa: PLC0415 + from pathlib import Path # noqa: PLC0415 + + locales = Path(__file__).resolve().parents[2] / "frontend/src/lib/i18n/locales" + for name in ("en", "de"): + catalogue = json.loads((locales / f"{name}.json").read_text(encoding="utf-8")) + assert catalogue[ERR_ACTION_RATE_LIMITED] diff --git a/tests/ha/test_statistics.py b/tests/ha/test_statistics.py new file mode 100644 index 0000000..9ddff9a --- /dev/null +++ b/tests/ha/test_statistics.py @@ -0,0 +1,282 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Quarter-hour history from ``recorder/statistics_during_period`` (Phase 4 WP3).""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta, timezone +from typing import Any + +import pytest + +from suncore.core.series import SeriesUnit +from suncore.ha.connection import CommandError, NotConnectedError +from suncore.ha.statistics import ( + HOUR_MS, + STATISTICS_UNITS, + StatisticSpec, + fetch_quarter_history, + hourly_values, + quarter_hours, +) +from suncore.ha.transport import DirectTransport +from suncore.ha.websocket import WebSocketConnection +from tests.ha.fakes import TOKEN, FakeClock, FakeHA + +H0 = datetime(2026, 10, 10, 0, 0, tzinfo=UTC) +H0_MS = int(H0.timestamp() * 1000) + + +def _row(hour: int, **values: Any) -> dict[str, Any]: + start = H0_MS + hour * HOUR_MS + return {"start": start, "end": start + HOUR_MS, **values} + + +# --- parsing --------------------------------------------------------------------------- + + +def test_mean_rows_become_hourly_values_and_skip_null() -> None: + rows = [_row(0, mean=100.0), _row(1, mean=None), _row(2, mean=300.0)] + assert hourly_values(rows, "mean") == {H0_MS: 100.0, H0_MS + 2 * HOUR_MS: 300.0} + + +def test_rows_off_the_grid_or_not_hourly_are_skipped() -> None: + odd_start = {"start": H0_MS + 1000, "end": H0_MS + 1000 + HOUR_MS, "mean": 1.0} + five_minutes = {"start": H0_MS, "end": H0_MS + 300_000, "mean": 2.0} + no_start = {"mean": 3.0} + bad_type = {"start": "x", "end": 1, "mean": 4.0} + assert hourly_values([odd_start, five_minutes, no_start, bad_type], "mean") == {} + + +def test_energy_uses_change_else_the_difference_of_sum() -> None: + rows = [ + _row(0, change=500.0, sum=1000.0), + _row(1, change=None, sum=1800.0), # 800 from the sums of consecutive hours + _row(3, change=None, sum=3000.0), # previous row is not the previous hour: a gap + _row(4, change=-5.0, sum=3100.0), # negative: skipped + _row(5, change=0.0, sum=3100.0), + ] + assert hourly_values(rows, "energy") == { + H0_MS: 500.0, + H0_MS + HOUR_MS: 800.0, + H0_MS + 5 * HOUR_MS: 0.0, + } + + +def test_booleans_and_non_finite_numbers_are_not_values() -> None: + rows = [_row(0, mean=True), _row(1, mean=float("nan")), _row(2, mean=float("inf"))] + assert hourly_values(rows, "mean") == {} + + +# --- hourly -> quarter hours --------------------------------------------------------------- + + +def test_step_hold_preserves_energy_and_marks_missing_hours() -> None: + series = quarter_hours({H0_MS: 1000.0, H0_MS + 2 * HOUR_MS: 400.0}, unit=SeriesUnit.W) + assert series is not None + assert series.start == H0 + assert series.step == timedelta(minutes=15) + assert series.values == (1000.0,) * 4 + (None,) * 4 + (400.0,) * 4 + # energy of the covered hours: 1000 Wh + 400 Wh + assert sum(v * 0.25 for v in series.values if v is not None) == pytest.approx(1400.0) + + +def test_inversion_flips_the_sign_without_negative_zero() -> None: + series = quarter_hours({H0_MS: 250.0, H0_MS + HOUR_MS: 0.0}, unit=SeriesUnit.W, invert=True) + assert series is not None + assert series.values == (-250.0,) * 4 + (0.0,) * 4 + assert str(series.values[-1]) == "0.0" + + +def test_no_values_no_series_and_a_long_range_is_refused() -> None: + assert quarter_hours({}, unit=SeriesUnit.W) is None + with pytest.raises(ValueError, match="too long"): + quarter_hours({H0_MS: 1.0, H0_MS + 400 * 24 * HOUR_MS: 1.0}, unit=SeriesUnit.W) + + +# --- fetching through the fake connection (sim) ------------------------------------------------ + + +@pytest.fixture +async def sim() -> AsyncIterator[Any]: + from suncore.sim.fake_ha import FakeHAConnection # noqa: PLC0415 + from suncore.sim.scenario import load_scenario # noqa: PLC0415 + + scenario = load_scenario("sensor_dropout") + clock_time = scenario.start + + async def sleep(_delay: float) -> None: + await asyncio.Event().wait() # the feed never advances + + conn = FakeHAConnection(scenario, clock=lambda: clock_time, sleep=sleep, autorun=False) + await conn.connect() + yield conn + await conn.close() + + +async def test_fetch_from_the_sim_recorder_chunks_and_batches(sim: Any) -> None: + from suncore.sim.fake_ha import GRID_ID, HOUSE_ID, PV_ID # noqa: PLC0415 + + end = sim._scenario.start.replace(minute=0, second=0, microsecond=0) + specs = [ + StatisticSpec("house", HOUSE_ID), + StatisticSpec("pv", PV_ID), + StatisticSpec("grid", GRID_ID, invert=True), + StatisticSpec("ghost", "sensor.does_not_exist"), + ] + result = await fetch_quarter_history( + sim, specs, end - timedelta(hours=48), end, chunk_days=1, batch_size=2 + ) + assert set(result.series) == {"house", "pv", "grid"} + assert result.missing == ("sensor.does_not_exist",) + house = result.series["house"] + assert house.start == end - timedelta(hours=48) + assert len(house) == 48 * 4 + assert None not in house.values + # the sign of `grid` is inverted + plain = ( + await fetch_quarter_history( + sim, [StatisticSpec("g", GRID_ID)], end - timedelta(hours=48), end + ) + ).series["g"] + assert result.series["grid"].values == tuple(-v for v in plain.values if v is not None) + # 2 chunks (1 day each) x 2 batches (ids 2 + 2) + calls = sim.recorder.calls + assert len({c["start"] for c in calls}) == 2 + assert all(len(c["ids"]) <= 2 for c in calls) + assert all(c["period"] == "hour" for c in calls) + + +async def test_fetch_aligns_a_ragged_range_to_whole_hours(sim: Any) -> None: + from suncore.sim.fake_ha import HOUSE_ID # noqa: PLC0415 + + end = sim._scenario.start.replace(minute=0, second=0, microsecond=0) + await fetch_quarter_history( + sim, + [StatisticSpec("h", HOUSE_ID)], + end - timedelta(hours=5, minutes=20), + end + timedelta(minutes=10), + ) + call = sim.recorder.calls[0] + assert call["start"] == end - timedelta(hours=6) + assert call["end"] == end + timedelta(hours=1) + + +async def test_fetch_validates_its_arguments(sim: Any) -> None: + with pytest.raises(ValueError, match="before end"): + await fetch_quarter_history(sim, [], H0, H0) + with pytest.raises(ValueError, match="positive"): + await fetch_quarter_history(sim, [], H0, H0 + timedelta(hours=1), chunk_days=0) + + +async def test_sim_recorder_overrides_errors_and_connection_state(sim: Any) -> None: + sim.recorder.set_rows("sensor.custom", [_row(0, mean=5.0), _row(1, mean=6.0)]) + got = await sim.statistics_during_period(["sensor.custom"], H0, H0 + timedelta(hours=1)) + assert got == {"sensor.custom": [_row(0, mean=5.0)]} + sim.recorder.set_rows("sensor.custom", []) + assert await sim.statistics_during_period(["sensor.custom"], H0) == {} + assert await sim.statistics_during_period([], H0) == {} + with pytest.raises(CommandError): + await sim.statistics_during_period(["sensor.custom"], H0, period="5minute") + sim.statistics_error = CommandError("unauthorized", "no") + with pytest.raises(CommandError): + await sim.statistics_during_period(["sensor.custom"], H0) + await sim.close() + sim.statistics_error = None + with pytest.raises(NotConnectedError): + await sim.statistics_during_period(["sensor.custom"], H0) + + +# --- the WebSocket command --------------------------------------------------------------------- + + +@pytest.fixture +async def ha() -> AsyncIterator[FakeHA]: + server = FakeHA() + await server.start() + yield server + await server.stop() + + +@pytest.fixture +async def conn(ha: FakeHA) -> AsyncIterator[WebSocketConnection]: + clock = FakeClock() + connection = WebSocketConnection( + DirectTransport.from_base_url(ha.base_url, TOKEN), + clock=clock.monotonic, + sleep=clock.sleep, + rng=lambda: 0.5, + heartbeat_interval=777.0, + heartbeat_timeout=666.0, + resync_interval=555.0, + command_timeout=888.0, + connect_timeout=999.0, + ) + await asyncio.wait_for(connection.connect(), 5) + yield connection + await connection.close() + + +async def test_command_shape_offsets_units_and_absent_ids( + conn: WebSocketConnection, ha: FakeHA +) -> None: + ha.statistics["sensor.house"] = [_row(0, mean=100.0), _row(1, mean=200.0)] + berlin = timezone(timedelta(hours=2)) + got = await conn.statistics_during_period( + ["sensor.house", "sensor.missing"], + datetime(2026, 10, 10, 2, 0, tzinfo=berlin), + datetime(2026, 10, 10, 4, 0, tzinfo=berlin), + period="hour", + types=["mean"], + units=STATISTICS_UNITS, + ) + assert got == {"sensor.house": [_row(0, mean=100.0), _row(1, mean=200.0)]} # missing id absent + sent = next(m for m in ha.received if m["type"] == "recorder/statistics_during_period") + assert sent["start_time"] == "2026-10-10T02:00:00+02:00" # offset kept: never HA-local naive + assert sent["end_time"] == "2026-10-10T04:00:00+02:00" + assert sent["period"] == "hour" + assert sent["types"] == ["mean"] + assert sent["units"] == {"power": "W", "energy": "Wh"} + assert sent["statistic_ids"] == ["sensor.house", "sensor.missing"] + + +async def test_command_optional_fields_validation_and_errors( + conn: WebSocketConnection, ha: FakeHA +) -> None: + await conn.statistics_during_period(["sensor.a"], H0) + sent = next(m for m in ha.received if m["type"] == "recorder/statistics_during_period") + assert "end_time" not in sent + assert "types" not in sent + assert "units" not in sent + assert await conn.statistics_during_period([], H0) == {} + with pytest.raises(ValueError, match="period"): + await conn.statistics_during_period(["sensor.a"], H0, period="minute") + with pytest.raises(ValueError, match="timezone-aware"): + await conn.statistics_during_period(["sensor.a"], datetime(2026, 1, 1)) # noqa: DTZ001 + ha.fail_commands["recorder/statistics_during_period"] = ("unknown_command", "Unknown command.") + with pytest.raises(CommandError): + await conn.statistics_during_period(["sensor.a"], H0) + + +async def test_fetch_through_the_websocket_with_energy_and_missing_ids( + conn: WebSocketConnection, ha: FakeHA +) -> None: + ha.statistics["sensor.house_power"] = [_row(h, mean=100.0 * (h + 1)) for h in range(3)] + ha.statistics["sensor.pv_energy"] = [ + _row(0, change=400.0, sum=10_000.0), + _row(1, change=None, sum=10_900.0), + ] + specs = [ + StatisticSpec("house", "sensor.house_power"), + StatisticSpec("pv", "sensor.pv_energy", kind="energy"), + StatisticSpec("gone", "sensor.gone"), + ] + result = await fetch_quarter_history(conn, specs, H0, H0 + timedelta(hours=3)) + assert result.series["house"].values == (100.0,) * 4 + (200.0,) * 4 + (300.0,) * 4 + assert result.series["pv"].values == (400.0,) * 4 + (900.0,) * 4 + assert result.missing == ("sensor.gone",) + types = { + tuple(m["types"]) for m in ha.received if m["type"] == "recorder/statistics_during_period" + } + assert types == {("mean",), ("change", "sum")} diff --git a/tests/safety/test_boundary.py b/tests/safety/test_boundary.py index 90b0947..49dacf5 100644 --- a/tests/safety/test_boundary.py +++ b/tests/safety/test_boundary.py @@ -30,8 +30,9 @@ operating modes only through the injected ``ModeProvider`` (Phase 3 WP3) and the runtime only through the injected ``RuntimeStore``. 7. ``call_action_response`` (read-only action with ``return_response``) is defined and used - only in ``suncore.ha`` (implementation), ``suncore.sim`` (fake) and ``suncore.app`` - (wiring of the tariff/forecast adapters later); ``READ_ONLY_ACTIONS`` is the fixed allowlist. + only in ``suncore.ha`` (implementation, rate limited per action), ``suncore.sim`` (fake) and + the one adapter wiring module ``suncore.app.adapters`` (Phase 4); no other part of + ``suncore.app`` may use it. ``READ_ONLY_ACTIONS`` is the fixed allowlist. The scan is robust to packages that are still empty (``suncore/ha`` until WP3) because it only looks at the files that exist. @@ -63,7 +64,7 @@ API_FORBIDDEN = (SAFETY, HA, "suncore.devices") WRITE_CAPABILITY_NAMES = frozenset({"ServiceCaller", "ServiceCallerProvider"}) SAFETY_FORBIDDEN_IMPORTS = ("suncore.core.config", "suncore.storage") -ACTION_RESPONSE_OWNERS = ("suncore.ha", "suncore.sim") +ACTION_RESPONSE_OWNERS = ("suncore.ha", "suncore.sim", "suncore.app.adapters") def module_name(path: Path) -> str: @@ -325,12 +326,25 @@ def test_safety_does_not_read_config_or_storage(path: Path) -> None: @pytest.mark.parametrize("path", SOURCES, ids=lambda p: str(p.relative_to(ROOT))) def test_call_action_response_is_confined(path: Path) -> None: - """Rule 7: the response-returning call lives in ha (+ the fake); app joins later.""" + """Rule 7: the response-returning call lives in ha (+ the fake) and app.adapters only.""" module = module_name(path) if any(in_package(module, owner) for owner in ACTION_RESPONSE_OWNERS): return offenders = fmt(path, attribute_refs(parse(path), "call_action_response")) - assert not offenders, f"call_action_response referenced outside ha/sim: {offenders}" + assert not offenders, ( + f"call_action_response referenced outside ha/sim/app.adapters: {offenders}" + ) + + +def test_action_response_owner_is_the_adapter_module_not_the_whole_app() -> None: + """Rule 7 admits ``suncore.app.adapters`` only: its siblings in ``suncore.app`` stay out.""" + assert "suncore.app" not in ACTION_RESPONSE_OWNERS + assert (PACKAGE / "app" / "adapters.py").is_file() + for owner in ACTION_RESPONSE_OWNERS: + assert not in_package("suncore.app.control", owner) + assert not in_package("suncore.app.runtime", owner) + assert not in_package("suncore.app.adapters_extra", owner) + assert in_package("suncore.app.adapters", "suncore.app.adapters") def test_read_only_action_allowlist_is_fixed() -> None: diff --git a/tests/storage/test_config_sections.py b/tests/storage/test_config_sections.py index 132c239..bd63283 100644 --- a/tests/storage/test_config_sections.py +++ b/tests/storage/test_config_sections.py @@ -108,7 +108,7 @@ async def test_v1_export_file_imports(db: Database) -> None: assert site.grid.is_pair assert [a.power for a in site.pv] == [EntityRef(entity_id="sensor.pv_total")] exported = await store.export_json() - assert exported["db_schema_version"] == latest_version() == 3 + assert exported["db_schema_version"] == latest_version() == 4 assert exported["sections"]["site"]["schema_version"] == 2 diff --git a/tests/storage/test_db_cancellation.py b/tests/storage/test_db_cancellation.py new file mode 100644 index 0000000..c3adf25 --- /dev/null +++ b/tests/storage/test_db_cancellation.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A cancelled writer must never leave a transaction open on the shared connection.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from suncore.storage import Database + + +@pytest.mark.parametrize("turns", range(8)) +async def test_cancelling_a_transaction_leaves_the_database_usable( + db: Database, turns: int +) -> None: + await db.execute("CREATE TABLE t (x INTEGER)") + rows = [(i,) for i in range(5000)] + task = asyncio.create_task(db.executemany("INSERT INTO t VALUES (?)", rows)) + for _ in range(turns): + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + # the next writer starts a transaction without "within a transaction" and sees either + # none or all of the cancelled batch (it was rolled back or committed as a whole) + await db.executemany("INSERT INTO t VALUES (?)", [(-1,)]) + row = await db.fetchone("SELECT COUNT(*) FROM t") + assert row in {(1,), (5001,)} + + +async def test_cancelled_while_waiting_for_the_lock_is_harmless(db: Database) -> None: + await db.execute("CREATE TABLE t (x INTEGER)") + async with db.transaction() as tx: + waiter = asyncio.create_task(db.executemany("INSERT INTO t VALUES (?)", [(1,)])) + await asyncio.sleep(0) + waiter.cancel() + await tx.execute("INSERT INTO t VALUES (7)") + with pytest.raises(asyncio.CancelledError): + await waiter + assert await db.fetchone("SELECT COUNT(*) FROM t") == (1,) diff --git a/tests/storage/test_plans_learned.py b/tests/storage/test_plans_learned.py new file mode 100644 index 0000000..722d41b --- /dev/null +++ b/tests/storage/test_plans_learned.py @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Schema 4 of ``suncore.db``: plan snapshots and learned parameters.""" + +from __future__ import annotations + +import sqlite3 +import zlib +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import aiosqlite +import pytest + +from suncore.storage import ( + Database, + LearnedParamStore, + PlanSnapshotStore, + SchemaTooNewError, +) +from suncore.storage.learned import NS_BLEND_WEIGHTS, NS_FORECAST_BIAS, NS_ML_STATE +from suncore.storage.migrations import latest_version, load_migrations, run_migrations +from suncore.storage.plans import decode_payload, encode_payload + +NOW = datetime(2026, 10, 17, 12, 0, tzinfo=UTC) + + +# --- migration 0004 --------------------------------------------------------------------- + + +async def test_v3_database_upgrades_to_v4_and_keeps_data(tmp_path: Path) -> None: + path = tmp_path / "v3.db" + async with aiosqlite.connect(path, isolation_level=None) as conn: + assert await run_migrations(conn, load_migrations()[:3]) == 3 + await conn.execute("INSERT INTO learned VALUES ('k', '1', 't')") + await conn.execute("INSERT INTO safety_state VALUES ('emergency_stop', '0', 't')") + async with await Database.open(path) as db: + assert await db.user_version() == latest_version() == 4 + assert await db.fetchone("SELECT value FROM learned") == ("1",) + assert await db.fetchone("SELECT value FROM safety_state") == ("0",) + tables = {r[0] for r in await db.fetchall("SELECT name FROM sqlite_master")} + assert {"plan_snapshot", "learned_param"} <= tables + + +async def test_fresh_database_has_schema_4(tmp_path: Path) -> None: + async with await Database.open(tmp_path / "s.db") as db: + assert await db.user_version() == 4 + + +async def test_reader_of_schema_3_refuses_schema_4(tmp_path: Path) -> None: + path = tmp_path / "v4.db" + async with await Database.open(path): + pass + async with aiosqlite.connect(path, isolation_level=None) as conn: + with pytest.raises(SchemaTooNewError) as info: + await run_migrations(conn, load_migrations()[:3]) + assert (info.value.db_version, info.value.code_version) == (4, 3) + con = sqlite3.connect(path) + assert con.execute("PRAGMA user_version").fetchone() == (4,) + con.close() + + +# --- plan snapshots ----------------------------------------------------------------------- + + +@pytest.fixture +def plans(db: Database) -> PlanSnapshotStore: + return PlanSnapshotStore(db, clock=lambda: NOW) + + +async def test_snapshot_round_trip_and_compression(db: Database, plans: PlanSnapshotStore) -> None: + payload = {"intervals": [{"t": i, "w": i * 1.5} for i in range(500)], "name": "plän"} + snapshot_id = await plans.save(payload, source="planner", schema_version=2) + got = await plans.get(snapshot_id) + assert got is not None + assert got.payload == payload + assert (got.source, got.schema_version, got.created_at) == ("planner", 2, NOW) + row = await db.fetchone("SELECT length(payload) FROM plan_snapshot") + assert row is not None + assert int(str(row[0])) < len(str(payload)) / 3 # compressed + assert await plans.get(snapshot_id + 1) is None + + +async def test_payload_codec_is_strict() -> None: + assert decode_payload(encode_payload({"a": 1})) == {"a": 1} + with pytest.raises(ValueError, match="not an object"): + decode_payload(zlib.compress(b"[1, 2]")) + with pytest.raises(ValueError, match="unreadable"): + decode_payload(b"garbage") + with pytest.raises(ValueError, match="unreadable"): + decode_payload(zlib.compress(b"{not json")) + with pytest.raises(ValueError): # noqa: PT011 - json refuses NaN with allow_nan=False + encode_payload({"x": float("nan")}) + + +async def test_latest_headers_and_unreadable_rows(db: Database, plans: PlanSnapshotStore) -> None: + first = await plans.save({"n": 1}, source="a", schema_version=1, created_at=NOW) + second = await plans.save({"n": 2}, source="b", schema_version=2, created_at=NOW) + await db.execute( + "INSERT INTO plan_snapshot (created_at, source, schema_version, payload) " + "VALUES (?, 'bad', 1, ?)", + (NOW.isoformat(), b"not zlib"), + ) + latest = await plans.latest() + assert latest is not None + assert latest.id == second # the unreadable newest row is skipped + headers = await plans.headers() + assert [h.source for h in headers] == ["bad", "b", "a"] + assert [h.id for h in await plans.headers(limit=1)] == [headers[0].id] + assert await plans.count() == 3 + assert first < second + + +async def test_latest_is_none_without_readable_snapshots(plans: PlanSnapshotStore) -> None: + assert await plans.latest() is None + + +async def test_prune_keeps_at_most_n_and_14_days(plans: PlanSnapshotStore) -> None: + for i in range(10): + await plans.save( + {"i": i}, source="p", schema_version=1, created_at=NOW - timedelta(hours=i) + ) + assert await plans.prune(NOW, keep_last=4) == 6 + assert [h.id for h in await plans.headers()] == [10, 9, 8, 7] + await plans.save({"old": 1}, source="p", schema_version=1, created_at=NOW - timedelta(days=20)) + # the newest id is the old one: the age rule removes it, the others stay + assert await plans.prune(NOW) == 1 + assert await plans.count() == 4 + + +async def test_prune_defaults_are_200_and_14_days(plans: PlanSnapshotStore) -> None: + from suncore.storage.plans import KEEP_DAYS, KEEP_LAST # noqa: PLC0415 + + assert (KEEP_LAST, KEEP_DAYS) == (200, 14) + assert await plans.prune(NOW) == 0 + + +async def test_prune_by_age(plans: PlanSnapshotStore) -> None: + await plans.save({"a": 1}, source="p", schema_version=1, created_at=NOW - timedelta(days=15)) + await plans.save({"b": 1}, source="p", schema_version=1, created_at=NOW - timedelta(days=13)) + assert await plans.prune(NOW) == 1 + remaining = await plans.latest() + assert remaining is not None + assert remaining.payload == {"b": 1} + + +# --- learned parameters -------------------------------------------------------------------- + + +@pytest.fixture +def learned(db: Database) -> LearnedParamStore: + return LearnedParamStore(db, clock=lambda: NOW) + + +async def test_learned_round_trip_and_replace(learned: LearnedParamStore) -> None: + await learned.put(NS_FORECAST_BIAS, "solcast", {"hour_12": 1.05}, schema_version=2) + got = await learned.get(NS_FORECAST_BIAS, "solcast") + assert got is not None + assert (got.payload, got.schema_version, got.updated_at) == ({"hour_12": 1.05}, 2, NOW) + await learned.put(NS_FORECAST_BIAS, "solcast", {"hour_12": 1.1}) + replaced = await learned.get(NS_FORECAST_BIAS, "solcast") + assert replaced is not None + assert (replaced.payload, replaced.schema_version) == ({"hour_12": 1.1}, 1) + assert await learned.get(NS_FORECAST_BIAS, "other") is None + + +async def test_learned_namespaces_are_separate(learned: LearnedParamStore) -> None: + await learned.put(NS_FORECAST_BIAS, "b", {"x": 1}) + await learned.put(NS_FORECAST_BIAS, "a", {"x": 2}) + await learned.put(NS_BLEND_WEIGHTS, "a", {"w": 0.5}) + listed = await learned.in_namespace(NS_FORECAST_BIAS) + assert [p.key for p in listed] == ["a", "b"] + assert await learned.in_namespace(NS_ML_STATE) == [] + assert await learned.delete(NS_FORECAST_BIAS, "a") is True + assert await learned.delete(NS_FORECAST_BIAS, "a") is False + assert await learned.delete_namespace(NS_FORECAST_BIAS) == 1 + assert len(await learned.in_namespace(NS_BLEND_WEIGHTS)) == 1 + + +async def test_learned_unreadable_payload_is_missing( + db: Database, learned: LearnedParamStore +) -> None: + await db.execute( + "INSERT INTO learned_param VALUES ('net_error', 'list', '[1, 2]', 1, ?)", (NOW.isoformat(),) + ) + await learned.put("net_error", "ok", {"q": 1}) + assert await learned.get("net_error", "list") is None + assert [p.key for p in await learned.in_namespace("net_error")] == ["ok"] + + +async def test_learned_rejects_bad_names_and_non_json( + db: Database, learned: LearnedParamStore +) -> None: + with pytest.raises(ValueError, match="namespace"): + await learned.put("Bad Name", "k", {}) + with pytest.raises(ValueError, match="key"): + await learned.put(NS_ML_STATE, "", {}) + with pytest.raises(ValueError, match="key"): + await learned.get(NS_ML_STATE, "x" * 201) + with pytest.raises(ValueError, match="namespace"): + await learned.in_namespace("../x") + with pytest.raises(ValueError, match=r"NaN|Out of range"): + await learned.put(NS_ML_STATE, "k", {"x": float("nan")}) + assert await db.fetchone("SELECT COUNT(*) FROM learned_param") == (0,) diff --git a/tests/storage/test_runtime_store.py b/tests/storage/test_runtime_store.py index eb8f0a0..06d0a38 100644 --- a/tests/storage/test_runtime_store.py +++ b/tests/storage/test_runtime_store.py @@ -44,7 +44,7 @@ async def test_v2_database_migrates_to_v3_and_keeps_data(tmp_path: Path) -> None assert await run_migrations(conn, migrations) == 2 await conn.execute("INSERT INTO learned VALUES ('k', '1', 't')") async with await Database.open(path) as db: - assert await db.user_version() == latest_version() == 3 + assert await db.user_version() == latest_version() == 4 assert await db.fetchone("SELECT value FROM learned") == ("1",) tables = {r[0] for r in await db.fetchall("SELECT name FROM sqlite_master")} assert {"device_runtime", "safety_state", "power_model_state"} <= tables @@ -52,8 +52,8 @@ async def test_v2_database_migrates_to_v3_and_keeps_data(tmp_path: Path) -> None async def test_reader_expecting_v2_refuses_v3(tmp_path: Path) -> None: path = tmp_path / "v3.db" - async with await Database.open(path): - pass + async with aiosqlite.connect(path, isolation_level=None) as conn: + assert await run_migrations(conn, load_migrations()[:3]) == 3 async with aiosqlite.connect(path, isolation_level=None) as conn: with pytest.raises(SchemaTooNewError) as info: await run_migrations(conn, load_migrations()[:2]) diff --git a/tests/storage/test_timeseries.py b/tests/storage/test_timeseries.py new file mode 100644 index 0000000..de99f9e --- /dev/null +++ b/tests/storage/test_timeseries.py @@ -0,0 +1,504 @@ +# SPDX-License-Identifier: Apache-2.0 +"""``timeseries.db``: migrations, repository round trips, downsampling and retention.""" + +from __future__ import annotations + +import sqlite3 +from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta +from pathlib import Path +from zoneinfo import ZoneInfo + +import pytest + +from suncore.core.measurements import SITE_GRID_W, SITE_SOC_PCT, device_power_key +from suncore.core.series import ( + PriceSeries, + QuantileSeries, + SeriesUnit, + TimeRange, + TimeSeries, + floor_to_step, +) +from suncore.storage import Database, SchemaTooNewError +from suncore.storage.migrations import latest_version +from suncore.storage.timeseries import ( + MINUTE_KEEP, + RAW_KEEP, + TimeseriesStore, + open_timeseries, + open_timeseries_or_recover, + timeseries_migrations, +) + +T0 = datetime(2026, 10, 1, 0, 0, tzinfo=UTC) +Q = timedelta(minutes=15) + + +@pytest.fixture +async def tsdb(tmp_path: Path) -> AsyncIterator[Database]: + db = await open_timeseries(tmp_path / "timeseries.db") + yield db + await db.close() + + +@pytest.fixture +def store(tsdb: Database) -> TimeseriesStore: + return TimeseriesStore(tsdb) + + +def _series(values: list[float | None], start: datetime = T0, step: timedelta = Q) -> TimeSeries: + return TimeSeries(start=start, step=step, values=tuple(values), unit=SeriesUnit.W) + + +def _quantiles( + p10: list[float | None], p50: list[float | None], p90: list[float | None], start: datetime = T0 +) -> QuantileSeries: + return QuantileSeries(p10=_series(p10, start), p50=_series(p50, start), p90=_series(p90, start)) + + +# --- migrations ----------------------------------------------------------------------- + + +async def test_fresh_timeseries_db_is_migrated_wal(tmp_path: Path) -> None: + async with await open_timeseries(tmp_path / "t.db") as db: + assert await db.user_version() == len(timeseries_migrations()) >= 1 + assert await db.fetchone("PRAGMA journal_mode") == ("wal",) + tables = {r[0] for r in await db.fetchall("SELECT name FROM sqlite_master")} + assert {"measurement", "measurement_1m", "measurement_15m", "forecast", "price"} <= tables + + +async def test_timeseries_migrations_are_independent_of_suncore_db(tmp_path: Path) -> None: + async with await open_timeseries(tmp_path / "t.db") as db: + tables = {r[0] for r in await db.fetchall("SELECT name FROM sqlite_master")} + assert "config_doc" not in tables + assert "plan_snapshot" not in tables + assert latest_version() >= 4 # suncore.db has its own, longer history + + +async def test_reopen_keeps_data(tmp_path: Path) -> None: + path = tmp_path / "t.db" + async with await open_timeseries(path) as db: + await TimeseriesStore(db).append_measurements([("k", T0, 1.0)]) + async with await open_timeseries(path) as db: + assert await TimeseriesStore(db).read_raw("k", TimeRange(start=T0, end=T0 + Q)) == [ + (T0, 1.0) + ] + + +async def test_newer_timeseries_db_is_refused_and_untouched(tmp_path: Path) -> None: + path = tmp_path / "t.db" + con = sqlite3.connect(path) + con.execute("PRAGMA user_version = 99") + con.close() + with pytest.raises(SchemaTooNewError): + await open_timeseries(path) + assert await open_timeseries_or_recover(path) is None + con = sqlite3.connect(path) + assert con.execute("PRAGMA user_version").fetchone() == (99,) + con.close() + assert not (tmp_path / "t.db.corrupt").exists() + + +async def test_corrupt_timeseries_db_is_moved_aside_and_recreated(tmp_path: Path) -> None: + path = tmp_path / "t.db" + path.write_bytes(b"this is not a database" * 100) + db = await open_timeseries_or_recover(path) + assert db is not None + try: + assert await db.user_version() >= 1 + finally: + await db.close() + assert (tmp_path / "t.db.corrupt").read_bytes().startswith(b"this is not") + + +async def test_unopenable_path_gives_none(tmp_path: Path) -> None: + assert await open_timeseries_or_recover(tmp_path / "missing" / "dir" / "t.db") is None + + +# --- measurements --------------------------------------------------------------------- + + +async def test_append_skips_gaps_and_replaces_duplicates(store: TimeseriesStore) -> None: + written = await store.append_measurements( + [ + ("k", T0, 1.0), + ("k", T0 + timedelta(seconds=5), None), + ("k", T0 + timedelta(seconds=10), float("nan")), + ("k", T0 + timedelta(seconds=15), float("inf")), + ("k", T0 + timedelta(seconds=20), 2.0), + ("k", T0 + timedelta(seconds=20, milliseconds=400), 3.0), # same second: replaces + ] + ) + assert written == 3 + rows = await store.read_raw("k", TimeRange(start=T0, end=T0 + timedelta(minutes=1))) + assert rows == [(T0, 1.0), (T0 + timedelta(seconds=20), 3.0)] + + +async def test_naive_timestamps_are_rejected(store: TimeseriesStore) -> None: + with pytest.raises(ValueError, match="timezone-aware"): + await store.append_measurements([("k", datetime(2026, 1, 1), 1.0)]) # noqa: DTZ001 + + +async def test_downsample_is_sample_weighted_and_energy_preserving(store: TimeseriesStore) -> None: + # 5 s ticks for 30 minutes; a ramp, so every minute has a different mean + rows = [("k", T0 + timedelta(seconds=5 * i), float(i % 97)) for i in range(360)] + await store.append_measurements(rows) + now = T0 + timedelta(minutes=45) + report = await store.downsample(now) + assert report.minute_rows == 30 + assert report.quarter_rows == 2 + raw_mean = sum(v for _, _, v in rows[:180]) / 180 + q = await store.read_series( + "k", TimeRange(start=T0, end=T0 + timedelta(minutes=30)), "15m", include_recent=False + ) + assert q.values[0] == pytest.approx(raw_mean) + # energy of the first quarter hour: mean W x 0.25 h equals the sum of the samples x 5 s + sample_wh = sum(v for _, _, v in rows[:180]) * 5 / 3600 + assert q.values[0] * 0.25 == pytest.approx(sample_wh) + row = await store.database.fetchone( + "SELECT min, max, n FROM measurement_15m WHERE ts = ?", (int(T0.timestamp()),) + ) + assert row == (0.0, 96.0, 180) + + +async def test_downsample_is_idempotent(store: TimeseriesStore) -> None: + await store.append_measurements( + [("k", T0 + timedelta(seconds=10 * i), float(i)) for i in range(200)] + ) + now = T0 + timedelta(hours=1) + await store.downsample(now) + first = await store.database.fetchall("SELECT * FROM measurement_15m ORDER BY key, ts") + first_1m = await store.database.fetchall("SELECT * FROM measurement_1m ORDER BY key, ts") + await store.downsample(now) + await store.downsample(now + timedelta(hours=1)) + assert await store.database.fetchall("SELECT * FROM measurement_15m ORDER BY key, ts") == first + assert ( + await store.database.fetchall("SELECT * FROM measurement_1m ORDER BY key, ts") == first_1m + ) + + +async def test_incomplete_buckets_are_not_aggregated(store: TimeseriesStore) -> None: + await store.append_measurements( + [("k", T0 + timedelta(seconds=s), 1.0) for s in range(0, 100, 10)] + ) + # now = 00:01:40 -> minute 00:01 is incomplete, quarter hour 00:00 too + report = await store.downsample(T0 + timedelta(seconds=100)) + assert (report.minute_rows, report.quarter_rows) == (1, 0) + + +async def test_15m_weights_minutes_by_sample_count(store: TimeseriesStore) -> None: + # minute 0: 6 samples of 10; minute 1: 1 sample of 70 -> sample mean (60 + 70) / 7 + rows = [("k", T0 + timedelta(seconds=10 * i), 10.0) for i in range(6)] + rows.append(("k", T0 + timedelta(seconds=60), 70.0)) + await store.append_measurements(rows) + await store.downsample(T0 + timedelta(minutes=30)) + series = await store.read_series("k", TimeRange(start=T0, end=T0 + Q), include_recent=False) + assert series.values == (pytest.approx(130 / 7),) + + +async def test_retention_downsamples_before_it_deletes(store: TimeseriesStore) -> None: + old = T0 + await store.append_measurements( + [("k", old + timedelta(seconds=30 * i), 5.0) for i in range(60)] + ) + later = old + RAW_KEEP + timedelta(days=3) + report = await store.maintain(later) + assert report.retention.raw == 60 + assert await store.database.fetchone("SELECT COUNT(*) FROM measurement") == (0,) + # the aggregates survived the deletion of the raw rows + series = await store.read_series("k", TimeRange(start=old, end=old + timedelta(minutes=30))) + assert series.values[:2] == (5.0, 5.0) + + +async def test_retention_cutoffs_per_table(store: TimeseriesStore) -> None: + now = T0 + timedelta(days=100) + db = store.database + for table in ("measurement_1m", "measurement_15m"): + await db.execute( + f"INSERT INTO {table} (key, ts, mean, min, max, n) VALUES ('k', ?, 1, 1, 1, 1)", + (int((now - MINUTE_KEEP - timedelta(days=1)).timestamp()) // 900 * 900,), + ) + await db.execute( + f"INSERT INTO {table} (key, ts, mean, min, max, n) VALUES ('k', ?, 1, 1, 1, 1)", + (int((now - timedelta(days=1)).timestamp()) // 900 * 900,), + ) + report = await store.apply_retention(now) + assert report.minute == 1 + assert await db.fetchone("SELECT COUNT(*) FROM measurement_1m") == (1,) + assert await db.fetchone("SELECT COUNT(*) FROM measurement_15m") == (2,) # unlimited + + +async def test_maintain_on_an_empty_database_and_size_report(store: TimeseriesStore) -> None: + report = await store.maintain(T0) + assert report.retention.total == 0 + assert report.vacuumed is False + size = await store.size_report() + assert size.file_bytes > 0 + assert size.total_bytes >= size.file_bytes + assert set(size.rows) == { + "measurement", + "measurement_1m", + "measurement_15m", + "forecast", + "price", + } + assert all(count == 0 for count in size.rows.values()) + + +async def test_vacuum_runs_only_when_worth_it( + store: TimeseriesStore, monkeypatch: pytest.MonkeyPatch +) -> None: + import suncore.storage.timeseries as module # noqa: PLC0415 + + monkeypatch.setattr(module, "VACUUM_MIN_FREE_BYTES", 4096) + await store.append_measurements( + [(f"key{i % 50}", T0 + timedelta(seconds=i), float(i)) for i in range(20_000)] + ) + assert (await store.maintain(T0 + timedelta(hours=1))).vacuumed is False + before = (await store.size_report()).total_bytes + report = await store.maintain(T0 + RAW_KEEP + timedelta(days=10)) + assert report.retention.raw == 20_000 + assert report.vacuumed is True + assert (await store.size_report()).total_bytes < before + + +# --- read_series ---------------------------------------------------------------------- + + +async def test_read_series_gaps_are_none_and_units_follow_the_key(store: TimeseriesStore) -> None: + await store.store_quarter_hours(SITE_GRID_W, _series([1.0, None, 3.0])) + series = await store.read_series(SITE_GRID_W, TimeRange(start=T0, end=T0 + 4 * Q)) + assert series.values == (1.0, None, 3.0, None) + assert series.unit is SeriesUnit.W + assert series.step == Q + soc = await store.read_series(SITE_SOC_PCT, TimeRange(start=T0, end=T0 + Q)) + assert soc.unit is SeriesUnit.PERCENT + + +async def test_read_series_includes_the_not_yet_aggregated_recent_data( + store: TimeseriesStore, +) -> None: + await store.append_measurements([("k", T0 + timedelta(seconds=30 * i), 2.0) for i in range(30)]) + window = TimeRange(start=T0, end=T0 + Q) + assert (await store.read_series("k", window, include_recent=False)).values == (None,) + assert (await store.read_series("k", window)).values == (2.0,) + await store.downsample(T0 + timedelta(hours=1)) + await store.apply_retention(T0 + timedelta(hours=1)) + assert (await store.read_series("k", window)).values == (2.0,) + + +async def test_read_series_one_minute_resolution(store: TimeseriesStore) -> None: + await store.append_measurements( + [("k", T0 + timedelta(seconds=10 * i), float(i)) for i in range(12)] + ) + await store.downsample(T0 + timedelta(hours=1)) + series = await store.read_series("k", TimeRange(start=T0, end=T0 + timedelta(minutes=2)), "1m") + assert series.step == timedelta(minutes=1) + assert series.values == (2.5, 8.5) + + +async def test_read_series_refuses_a_huge_window(store: TimeseriesStore) -> None: + with pytest.raises(ValueError, match="exceed"): + await store.read_series("k", TimeRange(start=T0, end=T0 + timedelta(days=30)), "1m") + + +async def test_derived_rows_never_overwrite_measured_ones(store: TimeseriesStore) -> None: + await store.store_quarter_hours("k", _series([10.0, None]), derived=False) + await store.store_quarter_hours("k", _series([99.0, 20.0, 30.0]), derived=True) + series = await store.read_series("k", TimeRange(start=T0, end=T0 + 3 * Q)) + assert series.values == (10.0, 20.0, 30.0) + await store.store_quarter_hours( + "k", _series([1.0, 2.0, 3.0]), derived=True + ) # derived on derived + series = await store.read_series("k", TimeRange(start=T0, end=T0 + 3 * Q)) + assert series.values == (10.0, 2.0, 3.0) + assert await store.database.fetchall("SELECT derived FROM measurement_15m ORDER BY ts") == [ + (0,), + (1,), + (1,), + ] + + +async def test_real_aggregates_replace_derived_rows(store: TimeseriesStore) -> None: + await store.store_quarter_hours("k", _series([99.0]), derived=True) + assert await store.has_quarter_history() + await store.append_measurements([("k", T0 + timedelta(seconds=10 * i), 4.0) for i in range(90)]) + await store.downsample(T0 + timedelta(hours=1)) + assert (await store.read_series("k", TimeRange(start=T0, end=T0 + Q))).values == (4.0,) + assert await store.database.fetchone("SELECT derived, n FROM measurement_15m") == (0, 90) + + +async def test_store_quarter_hours_needs_a_15_minute_series(store: TimeseriesStore) -> None: + with pytest.raises(ValueError, match="15-minute"): + await store.store_quarter_hours("k", _series([1.0], step=timedelta(hours=1))) + + +async def test_has_quarter_history(store: TimeseriesStore) -> None: + assert not await store.has_quarter_history() + await store.store_quarter_hours("k", _series([1.0])) + assert await store.has_quarter_history() + + +async def test_device_keys_are_plain_measurements(store: TimeseriesStore) -> None: + key = device_power_key("pool") + await store.append_measurements([(key, T0, 1500.0)]) + assert (await store.read_raw(key, TimeRange(start=T0, end=T0 + Q))) == [(T0, 1500.0)] + + +# --- forecasts ------------------------------------------------------------------------ + + +async def test_forecast_round_trip_with_gaps(store: TimeseriesStore) -> None: + issued = T0 - timedelta(hours=2) + forecast = _quantiles( + [None, 1.0, 2.0, None], [None, 2.0, 3.0, 4.0], [None, 3.0, 4.0, None], start=T0 + ) + assert await store.store_forecast("solcast", "pv", issued, forecast) == 4 + assert await store.read_forecast("solcast", "pv", issued) == forecast # leading None kept + assert await store.read_forecast("solcast", "load", issued) is None + assert await store.read_forecast("other", "pv", issued) is None + + +async def test_forecast_restore_replaces_the_same_issue(store: TimeseriesStore) -> None: + issued = T0 + await store.store_forecast("s", "pv", issued, QuantileSeries.deterministic(_series([1.0, 2.0]))) + await store.store_forecast("s", "pv", issued, QuantileSeries.deterministic(_series([5.0]))) + got = await store.read_forecast("s", "pv", issued) + assert got is not None + assert got.p50.values == (5.0,) + + +async def test_forecast_window_clip_and_plain_time_series(store: TimeseriesStore) -> None: + await store.store_forecast("s", "load", T0, _series([1.0, 2.0, 3.0, 4.0])) + got = await store.read_forecast("s", "load", T0, TimeRange(start=T0 + Q, end=T0 + 3 * Q)) + assert got is not None + assert got.p50.values == (2.0, 3.0) + assert got.p10.values == (None, None) # a plain series fills p50 only + assert got.start == T0 + Q + + +async def test_latest_forecast_before_semantics(store: TimeseriesStore) -> None: + # issue A (00:00) covers 00:00..00:45 with 1; issue B (00:30) covers 00:30..01:15 with 2 + await store.store_forecast("s", "pv", T0, QuantileSeries.deterministic(_series([1.0] * 4))) + await store.store_forecast( + "s", "pv", T0 + 2 * Q, QuantileSeries.deterministic(_series([2.0] * 4, start=T0 + 2 * Q)) + ) + before_b = await store.latest_forecast_before("s", "pv", T0 + Q) + assert before_b is not None + assert before_b.p50.values == (1.0, 1.0, 1.0, 1.0) + after_b = await store.latest_forecast_before("s", "pv", T0 + 2 * Q) + assert after_b is not None + assert after_b.p50.values == (1.0, 1.0, 2.0, 2.0, 2.0, 2.0) + assert await store.latest_forecast_before("s", "pv", T0 - Q) is None + assert await store.forecast_issues("s", "pv") == [T0, T0 + 2 * Q] + assert await store.forecast_issues("s", "pv", since=T0 + Q) == [T0 + 2 * Q] + + +async def test_forecast_retention_by_slot(store: TimeseriesStore) -> None: + await store.store_forecast("s", "pv", T0, QuantileSeries.deterministic(_series([1.0]))) + report = await store.apply_retention(T0 + timedelta(days=60)) + assert report.forecast == 1 + + +# --- prices --------------------------------------------------------------------------- + + +def _prices( + imp: list[float | None], + exp: list[float | None] | None, + start: datetime = T0, + step: timedelta = Q, +) -> PriceSeries: + def make(values: list[float | None]) -> TimeSeries: + return TimeSeries(start=start, step=step, values=tuple(values), unit=SeriesUnit.PRICE) + + return PriceSeries( + source_id="tibber", + price_includes="all_in", + import_price=make(imp), + export_price=None if exp is None else make(exp), + ) + + +async def test_prices_round_trip_with_negative_values_and_gaps(store: TimeseriesStore) -> None: + prices = _prices([0.30, -0.05, None, 0.28], [0.08, 0.0, None, 0.08]) + await store.store_prices(prices) + got = await store.read_prices("tibber", TimeRange(start=T0, end=T0 + 4 * Q)) + assert got == [prices] + + +async def test_prices_without_export_series(store: TimeseriesStore) -> None: + prices = _prices([0.3, 0.4], None) + await store.store_prices(prices) + got = await store.read_prices("tibber", TimeRange(start=T0, end=T0 + 2 * Q)) + assert got == [prices] + assert got[0].export_price is None + + +async def test_prices_mixed_resolution_segments_and_replacement(store: TimeseriesStore) -> None: + quarters = _prices([0.1, 0.2, 0.3, 0.4], None) + hourly = _prices([0.5, 0.6], None, start=T0 + timedelta(hours=1), step=timedelta(hours=1)) + await store.store_prices(quarters) + await store.store_prices(hourly) + got = await store.read_prices("tibber", TimeRange(start=T0, end=T0 + timedelta(hours=3))) + assert [s.step for s in got] == [Q, timedelta(hours=1)] + assert got[1].import_price.values == (0.5, 0.6) + # an hourly update of the first hour replaces the four quarter-hour rows it overlaps + await store.store_prices(_prices([0.9], None, step=timedelta(hours=1))) + got = await store.read_prices("tibber", TimeRange(start=T0, end=T0 + timedelta(hours=3))) + assert [s.step for s in got] == [timedelta(hours=1)] + assert got[0].import_price.values == (0.9, 0.5, 0.6) + + +async def test_prices_are_per_source_and_retained_long(store: TimeseriesStore) -> None: + await store.store_prices(_prices([0.3], None)) + assert await store.read_prices("nordpool", TimeRange(start=T0, end=T0 + Q)) == [] + assert (await store.apply_retention(T0 + timedelta(days=300))).price == 0 + assert (await store.apply_retention(T0 + timedelta(days=500))).price == 1 + + +async def test_price_gap_that_is_not_a_whole_slot_starts_a_new_segment( + store: TimeseriesStore, +) -> None: + await store.store_prices(_prices([0.1], None)) + await store.store_prices(_prices([0.2], None, start=T0 + timedelta(minutes=20))) + got = await store.read_prices("tibber", TimeRange(start=T0, end=T0 + timedelta(hours=1))) + assert len(got) == 2 + + +# --- DST ------------------------------------------------------------------------------ + + +@pytest.mark.parametrize( + ("day", "hours"), + [("2026-03-29", 23), ("2026-10-25", 25), ("2026-06-01", 24)], +) +async def test_series_round_trip_on_dst_days(store: TimeseriesStore, day: str, hours: int) -> None: + berlin = ZoneInfo("Europe/Berlin") + start = datetime.fromisoformat(day).replace(tzinfo=berlin).astimezone(UTC) + end = (datetime.fromisoformat(day) + timedelta(days=1)).replace(tzinfo=berlin).astimezone(UTC) + assert (end - start) == timedelta(hours=hours) + count = hours * 4 + values: list[float | None] = [float(i) if i % 7 else None for i in range(count)] + series = _series(values, start=start) + await store.store_quarter_hours("k", series) + got = await store.read_series("k", TimeRange(start=start, end=end)) + assert got.values == series.values + assert got.start == start + assert len(got) == count + # forecast issued the day before round-trips as well + forecast = QuantileSeries.deterministic(series) + await store.store_forecast("s", "pv", start - timedelta(hours=6), forecast) + assert await store.read_forecast("s", "pv", start - timedelta(hours=6)) == forecast + + +async def test_timestamps_are_stored_in_utc_whatever_the_input_zone(store: TimeseriesStore) -> None: + berlin = ZoneInfo("Europe/Berlin") + local = datetime(2026, 10, 25, 2, 30, tzinfo=berlin, fold=1) # the second 02:30 + await store.append_measurements([("k", local, 5.0)]) + utc = local.astimezone(UTC) # local arithmetic would reset `fold`: windows are UTC + rows = await store.read_raw("k", TimeRange(start=utc, end=utc + Q)) + assert rows == [(utc, 5.0)] + assert utc == datetime(2026, 10, 25, 1, 30, tzinfo=UTC) + assert floor_to_step(utc) == utc From 4f9e6e9b564737b9682ae277446795c75ff0e252 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 02:48:31 +0000 Subject: [PATCH 02/11] docs: note schema 4 and the time-series database for PR 4-B Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- addon/suncore/DOCS.md | 7 +++++-- addon/suncore_beta/DOCS.md | 7 +++++-- docs/STATUS.md | 17 ++++++++++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/addon/suncore/DOCS.md b/addon/suncore/DOCS.md index 45d835d..3a5ad2e 100644 --- a/addon/suncore/DOCS.md +++ b/addon/suncore/DOCS.md @@ -63,8 +63,11 @@ Supported architectures: `amd64` and `aarch64`. the assistant; **Earlier versions** restores a previous save; **Export configuration** and **Import configuration** back up everything as a file. The settings are stored in the app's `/data` folder and are part of the app -backup. This version uses database schema 3, which earlier versions cannot open: -export the configuration before updating. +backup. This version uses database schema 4, which earlier versions cannot open: +export the configuration before updating. Measurements, forecasts and prices +live in a separate `timeseries.db`, which is left out of the app backup; SunCore +rebuilds its quarter-hour history from Home Assistant's statistics when the file +is missing. ### Sign convention diff --git a/addon/suncore_beta/DOCS.md b/addon/suncore_beta/DOCS.md index 7da86da..36975b2 100644 --- a/addon/suncore_beta/DOCS.md +++ b/addon/suncore_beta/DOCS.md @@ -66,8 +66,11 @@ Supported architectures: `amd64` and `aarch64`. the assistant; **Earlier versions** restores a previous save; **Export configuration** and **Import configuration** back up everything as a file. The settings are stored in the app's `/data` folder and are part of the app -backup. This version uses database schema 3, which earlier versions cannot open: -export the configuration before updating. +backup. This version uses database schema 4, which earlier versions cannot open: +export the configuration before updating. Measurements, forecasts and prices +live in a separate `timeseries.db`, which is left out of the app backup; SunCore +rebuilds its quarter-hour history from Home Assistant's statistics when the file +is missing. ### Sign convention diff --git a/docs/STATUS.md b/docs/STATUS.md index d3f3843..3447378 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -308,7 +308,22 @@ maintainer's shadow test · **Branch:** `claude/zealous-mayer-j5oda9` (WP12, optional, after 4-E). Merge only after the Phase 3 acceptance. - **ADR-0013 must be accepted before WP10** (4-G). Its contract changes (`BatteryAction` `None` = baseline, `BatteryBand.grid_charge_max_w`/`discharge_lock`, `effective_reserve_pct`, - `battery_identity`) land in a WP2 fix before 4-B. + `battery_identity`) land at the start of 4-G, after the maintainer accepted it. +- **PR 4-B (WP3, branch `phase-4/b-storage`)**: `/data/timeseries.db` (own migrations; tables + `measurement` 48 h, `measurement_1m` 30 days, `measurement_15m` unlimited with a `derived` flag, + `forecast` versioned by issue time, kept 45 days after the slot, `price` 400 days; + `TimeseriesStore`; nightly job 03:30 local: downsample, retention, VACUUM only when worth it; a + corrupt file is moved aside, a newer one is left alone and SunCore runs without time series). + Rebuild of 28 days of site history from `recorder/statistics_during_period` (period `hour`, + explicit units) once after the first connect when the 15-min table is empty + (`suncore/ha/statistics.py`, `suncore/app/history.py`). Controller measurements recorded through + a batched writer (flush every 60 s, only OK-quality values). `suncore.db` migration 0004 + (`plan_snapshot` zlib JSON, 200 rows / 14 days; `learned_param`): **DB schema 4, earlier versions + refuse it – export before updating.** `ActionRateLimiter` per `domain.service` for + `call_action_response` (default 30 s, `ActionRateLimitedError`, follow-up A3); boundary rule 7 + admits only `suncore.app.adapters`. Fixed: a cancelled `Database.transaction` could leave a + transaction open on the shared connection. `UNVERIFIED`: Supervisor proxy passes `recorder/*`; + `change`/`sum` of real energy counters. - Maintainer questions from 4-A: plan §6 "Offene Fragen aus PR 4-A" (grid-charge schedule precedence, VAT on negative spot prices, export tiers, grid charging on the Kostal). From 011a92cd538dab6c70909e01b7078e0bb5330199 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 03:20:49 +0000 Subject: [PATCH 03/11] fix(storage): move timeseries.db aside only on unambiguous corruption Only SQLite's "not a database"/"malformed" errors and a failing PRAGMA quick_check move the file; a locked, busy, full, unreadable or permission-denied database is logged and the app runs without time series (retry on the next start, nothing moved). The file is moved with its -wal/-shm as timeseries.db.corrupt-, the newest two are kept, and timeseries.db.corrupt* is excluded from backups in both apps. The open now happens inside the try of build_runtime. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- addon/suncore/config.yaml | 1 + addon/suncore_beta/config.yaml | 1 + suncore/app/main.py | 5 +- suncore/storage/timeseries.py | 96 ++++++++++++++++++++++------ tests/storage/test_timeseries.py | 105 ++++++++++++++++++++++++++++++- 5 files changed, 183 insertions(+), 25 deletions(-) diff --git a/addon/suncore/config.yaml b/addon/suncore/config.yaml index fe73cfe..d84fefe 100644 --- a/addon/suncore/config.yaml +++ b/addon/suncore/config.yaml @@ -28,6 +28,7 @@ backup_exclude: - timeseries.db - timeseries.db-wal - timeseries.db-shm + - timeseries.db.corrupt* map: - type: homeassistant_config read_only: true diff --git a/addon/suncore_beta/config.yaml b/addon/suncore_beta/config.yaml index e70b4dd..0a4837d 100644 --- a/addon/suncore_beta/config.yaml +++ b/addon/suncore_beta/config.yaml @@ -26,6 +26,7 @@ backup_exclude: - timeseries.db - timeseries.db-wal - timeseries.db-shm + - timeseries.db.corrupt* map: - type: homeassistant_config read_only: true diff --git a/suncore/app/main.py b/suncore/app/main.py index f6d1e57..1b2a4a0 100644 --- a/suncore/app/main.py +++ b/suncore/app/main.py @@ -96,7 +96,7 @@ def _demo_wiring(settings: Settings) -> DemoWiring: ) -async def build_runtime( +async def build_runtime( # noqa: PLR0915 - the composition root settings: Settings, *, env: Mapping[str, str] | None = None, @@ -126,8 +126,9 @@ async def build_runtime( data_dir = settings.data_dir data_dir.mkdir(parents=True, exist_ok=True) db = await Database.open(data_dir / DB_FILENAME) - timeseries_db = await open_timeseries_or_recover(data_dir / TS_FILENAME) + timeseries_db: Database | None = None try: + timeseries_db = await open_timeseries_or_recover(data_dir / TS_FILENAME) config = ConfigStore(db) if connection is None: connection = WebSocketConnection(transport_from_env(env)) diff --git a/suncore/storage/timeseries.py b/suncore/storage/timeseries.py index 8518952..aa821d6 100644 --- a/suncore/storage/timeseries.py +++ b/suncore/storage/timeseries.py @@ -31,8 +31,9 @@ import math import sqlite3 +import time from collections import Counter -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from datetime import UTC, datetime, timedelta from importlib import resources @@ -93,38 +94,93 @@ async def open_timeseries(path: Path | str) -> Database: return await Database.open(path, migrations=timeseries_migrations()) -async def open_timeseries_or_recover(path: Path | str) -> Database | None: +KEEP_CORRUPT_FILES: Final = 2 +"""How many moved-aside corrupt databases are kept (older ones are deleted).""" + +_CORRUPT_CODES: Final = frozenset({sqlite3.SQLITE_CORRUPT, sqlite3.SQLITE_NOTADB}) +_CORRUPT_TEXT: Final = ("file is not a database", "malformed") + + +def _is_corruption(exc: BaseException) -> bool: + """Unambiguous corruption only; locked, busy, full, I/O and permission errors are not.""" + seen: BaseException | None = exc + while seen is not None: + code = getattr(seen, "sqlite_errorcode", None) + if isinstance(code, int) and code & 0xFF in _CORRUPT_CODES: + return True + if isinstance(seen, sqlite3.DatabaseError) and any( + t in str(seen).lower() for t in _CORRUPT_TEXT + ): + return True + seen = seen.__cause__ + return False + + +async def _quick_check(db: Database) -> bool: + row = await db.fetchone("PRAGMA quick_check(1)") + return row is not None and str(row[0]).lower() == "ok" + + +async def open_timeseries_or_recover( + path: Path | str, *, now: Callable[[], float] = time.time +) -> Database | None: """Open ``timeseries.db`` for the application; ``None`` = run without time series. - The file is rebuildable, so a corrupt one (not a database, a migration that fails) is - moved aside as ``timeseries.db.corrupt`` and a fresh one is created. A file that is - *newer* than this code is never touched: the application runs without time series and - says so (the newer version may still need the data). Control never depends on this file. + The file is rebuildable, but only *unambiguous corruption* (SQLite's "not a database" or + "malformed" errors, a failing ``PRAGMA quick_check``) moves it aside, together with its + ``-wal`` and ``-shm``, as ``timeseries.db.corrupt-`` (the newest + :data:`KEEP_CORRUPT_FILES` are kept) before a fresh file is created. Anything else - a + locked or busy file, a full disk, an I/O or permission error, a migration that fails for + another reason - is logged and the application runs without time series; the next start + tries again and nothing is moved. A file that is *newer* than this code is never + touched either. Control never depends on this file. """ target = Path(path) for attempt in (1, 2): try: - return await open_timeseries(target) + db = await open_timeseries(target) except SchemaTooNewError as exc: log.error("timeseries.schema_too_new", path=str(target), error=str(exc)) return None - except (sqlite3.DatabaseError, MigrationError, OSError) as exc: - log.error("timeseries.open_failed", path=str(target), attempt=attempt, error=str(exc)) - if attempt == 2: - return None + except (sqlite3.Error, MigrationError, OSError) as exc: + corrupt = _is_corruption(exc) + log.error("timeseries.open_failed", path=str(target), corrupt=corrupt, error=str(exc)) + else: try: - _move_aside(target) - except OSError: - log.exception("timeseries.move_aside_failed", path=str(target)) - return None + if await _quick_check(db): + return db + corrupt = True + log.error("timeseries.quick_check_failed", path=str(target)) + except sqlite3.Error as exc: + corrupt = _is_corruption(exc) + log.error("timeseries.quick_check_error", path=str(target), error=str(exc)) + await db.close() + if not corrupt or attempt == 2: + return None + try: + _move_aside(target, now()) + except OSError: + log.exception("timeseries.move_aside_failed", path=str(target)) + return None return None -def _move_aside(path: Path) -> None: - for suffix in ("-wal", "-shm"): - path.with_name(path.name + suffix).unlink(missing_ok=True) - if path.exists(): - path.replace(path.with_name(path.name + ".corrupt")) +def _move_aside(path: Path, now: float) -> None: + """Rename the database and its WAL/SHM to ``.corrupt-``; keep the newest two.""" + stamp = datetime.fromtimestamp(now, UTC).strftime("%Y%m%dT%H%M%SZ") + moved = path.with_name(f"{path.name}.corrupt-{stamp}") + for suffix in ("", "-wal", "-shm"): + source = path.with_name(path.name + suffix) + if source.exists(): + source.replace(moved.with_name(moved.name + suffix)) + sets = sorted( + p + for p in path.parent.glob(f"{path.name}.corrupt-*") + if not p.name.endswith(("-wal", "-shm")) + ) + for old in sets[:-KEEP_CORRUPT_FILES]: + for suffix in ("", "-wal", "-shm"): + old.with_name(old.name + suffix).unlink(missing_ok=True) # --- helpers ------------------------------------------------------------------------ diff --git a/tests/storage/test_timeseries.py b/tests/storage/test_timeseries.py index de99f9e..1cd1c4f 100644 --- a/tests/storage/test_timeseries.py +++ b/tests/storage/test_timeseries.py @@ -3,6 +3,7 @@ from __future__ import annotations +import asyncio import sqlite3 from collections.abc import AsyncIterator from datetime import UTC, datetime, timedelta @@ -100,20 +101,104 @@ async def test_newer_timeseries_db_is_refused_and_untouched(tmp_path: Path) -> N assert not (tmp_path / "t.db.corrupt").exists() -async def test_corrupt_timeseries_db_is_moved_aside_and_recreated(tmp_path: Path) -> None: +async def test_corrupt_timeseries_db_is_moved_aside_with_its_wal(tmp_path: Path) -> None: path = tmp_path / "t.db" path.write_bytes(b"this is not a database" * 100) - db = await open_timeseries_or_recover(path) + db = await open_timeseries_or_recover(path, now=lambda: 1_800_000_000.0) assert db is not None try: assert await db.user_version() >= 1 finally: await db.close() - assert (tmp_path / "t.db.corrupt").read_bytes().startswith(b"this is not") + moved = tmp_path / "t.db.corrupt-20270115T080000Z" + assert moved.read_bytes().startswith(b"this is not") + + +def test_move_aside_takes_wal_and_shm_along_and_never_unlinks_them(tmp_path: Path) -> None: + from suncore.storage.timeseries import _move_aside # noqa: PLC0415 + + path = tmp_path / "t.db" + for suffix, data in (("", b"db"), ("-wal", b"wal"), ("-shm", b"shm")): + (tmp_path / f"t.db{suffix}").write_bytes(data) + _move_aside(path, 1_800_000_000.0) + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "t.db.corrupt-20270115T080000Z", + "t.db.corrupt-20270115T080000Z-shm", + "t.db.corrupt-20270115T080000Z-wal", + ] + assert (tmp_path / "t.db.corrupt-20270115T080000Z-wal").read_bytes() == b"wal" + + +async def test_failing_quick_check_counts_as_corruption( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import suncore.storage.timeseries as module # noqa: PLC0415 + + path = tmp_path / "t.db" + await (await open_timeseries(path)).close() # a valid file first + calls = {"n": 0} + + async def fake_check(_db: Database) -> bool: + calls["n"] += 1 + return calls["n"] > 1 # the first (damaged) file fails, the fresh one passes + + monkeypatch.setattr(module, "_quick_check", fake_check) + db = await open_timeseries_or_recover(path, now=lambda: 1_800_000_000.0) + assert db is not None + await db.close() + assert (tmp_path / "t.db.corrupt-20270115T080000Z").exists() + + +@pytest.mark.parametrize( + "error", + [ + sqlite3.OperationalError("database is locked"), + sqlite3.OperationalError("disk I/O error"), + sqlite3.OperationalError("database or disk is full"), + sqlite3.OperationalError("unable to open database file"), + PermissionError("denied"), + ], +) +async def test_transient_errors_never_move_the_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, error: Exception +) -> None: + import suncore.storage.timeseries as module # noqa: PLC0415 + + path = tmp_path / "t.db" + async with await open_timeseries(path) as db: + await TimeseriesStore(db).append_measurements([("k", T0, 1.0)]) + + async def failing(_path: Path | str) -> Database: + raise error + + monkeypatch.setattr(module, "open_timeseries", failing) + assert await open_timeseries_or_recover(path) is None + monkeypatch.undo() + assert not await asyncio.to_thread(lambda: list(tmp_path.glob("*.corrupt*"))) + async with await open_timeseries(path) as db: # still there, data intact, next start works + assert await TimeseriesStore(db).read_raw("k", TimeRange(start=T0, end=T0 + Q)) == [ + (T0, 1.0) + ] + + +async def test_corrupt_files_are_rotated_keeping_two(tmp_path: Path) -> None: + path = tmp_path / "t.db" + for i in range(4): + path.write_bytes(b"garbage" * 100) + (tmp_path / "t.db-wal").write_bytes(b"w") + db = await open_timeseries_or_recover(path, now=lambda i=i: 1_800_000_000.0 + i * 60) + assert db is not None + await db.close() + path.unlink() + for suffix in ("-wal", "-shm"): + (tmp_path / f"t.db{suffix}").unlink(missing_ok=True) + kept = await asyncio.to_thread(lambda: sorted(p.name for p in tmp_path.glob("t.db.corrupt-*"))) + assert kept == ["t.db.corrupt-20270115T080200Z", "t.db.corrupt-20270115T080300Z"] async def test_unopenable_path_gives_none(tmp_path: Path) -> None: assert await open_timeseries_or_recover(tmp_path / "missing" / "dir" / "t.db") is None + assert not await asyncio.to_thread(lambda: list(tmp_path.glob("**/*.corrupt*"))) # --- measurements --------------------------------------------------------------------- @@ -502,3 +587,17 @@ async def test_timestamps_are_stored_in_utc_whatever_the_input_zone(store: Times assert rows == [(utc, 5.0)] assert utc == datetime(2026, 10, 25, 1, 30, tzinfo=UTC) assert floor_to_step(utc) == utc + + +@pytest.mark.parametrize("app", ["suncore", "suncore_beta"]) +def test_backup_excludes_the_database_its_wal_and_the_corrupt_copies(app: str) -> None: + config = (Path(__file__).resolve().parents[2] / "addon" / app / "config.yaml").read_text( + "utf-8" + ) + for entry in ( + "timeseries.db", + "timeseries.db-wal", + "timeseries.db-shm", + "timeseries.db.corrupt*", + ): + assert f" - {entry}\n" in config From 43f1a10511eb116d589a2e2788c6c0077f717f41 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 03:28:49 +0000 Subject: [PATCH 04/11] fix(storage): time-first primary keys, quarter-aligned 1m tier, measured beats derived - measurement and measurement_1m use PRIMARY KEY (ts, key) (schema unreleased, 0001 changed). One flush now touches the last pages of the table instead of one page per key. Measured on a 48 h preload with 10 keys, 120 rows per flush, WAL bytes per flush: (key, ts) 131841, (ts, key) 23897 (5.5x less). Per-key range reads seek on ts (query-plan test); no extra index. - downsample writes the 1-minute tier only up to floor(now, 15 min), so a bucket is either whole in measurement_1m or whole in the raw rows and read_series never mixes the two inside one quarter hour (regression test with the review example: job at 03:35, 5 min x 100 W then 10 min x 1000 W reads 700 W). - read_series: a derived (rebuilt) row never beats data measured live. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- suncore/storage/timeseries.py | 23 ++++++--- .../storage/ts_migrations/0001_initial.sql | 10 ++-- tests/storage/test_timeseries.py | 51 ++++++++++++++++++- 3 files changed, 72 insertions(+), 12 deletions(-) diff --git a/suncore/storage/timeseries.py b/suncore/storage/timeseries.py index aa821d6..45183f5 100644 --- a/suncore/storage/timeseries.py +++ b/suncore/storage/timeseries.py @@ -338,11 +338,16 @@ async def read_series( if include_recent and resolution == "15m": means.update(await self._recent_quarters(key, first, last)) table = _TABLE_OF[resolution] + derived_col = ", derived" if resolution == "15m" else ", 0" rows = await self._db.fetchall( - f"SELECT ts, mean FROM {table} WHERE key = ? AND ts >= ? AND ts < ?", + f"SELECT ts, mean{derived_col} FROM {table} WHERE key = ? AND ts >= ? AND ts < ?", (key, first, last), ) - means.update({_int(r[0]): _num(r[1]) for r in rows}) + for row in rows: + ts_ = _int(row[0]) + if _int(row[2]) and ts_ in means: + continue # a derived (rebuilt) value never beats what was measured + means[ts_] = _num(row[1]) values = tuple(means.get(first + i * step) for i in range(count)) return TimeSeries( start=_from_epoch(first), @@ -352,7 +357,7 @@ async def read_series( ) async def _recent_quarters(self, key: str, first: int, last: int) -> dict[int, float]: - """Quarter-hour means from the finer tables (1 minute wins over raw per bucket).""" + """Quarter-hour means from the finer tables (a bucket is whole in one of them).""" out: dict[int, float] = {} raw = await self._db.fetchall( "SELECT (ts / 900) * 900 AS b, AVG(value) FROM measurement " @@ -560,16 +565,22 @@ async def read_prices(self, source_id: str, window: TimeRange) -> list[PriceSeri # --- maintenance -------------------------------------------------------------- async def downsample(self, now: datetime) -> DownsampleReport: - """Aggregate every complete minute and quarter hour (idempotent, see module doc).""" + """Aggregate every complete quarter hour into the 1-minute and 15-minute tables. + + Idempotent (see module doc). The 1-minute tier is written only for buckets that end at + or before ``floor(now, 15 min)``, never for a partial quarter: a bucket is then either + entirely in ``measurement_1m`` or entirely in the raw rows, which keeps + :meth:`read_series` from mixing the two inside one quarter hour. + """ now_s = _epoch(now) async with self._db.transaction() as tx: minutes = await tx.execute( "INSERT INTO measurement_1m (key, ts, mean, min, max, n) " "SELECT key, (ts / 60) * 60 AS b, AVG(value), MIN(value), MAX(value), COUNT(*) " "FROM measurement WHERE ts < ? GROUP BY key, b " - "ON CONFLICT(key, ts) DO UPDATE SET mean = excluded.mean, min = excluded.min, " + "ON CONFLICT(ts, key) DO UPDATE SET mean = excluded.mean, min = excluded.min, " "max = excluded.max, n = excluded.n", - (_floor(now_s, MINUTE_S),), + (_floor(now_s, QUARTER_S),), ) quarters = await tx.execute( "INSERT INTO measurement_15m (key, ts, mean, min, max, n, derived) " diff --git a/suncore/storage/ts_migrations/0001_initial.sql b/suncore/storage/ts_migrations/0001_initial.sql index f7dde6d..0c50910 100644 --- a/suncore/storage/ts_migrations/0001_initial.sql +++ b/suncore/storage/ts_migrations/0001_initial.sql @@ -2,15 +2,17 @@ -- timeseries.db (ADR-0004, Phase 4 WP3). All timestamps are integer seconds since the UNIX -- epoch (UTC); a gap is the absence of a row, never a stored zero. --- Controller resolution, kept 48 h. +-- Controller resolution, kept 48 h. The primary key is (ts, key): the writes of one flush are +-- consecutive in time, so they land on the last pages of the table instead of one page per key +-- (about 5x fewer WAL bytes per flush on an SD card). Per-key range reads scan the ts range. CREATE TABLE measurement ( key TEXT NOT NULL, ts INTEGER NOT NULL, value REAL NOT NULL, - PRIMARY KEY (key, ts) + PRIMARY KEY (ts, key) ) WITHOUT ROWID; --- 1-minute aggregates (ts = start of the minute), kept 30 days. +-- 1-minute aggregates (ts = start of the minute), kept 30 days; only complete quarter hours. CREATE TABLE measurement_1m ( key TEXT NOT NULL, ts INTEGER NOT NULL, @@ -18,7 +20,7 @@ CREATE TABLE measurement_1m ( min REAL NOT NULL, max REAL NOT NULL, n INTEGER NOT NULL CHECK (n > 0), - PRIMARY KEY (key, ts) + PRIMARY KEY (ts, key) ) WITHOUT ROWID; -- 15-minute aggregates (ts = start of the quarter hour), kept for ever. `derived` = 1 marks diff --git a/tests/storage/test_timeseries.py b/tests/storage/test_timeseries.py index 1cd1c4f..3d53f26 100644 --- a/tests/storage/test_timeseries.py +++ b/tests/storage/test_timeseries.py @@ -267,9 +267,56 @@ async def test_incomplete_buckets_are_not_aggregated(store: TimeseriesStore) -> await store.append_measurements( [("k", T0 + timedelta(seconds=s), 1.0) for s in range(0, 100, 10)] ) - # now = 00:01:40 -> minute 00:01 is incomplete, quarter hour 00:00 too + # now = 00:01:40: the quarter hour 00:00 is incomplete, so not even its minutes are written report = await store.downsample(T0 + timedelta(seconds=100)) - assert (report.minute_rows, report.quarter_rows) == (1, 0) + assert (report.minute_rows, report.quarter_rows) == (0, 0) + + +async def test_a_job_inside_a_quarter_hour_does_not_skew_the_quarter( + store: TimeseriesStore, +) -> None: + """Review example: job at 03:35, 5 min x 100 W before it, 10 min x 1000 W after it.""" + quarter = T0 + timedelta(hours=3, minutes=30) + window = TimeRange(start=quarter, end=quarter + Q) + low = [("k", quarter + timedelta(seconds=10 * i), 100.0) for i in range(30)] + high = [("k", quarter + timedelta(minutes=5, seconds=10 * i), 1000.0) for i in range(60)] + await store.append_measurements(low) + await store.downsample(quarter + timedelta(minutes=5)) # the 03:35 job + await store.append_measurements(high) + expected = (30 * 100 + 60 * 1000) / 90 # 700 + assert (await store.read_series("k", window)).values == (pytest.approx(expected),) + await store.downsample(quarter + timedelta(minutes=20)) # a later job completes it + assert (await store.read_series("k", window, include_recent=False)).values == ( + pytest.approx(expected), + ) + assert (await store.read_series("k", window)).values == (pytest.approx(expected),) + + +async def test_primary_keys_are_time_first_and_range_reads_use_them(store: TimeseriesStore) -> None: + """(ts, key) keeps one flush on consecutive pages (SD wear); reads still seek on ts.""" + db = store.database + for table in ("measurement", "measurement_1m"): + columns = await db.fetchall(f"PRAGMA table_info({table})") + order = sorted((int(str(c[5])), str(c[1])) for c in columns if int(str(c[5])) > 0) + assert [name for _, name in order] == ["ts", "key"] + plan = await db.fetchall( + "EXPLAIN QUERY PLAN SELECT ts, value FROM measurement WHERE key = ? AND ts >= ? AND ts < ?", + ("k", 0, 10), + ) + assert any("PRIMARY KEY" in str(r[3]) and "ts>" in str(r[3]) for r in plan) + + +async def test_measured_beats_derived_when_merging_recent_data(store: TimeseriesStore) -> None: + window = TimeRange(start=T0, end=T0 + Q) + await store.store_quarter_hours("k", _series([99.0]), derived=True) + await store.append_measurements([("k", T0 + timedelta(seconds=10 * i), 5.0) for i in range(30)]) + # not yet aggregated: the live data wins over the rebuilt value + assert (await store.read_series("k", window)).values == (5.0,) + assert (await store.read_series("k", window, include_recent=False)).values == (99.0,) + # a derived row alone is still used, and an aggregated measured row beats live data + other = TimeRange(start=T0 + Q, end=T0 + 2 * Q) + await store.store_quarter_hours("k", _series([7.0], start=T0 + Q), derived=True) + assert (await store.read_series("k", other)).values == (7.0,) async def test_15m_weights_minutes_by_sample_count(store: TimeseriesStore) -> None: From 187ef8969885d07d063ba1c8d5811851d66b1ed3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 03:33:19 +0000 Subject: [PATCH 05/11] fix(storage): rebuild the history until it succeeded once The rebuild no longer depends on "measurement_15m is empty" (which a maintenance run or the first samples defeated). A marker row in timeseries.db (table meta, key history_rebuilt_at; it dies with the file) is set after a successful rebuild. A timeout or Home Assistant error is retried after 1, 5, 15 and then every 60 minutes; unknown_command is final for the process. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- suncore/app/runtime.py | 48 ++++++++++++------ suncore/storage/timeseries.py | 16 ++++++ .../storage/ts_migrations/0001_initial.sql | 7 +++ tests/app/test_timeseries_runtime.py | 50 ++++++++++++++++--- 4 files changed, 101 insertions(+), 20 deletions(-) diff --git a/suncore/app/runtime.py b/suncore/app/runtime.py index cd76160..ad318e9 100644 --- a/suncore/app/runtime.py +++ b/suncore/app/runtime.py @@ -79,6 +79,7 @@ SqliteRuntimeStore, TimeseriesStore, ) +from suncore.storage.timeseries import META_HISTORY_REBUILT from suncore.templates import TemplateCatalog LOOPBACK_PEERS: tuple[str, ...] = ("127.0.0.1", "::1") @@ -88,6 +89,8 @@ DB_FILENAME = "suncore.db" PRUNE_INTERVAL_S = 3600.0 """Decision-log and plan-snapshot retention run hourly.""" +REBUILD_BACKOFF_S = (60.0, 300.0, 900.0, 3600.0) +"""Retry delays of the statistics rebuild after a timeout or Home Assistant error.""" MAINTENANCE_TIME = time(3, 30) """Local time of the nightly time-series job (downsample, retention, vacuum if worth it).""" @@ -390,20 +393,37 @@ def _start_history_rebuild(self) -> None: self._history_task = asyncio.create_task(self._rebuild_history(), name="suncore-history") async def _rebuild_history(self) -> None: - store, site = self.timeseries, self.site - if store is None or site is None: - return - try: - if not await store.has_quarter_history(): - await rebuild_history(self.connection, store, site, self.clock()) - except asyncio.CancelledError: - raise - except (HAError, TimeoutError) as exc: - # UNVERIFIED: the Supervisor proxy may not forward recorder/*. Not retried before - # the next start; the history then grows from the controller's own measurements. - log.warning("history.rebuild_failed", reason=type(exc).__name__, error=str(exc)) - except Exception: - log.exception("history.rebuild_crashed") + """Rebuild until it succeeded once (marker in ``timeseries.db``), with back-off. + + A timeout or Home Assistant error is retried after 1, 5, 15 and then every 60 minutes; + ``unknown_command`` (the recorder commands are not available, e.g. a proxy that does not + forward them - UNVERIFIED) is final for this process and retried at the next start. + """ + attempt = 0 + while True: + store, site = self.timeseries, self.site + if store is None or site is None: + return + try: + if await store.get_meta(META_HISTORY_REBUILT) is None: + await rebuild_history(self.connection, store, site, self.clock()) + await store.set_meta(META_HISTORY_REBUILT, self.clock().isoformat()) + except asyncio.CancelledError: + raise + except CommandError as exc: + log.warning("history.rebuild_failed", reason=exc.code, error=str(exc)) + if exc.code == "unknown_command": + break + except (HAError, TimeoutError) as exc: + log.warning("history.rebuild_failed", reason=type(exc).__name__, error=str(exc)) + except Exception: + log.exception("history.rebuild_crashed") + break + else: + break + delay = REBUILD_BACKOFF_S[min(attempt, len(REBUILD_BACKOFF_S) - 1)] + attempt += 1 + await self.sleep(delay) self._history_done = True async def _connect(self) -> None: diff --git a/suncore/storage/timeseries.py b/suncore/storage/timeseries.py index 45183f5..5d79606 100644 --- a/suncore/storage/timeseries.py +++ b/suncore/storage/timeseries.py @@ -60,6 +60,9 @@ TS_FILENAME: Final = "timeseries.db" +META_HISTORY_REBUILT: Final = "history_rebuilt_at" +"""Set (ISO time) once the rebuild from HA's statistics succeeded; it dies with the file.""" + RAW_KEEP: Final = timedelta(hours=48) MINUTE_KEEP: Final = timedelta(days=30) FORECAST_KEEP: Final = timedelta(days=45) @@ -402,6 +405,18 @@ async def store_quarter_hours( ) return len(batch) + async def get_meta(self, key: str) -> str | None: + """A fact about this file's content (see ``META_HISTORY_REBUILT``).""" + row = await self._db.fetchone("SELECT value FROM meta WHERE key = ?", (key,)) + return None if row is None else str(row[0]) + + async def set_meta(self, key: str, value: str) -> None: + await self._db.execute( + "INSERT INTO meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (key, value), + ) + async def has_quarter_history(self) -> bool: """Whether ``measurement_15m`` holds any row (the rebuild runs when it does not).""" return await self._db.fetchone("SELECT 1 FROM measurement_15m LIMIT 1") is not None @@ -758,6 +773,7 @@ def _price_segment(source_id: str, segment: Sequence[Row]) -> PriceSeries: __all__ = [ "FORECAST_KEEP", + "META_HISTORY_REBUILT", "MINUTE_KEEP", "PRICE_KEEP", "RAW_KEEP", diff --git a/suncore/storage/ts_migrations/0001_initial.sql b/suncore/storage/ts_migrations/0001_initial.sql index 0c50910..7bbf905 100644 --- a/suncore/storage/ts_migrations/0001_initial.sql +++ b/suncore/storage/ts_migrations/0001_initial.sql @@ -62,3 +62,10 @@ CREATE TABLE price ( export_price REAL, PRIMARY KEY (source_id, slot_start) ) WITHOUT ROWID; + +-- Small facts about the content of this file, e.g. that the history was rebuilt from Home +-- Assistant's statistics. They live here, not in suncore.db: a lost timeseries.db must lose them. +CREATE TABLE meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +) WITHOUT ROWID; diff --git a/tests/app/test_timeseries_runtime.py b/tests/app/test_timeseries_runtime.py index c25f024..5bd5df0 100644 --- a/tests/app/test_timeseries_runtime.py +++ b/tests/app/test_timeseries_runtime.py @@ -25,8 +25,8 @@ device_power_key, ) from suncore.core.series import TimeRange -from suncore.ha.connection import CommandError -from suncore.storage.timeseries import TS_FILENAME, TimeseriesStore +from suncore.ha.connection import CommandError, HAError +from suncore.storage.timeseries import META_HISTORY_REBUILT, TS_FILENAME, TimeseriesStore from tests.app.conftest import RecordingConnection, demo_settings T0 = datetime(2026, 6, 21, 0, 0, tzinfo=UTC) @@ -473,19 +473,56 @@ async def test_rebuild_runs_once_after_connect_when_the_15_minute_table_is_empty assert len(conn.recorder.calls) == calls -async def test_rebuild_is_skipped_when_history_exists( +async def test_rebuild_is_skipped_once_the_marker_is_set( rebuild_env: tuple[Runtime, RecordingConnection], ) -> None: + runtime, conn = rebuild_env + await store_of(runtime).set_meta(META_HISTORY_REBUILT, "2026-06-01T00:00:00+00:00") + await runtime.start() + await until(lambda: runtime._history_done) + assert conn.recorder.calls == [] + + +async def test_existing_quarter_history_does_not_suppress_the_rebuild( + rebuild_env: tuple[Runtime, RecordingConnection], +) -> None: + """Only a successful rebuild sets the marker: data (or a maintenance run) is no substitute.""" runtime, conn = rebuild_env from suncore.core.series import SeriesUnit, TimeSeries # noqa: PLC0415 - await store_of(runtime).store_quarter_hours( + store = store_of(runtime) + await store.store_quarter_hours( SITE_HOUSE_W, TimeSeries(start=T0, step=timedelta(minutes=15), values=(1.0,), unit=SeriesUnit.W), ) + assert await store.has_quarter_history() + await runtime.start() # the maintenance job runs at start, too + await until(lambda: runtime._history_done) + assert conn.recorder.calls + assert await store.get_meta(META_HISTORY_REBUILT) is not None + + +async def test_rebuild_retries_with_backoff_until_it_succeeded_once( + rebuild_env: tuple[Runtime, RecordingConnection], +) -> None: + runtime, conn = rebuild_env + sleep = GatedSleep() + runtime.sleep = sleep + runtime.flush_interval_s = 61.0 # keep the flush loop out of the 60 s back-off + conn.statistics_error = TimeoutError("slow") + store = store_of(runtime) await runtime.start() + await until(lambda: 60.0 in sleep.requested) # first retry after a minute + assert not runtime._history_done + assert await store.get_meta(META_HISTORY_REBUILT) is None + conn.statistics_error = HAError("down") + await sleep.release(60.0) + await until(lambda: 300.0 in sleep.requested) + conn.statistics_error = None # Home Assistant is back + await sleep.release(300.0) await until(lambda: runtime._history_done) - assert conn.recorder.calls == [] + assert await store.get_meta(META_HISTORY_REBUILT) is not None + assert await store.has_quarter_history() async def test_rebuild_failure_is_logged_and_does_not_stop_the_app( @@ -497,8 +534,9 @@ async def test_rebuild_failure_is_logged_and_does_not_stop_the_app( await runtime.start() await until(lambda: runtime._history_done) warning = next(e for e in logs if e["event"] == "history.rebuild_failed") - assert warning["reason"] == "CommandError" + assert warning["reason"] == "unknown_command" assert not await store_of(runtime).has_quarter_history() + assert await store_of(runtime).get_meta(META_HISTORY_REBUILT) is None # retried next start stack = await idle(runtime) await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) assert await stack.tick() is not None # control is unaffected From bc53958e6dba94f63facfc6865c8335d54cb477c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 03:37:14 +0000 Subject: [PATCH 06/11] fix(storage): keep the batch when a flush is cancelled; harden cancellation paths - MeasurementRecorder.flush puts the batch back on any BaseException (shutdown cancels the flush loop) and re-raises; the 7 hours (not 100 minutes) of the buffer cap are documented correctly. - Database: Transaction.executemany closes its cursor even if the task is cancelled, and the retrieved outcome of the shielded rollback no longer raises "exception never retrieved". - Includes the earlier fix that a cancelled writer could leave a transaction open on the shared connection ("cannot start a transaction within a transaction"; rollback is now queued behind the cancelled statements and shielded). It affects suncore.db as well. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- suncore/app/recorder.py | 9 ++++++--- suncore/storage/db.py | 7 +++++-- tests/app/test_timeseries_runtime.py | 26 ++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/suncore/app/recorder.py b/suncore/app/recorder.py index fe8258f..dd8add0 100644 --- a/suncore/app/recorder.py +++ b/suncore/app/recorder.py @@ -23,7 +23,7 @@ FLUSH_INTERVAL_S: Final = 60.0 MAX_BUFFERED_ROWS: Final = 50_000 -"""About 100 minutes of a 5 s tick with 10 keys: the cap for a database that stays broken.""" +"""About 7 hours of a 5 s tick with 10 keys (2 rows/s): the cap for a database that stays broken.""" class MeasurementRecorder: @@ -67,9 +67,12 @@ async def flush(self) -> int: return 0 try: return await self._store.append_measurements(batch) - except Exception: - log.exception("timeseries.flush_failed", rows=len(batch)) + except BaseException as exc: + # Also on cancellation (shutdown): the batch goes back, nothing is lost silently. combined = batch + self._buffer self.dropped += max(0, len(combined) - self._max) self._buffer = combined[-self._max :] + if not isinstance(exc, Exception): + raise + log.exception("timeseries.flush_failed", rows=len(batch)) return 0 diff --git a/suncore/storage/db.py b/suncore/storage/db.py index c4c5969..71989fb 100644 --- a/suncore/storage/db.py +++ b/suncore/storage/db.py @@ -31,7 +31,10 @@ async def run() -> None: with contextlib.suppress(sqlite3.OperationalError): await conn.execute("ROLLBACK") - await asyncio.shield(asyncio.ensure_future(run())) + task = asyncio.ensure_future(run()) + # Retrieve the outcome even if this await is cancelled: no "exception never retrieved". + task.add_done_callback(lambda t: t.cancelled() or t.exception()) + await asyncio.shield(task) class Transaction: @@ -51,7 +54,7 @@ async def execute(self, sql: str, params: Sequence[SqlParam] = ()) -> int: async def executemany(self, sql: str, rows: Iterable[Sequence[SqlParam]]) -> None: """Run a statement for every parameter row (one round trip to the SQLite thread).""" cur = await self._conn.executemany(sql, rows) - await cur.close() + await asyncio.shield(cur.close()) # closed even if this task is cancelled meanwhile async def fetchall(self, sql: str, params: Sequence[SqlParam] = ()) -> list[Row]: async with self._conn.execute(sql, params) as cur: diff --git a/tests/app/test_timeseries_runtime.py b/tests/app/test_timeseries_runtime.py index 5bd5df0..4c3b082 100644 --- a/tests/app/test_timeseries_runtime.py +++ b/tests/app/test_timeseries_runtime.py @@ -561,3 +561,29 @@ async def test_rebuild_waits_for_the_site( assert conn.recorder.calls finally: await runtime.stop() + + +async def test_a_cancelled_flush_puts_the_batch_back(tmp_path: Path) -> None: + from suncore.storage import open_timeseries # noqa: PLC0415 + + async with await open_timeseries(tmp_path / "t.db") as db: + store = TimeseriesStore(db) + recorder = MeasurementRecorder(store) + recorder.record(T0, {"a": 1.0, "b": 2.0}) + entered = asyncio.Event() + + async def hang(_rows: Any) -> int: + entered.set() + await asyncio.Event().wait() + return 0 + + store.append_measurements = hang # type: ignore[method-assign] + task = asyncio.create_task(recorder.flush()) + await entered.wait() + assert recorder.pending == 0 # in flight + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert recorder.pending == 2 # back in the buffer, to be written by the final flush + del store.append_measurements + assert await recorder.flush() == 2 From 72543a042c6881ee4182c34fc62bf996b5bd5d28 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 06:37:19 +0000 Subject: [PATCH 07/11] fix(ha): optional rate-limit sub-key; rate limit the fake like the real connection call_action_response takes rate_key (e.g. the target entity or date): the limiter key is domain.service or domain.service#key, so two different calls of one service do not block each other. A cancelled waiter keeps its reservation (documented, tested). FakeHAConnection uses the same ActionRateLimiter, so adapter tests meet the real behaviour. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- suncore/ha/connection.py | 15 ++++++-- suncore/ha/websocket.py | 4 ++- suncore/sim/fake_ha.py | 14 ++++++-- tests/ha/test_action_rate_limit.py | 57 ++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 5 deletions(-) diff --git a/suncore/ha/connection.py b/suncore/ha/connection.py index 0f5aea8..77dadb2 100644 --- a/suncore/ha/connection.py +++ b/suncore/ha/connection.py @@ -73,6 +73,12 @@ class ActionNotAllowedError(HAError): """``call_action_response`` was asked for an action that is not on the read-only allowlist.""" +def action_rate_key(domain: str, service: str, rate_key: str | None = None) -> str: + """Limiter key of an action call: ``domain.service`` plus an optional caller sub-key.""" + base = f"{domain}.{service}" + return base if not rate_key else f"{base}#{rate_key}" + + class ActionRateLimitedError(HAError): """A read-only action was asked for again before its minimum interval elapsed. @@ -248,6 +254,7 @@ async def call_action_response( target: Mapping[str, Any] | None = None, min_interval_s: float | None = None, max_wait_s: float = 0.0, + rate_key: str | None = None, ) -> dict[str, Any]: """Call a read-only action with ``return_response`` and return its response. @@ -258,7 +265,9 @@ async def call_action_response( call per ``min_interval_s`` (default :data:`DEFAULT_ACTION_INTERVAL_S`; a caller passes the interval of its configured polling). A call that is too early waits at most ``max_wait_s`` (default: not at all) and otherwise raises - :class:`ActionRateLimitedError`; it never blocks indefinitely. + :class:`ActionRateLimitedError`; it never blocks indefinitely. ``rate_key`` (e.g. the + target entity or the requested date) gives calls of one service that ask for different + things their own slot: the limiter key is ``domain.service`` or ``domain.service#key``. """ ... @@ -336,7 +345,9 @@ class ActionRateLimiter: """Minimum spacing per key (``domain.service``) with try-acquire and a wait deadline. Slots are handed out in order: a caller that waits reserves the slot after the previous - one, so two concurrent callers do not both pass. Clock and sleep are injected. A key that + one, so two concurrent callers do not both pass. A waiter that is cancelled keeps its + reservation (the slot stays taken until it would have elapsed): that is the safe direction + for an integration that must not be hammered. Clock and sleep are injected. A key that is too busy is refused instead of queued (:meth:`acquire` returns ``False``), so a slow integration can never stack up calls. """ diff --git a/suncore/ha/websocket.py b/suncore/ha/websocket.py index 9b0860b..6a80ed9 100644 --- a/suncore/ha/websocket.py +++ b/suncore/ha/websocket.py @@ -96,6 +96,7 @@ StateContext, StateListener, Unsubscribe, + action_rate_key, ) from suncore.ha.redact import DEFAULT_REGISTRY, SecretRegistry from suncore.ha.transport import Transport @@ -485,6 +486,7 @@ async def call_action_response( target: Mapping[str, Any] | None = None, min_interval_s: float | None = None, max_wait_s: float = 0.0, + rate_key: str | None = None, ) -> dict[str, Any]: """Read-only action call with ``return_response``; see :class:`HAConnection`. @@ -495,7 +497,7 @@ async def call_action_response( if not is_read_only_action(domain, service): msg = f"{domain}.{service} is not a read-only action" raise ActionNotAllowedError(msg) - key = f"{domain}.{service}" + key = action_rate_key(domain, service, rate_key) if not await self._action_limiter.acquire(key, min_interval_s, max_wait_s=max_wait_s): raise ActionRateLimitedError(key, self._action_limiter.retry_after(key)) payload: dict[str, Any] = { diff --git a/suncore/sim/fake_ha.py b/suncore/sim/fake_ha.py index 45bb5a4..41edd80 100644 --- a/suncore/sim/fake_ha.py +++ b/suncore/sim/fake_ha.py @@ -60,6 +60,8 @@ from suncore.ha.connection import ( REGISTRY_EVENT_TYPES, ActionNotAllowedError, + ActionRateLimitedError, + ActionRateLimiter, CommandError, ConnectionState, EntityListener, @@ -73,6 +75,7 @@ StateContext, StateListener, Unsubscribe, + action_rate_key, ) from suncore.sim import registry from suncore.sim.demo_devices import ( @@ -256,6 +259,9 @@ def __init__( self._event_time: datetime | None = None self._offset_s = 0 self._unavailable: frozenset[str] = frozenset() + self._action_limiter = ActionRateLimiter( + clock=lambda: self._clock().timestamp(), sleep=sleep + ) self.call_log: list[CallRecord] = [] """Every service call seen, oldest first.""" self.recorder = SimRecorder( @@ -427,14 +433,18 @@ async def call_action_response( data: Mapping[str, Any] | None = None, # noqa: ARG002 *, target: Mapping[str, Any] | None = None, # noqa: ARG002 - min_interval_s: float | None = None, # noqa: ARG002 - the fake is not rate limited - max_wait_s: float = 0.0, # noqa: ARG002 + min_interval_s: float | None = None, + max_wait_s: float = 0.0, + rate_key: str | None = None, ) -> dict[str, Any]: """Read-only action responses built from the demo data (allowlist like the real one).""" if not is_read_only_action(domain, service): msg = f"{domain}.{service} is not a read-only action" raise ActionNotAllowedError(msg) self._require_connected() + key = action_rate_key(domain, service, rate_key) + if not await self._action_limiter.acquire(key, min_interval_s, max_wait_s=max_wait_s): + raise ActionRateLimitedError(key, self._action_limiter.retry_after(key)) now = self._house_state.time if self._house_state is not None else self._clock() if (domain, service) == ("tibber", "get_prices"): start = now.replace(minute=(now.minute // 15) * 15, second=0, microsecond=0) diff --git a/tests/ha/test_action_rate_limit.py b/tests/ha/test_action_rate_limit.py index 10a3538..62a2e7e 100644 --- a/tests/ha/test_action_rate_limit.py +++ b/tests/ha/test_action_rate_limit.py @@ -222,3 +222,60 @@ def test_the_refusal_has_a_catalogue_key() -> None: for name in ("en", "de"): catalogue = json.loads((locales / f"{name}.json").read_text(encoding="utf-8")) assert catalogue[ERR_ACTION_RATE_LIMITED] + + +def test_action_rate_key_default_and_sub_key() -> None: + from suncore.ha.connection import action_rate_key # noqa: PLC0415 + + assert action_rate_key("tibber", "get_prices") == "tibber.get_prices" + assert action_rate_key("tibber", "get_prices", None) == "tibber.get_prices" + assert action_rate_key("tibber", "get_prices", "") == "tibber.get_prices" + assert action_rate_key("weather", "get_forecasts", "weather.home") == ( + "weather.get_forecasts#weather.home" + ) + + +async def test_sub_keys_do_not_block_each_other(conn: WebSocketConnection, ha: FakeHA) -> None: + ha.service_response = lambda _m: {} + await conn.call_action_response("weather", "get_forecasts", rate_key="weather.home") + await conn.call_action_response("weather", "get_forecasts", rate_key="weather.cabin") + with pytest.raises(ActionRateLimitedError) as info: + await conn.call_action_response("weather", "get_forecasts", rate_key="weather.home") + assert info.value.action == "weather.get_forecasts#weather.home" + await conn.call_action_response("weather", "get_forecasts") # the plain key: its own slot + with pytest.raises(ActionRateLimitedError): + await conn.call_action_response("weather", "get_forecasts") + assert _calls(ha) == 3 + + +async def test_a_cancelled_waiter_keeps_its_reservation() -> None: + clock = FakeClock() + limiter = _limiter(clock, 10.0) + assert await limiter.acquire("a.b") + waiter = asyncio.create_task(limiter.acquire("a.b", max_wait_s=100.0)) + await clock.settle() + waiter.cancel() + await clock.settle() + assert limiter.retry_after("a.b") == 20.0 # its slot (10 s) + the interval stays taken + + +async def test_the_fake_connection_is_limited_like_the_real_one() -> None: + from datetime import UTC, datetime # noqa: PLC0415 + + from suncore.sim.fake_ha import FakeHAConnection # noqa: PLC0415 + from suncore.sim.scenario import load_scenario # noqa: PLC0415 + + now = [datetime(2026, 6, 21, tzinfo=UTC)] + fake = FakeHAConnection( + load_scenario("summer_day"), clock=lambda: now[0], autorun=False, speed=1.0 + ) + await fake.connect() + try: + await fake.call_action_response("tibber", "get_prices") + with pytest.raises(ActionRateLimitedError): + await fake.call_action_response("tibber", "get_prices") + await fake.call_action_response("tibber", "get_prices", rate_key="other") + now[0] = datetime(2026, 6, 21, 0, 0, 31, tzinfo=UTC) + await fake.call_action_response("tibber", "get_prices") + finally: + await fake.close() From 95ec91b8e2aebea8e03a3f288ab04409f68bfe67 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 06:40:36 +0000 Subject: [PATCH 08/11] fix(app): nightly time-series job follows the time zone and never runs twice The loop wakes at most hourly and targets "the first 03:30 after the last run" in the current zone, so a zone that becomes known after start (Home Assistant's) moves the target instead of leaving a stale 24 h sleep, and a wake-up cannot repeat a run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- suncore/app/runtime.py | 37 ++++++++++++++----- tests/app/test_timeseries_runtime.py | 55 +++++++++++++++++++++++++--- 2 files changed, 77 insertions(+), 15 deletions(-) diff --git a/suncore/app/runtime.py b/suncore/app/runtime.py index ad318e9..a7b6c68 100644 --- a/suncore/app/runtime.py +++ b/suncore/app/runtime.py @@ -91,6 +91,8 @@ """Decision-log and plan-snapshot retention run hourly.""" REBUILD_BACKOFF_S = (60.0, 300.0, 900.0, 3600.0) """Retry delays of the statistics rebuild after a timeout or Home Assistant error.""" +MAINTENANCE_WAKE_S = 3600.0 +"""The nightly loop wakes at most this often to re-check the time (zone changes, clock jumps).""" MAINTENANCE_TIME = time(3, 30) """Local time of the nightly time-series job (downsample, retention, vacuum if worth it).""" @@ -363,21 +365,38 @@ async def maintain_timeseries(self) -> bool: return False return True - def seconds_until_maintenance(self) -> float: - """Seconds to the next ``maintenance_time`` in the site's time zone (DST safe).""" + def next_maintenance_after(self, moment: datetime) -> datetime: + """The first ``maintenance_time`` (site time zone, DST safe) strictly after ``moment``.""" tz = self._timezone() - now = self.clock().astimezone(tz) - target = datetime.combine(now.date(), self.maintenance_time, tzinfo=tz) - if target.astimezone(UTC) <= now.astimezone(UTC): + local = moment.astimezone(tz) + target = datetime.combine(local.date(), self.maintenance_time, tzinfo=tz) + if target.astimezone(UTC) <= moment.astimezone(UTC): target = datetime.combine( - now.date() + timedelta(days=1), self.maintenance_time, tzinfo=tz + local.date() + timedelta(days=1), self.maintenance_time, tzinfo=tz ) - return (target.astimezone(UTC) - now.astimezone(UTC)).total_seconds() + return target.astimezone(UTC) + + def seconds_until_maintenance(self) -> float: + """Seconds to the next ``maintenance_time`` in the site's time zone.""" + now = self.clock() + return (self.next_maintenance_after(now) - now.astimezone(UTC)).total_seconds() async def _maintenance_loop(self) -> None: + """Run at start, then at every ``maintenance_time``; wakes at most hourly. + + The target is always "the first 03:30 after the last run" in the *current* time zone, so + a zone that becomes known later (Home Assistant's, after the first connect) moves the + target instead of leaving a stale sleep, and one run can never happen twice. + """ + await self.maintain_timeseries() + last_run = self.clock() while True: - await self.maintain_timeseries() # at start too: a night the app slept through - await self.sleep(self.seconds_until_maintenance()) + remaining = (self.next_maintenance_after(last_run) - self.clock()).total_seconds() + if remaining <= 0: + await self.maintain_timeseries() + last_run = self.clock() + continue + await self.sleep(min(remaining, MAINTENANCE_WAKE_S)) def _start_history_rebuild(self) -> None: """Rebuild the 15-minute history from HA's statistics, once, in the background.""" diff --git a/tests/app/test_timeseries_runtime.py b/tests/app/test_timeseries_runtime.py index 4c3b082..00dbaa5 100644 --- a/tests/app/test_timeseries_runtime.py +++ b/tests/app/test_timeseries_runtime.py @@ -365,15 +365,58 @@ async def flaky(rows: Any) -> int: # --- the nightly job --------------------------------------------------------------------------- -async def test_nightly_job_runs_at_start_then_waits_for_the_next_0330_local( +async def test_nightly_job_runs_at_start_then_wakes_at_most_hourly( gated: tuple[Runtime, GatedSleep, RecordingConnection], ) -> None: runtime, sleep, _conn = gated - # the clock is 10:00 UTC = 12:00 CEST (Europe/Berlin, the demo region); next 03:30 CEST - # is 15.5 h away - expected = 15.5 * 3600 - await until(lambda: expected in sleep.requested) - assert runtime.seconds_until_maintenance() == expected + # the clock is 10:00 UTC = 12:00 CEST (Europe/Berlin, the demo region): 15.5 h to 03:30 + await until(lambda: 3600.0 in sleep.requested) + assert runtime.seconds_until_maintenance() == 15.5 * 3600 + + +async def test_nightly_job_runs_once_per_night_and_follows_the_time_zone( + tmp_path: Path, +) -> None: + conn = RecordingConnection(speed=6000.0) + runtime = await build_runtime(demo_settings(tmp_path), env={}, connection=conn) + sleep = GatedSleep() + runtime.sleep = sleep + now = [datetime(2026, 6, 21, 10, 0, tzinfo=UTC)] # 12:00 CEST + runtime.clock = lambda: now[0] + runs: list[datetime] = [] + + async def spy(at: datetime) -> None: + runs.append(at) + + store_of(runtime).maintain = spy # type: ignore[method-assign,assignment] + try: + await runtime.start() + await until(lambda: len(runs) == 1) # at start + await until(lambda: 3600.0 in sleep.requested) + # six hourly wakes later it is 18:00 CEST: no run + for hour in range(1, 7): + now[0] += timedelta(hours=1) + await sleep.release(3600.0) + await until(lambda n=hour: len(sleep.requested) > n) + assert len(runs) == 1 + # 03:30 CEST the next day: exactly one run, and waking again does not repeat it + now[0] = datetime(2026, 6, 22, 1, 31, tzinfo=UTC) + await sleep.release(3600.0) + await until(lambda: len(runs) == 2) + await sleep.release(3600.0) + await sleep.release(3600.0) + await asyncio.sleep(0.05) + assert len(runs) == 2 + # the zone changes (e.g. Home Assistant's zone becomes known): the target moves with it + from suncore.core.config import RegionConfig # noqa: PLC0415 + + before = runtime.next_maintenance_after(now[0]) + runtime.region = RegionConfig(region_id="de", currency="EUR", timezone="Asia/Tokyo") + after = runtime.next_maintenance_after(now[0]) + assert before != after + assert after == datetime(2026, 6, 22, 18, 30, tzinfo=UTC) # 03:30 JST + finally: + await runtime.stop() @pytest.mark.parametrize( From 800e4fc6fcc1d631be32f5a56ac22aded50601d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 06:43:37 +0000 Subject: [PATCH 09/11] test(storage): hypothesis properties for the downsampling Energy (sum of mean x n and min/max) is preserved per complete quarter hour and in the 1-minute tier, downsampling is idempotent for random gaps and job times, and rows sit on bucket boundaries strictly before floor(now, 15 min). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- tests/storage/test_timeseries_properties.py | 96 +++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/storage/test_timeseries_properties.py diff --git a/tests/storage/test_timeseries_properties.py b/tests/storage/test_timeseries_properties.py new file mode 100644 index 0000000..6757ef7 --- /dev/null +++ b/tests/storage/test_timeseries_properties.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Hypothesis properties of the downsampling: energy, idempotence, bucket boundaries.""" + +from __future__ import annotations + +import asyncio +import tempfile +from collections import defaultdict +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from suncore.storage.timeseries import TimeseriesStore, open_timeseries + +T0 = datetime(2026, 10, 1, tzinfo=UTC) +T0_S = int(T0.timestamp()) + +samples = st.lists( + st.tuples( + st.sampled_from(["a", "b"]), + st.integers(min_value=0, max_value=4 * 3600), # random gaps: sparse and clustered + st.integers(min_value=-500, max_value=3000), + ), + max_size=120, + unique_by=lambda s: (s[0], s[1]), +) +job_offsets = st.integers(min_value=0, max_value=6 * 3600) + + +async def _run(rows: list[tuple[str, int, int]], jobs: list[int]) -> tuple[list, list, list]: + with tempfile.TemporaryDirectory() as tmp: + async with await open_timeseries(Path(tmp) / "t.db") as db: + store = TimeseriesStore(db) + await store.append_measurements( + [(k, T0 + timedelta(seconds=o), float(v)) for k, o, v in rows] + ) + for job in jobs: + await store.downsample(T0 + timedelta(seconds=job)) + minute = await db.fetchall("SELECT key, ts, mean, min, max, n FROM measurement_1m") + quarter = await db.fetchall("SELECT key, ts, mean, min, max, n FROM measurement_15m") + snapshot = [await db.fetchall("SELECT * FROM measurement_15m ORDER BY key, ts")] + for job in jobs: # the same jobs again: nothing may change + await store.downsample(T0 + timedelta(seconds=job)) + again = await db.fetchall("SELECT * FROM measurement_15m ORDER BY key, ts") + return minute, quarter, [snapshot[0], again] + + +def _go(rows: list[tuple[str, int, int]], jobs: list[int]) -> tuple[list, list, list]: + return asyncio.run(_run(rows, jobs)) + + +FAST = settings( + max_examples=40, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture] +) + + +@FAST +@given(rows=samples, job=job_offsets) +def test_energy_is_preserved_per_complete_quarter(rows: list, job: int) -> None: + minute, quarter, _ = _go(rows, [job]) + horizon = (T0_S + job) // 900 * 900 # complete quarter hours end at or before this + by_quarter: dict[tuple[str, int], list[int]] = defaultdict(list) + for key, offset, value in rows: + bucket = (T0_S + offset) // 900 * 900 + if bucket < horizon: + by_quarter[(key, bucket)].append(value) + stored = {(str(r[0]), int(r[1])): r for r in quarter} + assert set(stored) == set(by_quarter) + for bucket, values in by_quarter.items(): + _key, _ts, mean, low, high, n = stored[bucket] + assert n == len(values) + assert abs(mean * n - sum(values)) < 1e-6 * max(1.0, abs(sum(values))) + assert (low, high) == (min(values), max(values)) + # the 1-minute tier adds up to the same energy + total = float(sum(sum(v) for v in by_quarter.values())) + minutes_total = sum(float(r[2]) * int(r[5]) for r in minute) + assert abs(minutes_total - total) < 1e-6 * max(1.0, abs(total)) + + +@FAST +@given(rows=samples, jobs=st.lists(job_offsets, min_size=1, max_size=3)) +def test_downsampling_is_idempotent_for_any_job_times(rows: list, jobs: list[int]) -> None: + _, _, (first, again) = _go(rows, jobs) + assert first == again + + +@FAST +@given(rows=samples, job=job_offsets) +def test_rows_sit_on_bucket_boundaries_and_never_in_the_future(rows: list, job: int) -> None: + minute, quarter, _ = _go(rows, [job]) + horizon = (T0_S + job) // 900 * 900 + assert all(int(r[1]) % 60 == 0 and int(r[1]) < horizon for r in minute) + assert all(int(r[1]) % 900 == 0 and int(r[1]) < horizon for r in quarter) + assert all(int(r[5]) > 0 for r in minute + quarter) From 8250d863560fcc365e585e94447c03c51064c8e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 06:53:39 +0000 Subject: [PATCH 10/11] docs: STATUS for the reviewed PR 4-B 4-B moves to Done, header updated, two UNVERIFIED rows added (recorder/* via the proxy, change/sum of real counters and the backup_exclude glob), follow-ups noted (size_report not on a diagnostics page, a too-new timeseries.db is only logged). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- docs/STATUS.md | 50 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/docs/STATUS.md b/docs/STATUS.md index 3447378..b4ff05b 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -3,9 +3,9 @@ Read this first in every session, then `CLAUDE.md` (once it exists) and the ADRs relevant to your task. Keep it short; history lives in git. -**Last update:** 2026-10-16 · **Phase:** 4 in progress – PR 4-A contracts; Phase 3 (3-A +**Last update:** 2026-10-17 · **Phase:** 4 in progress – PR 4-A contracts, PR 4-B storage; Phase 3 (3-A `phase-3/a-safety`, 3-B `phase-3/b-controller`, 3-C `phase-3/c-ui`) implemented, waiting for the -maintainer's shadow test · **Branch:** `claude/zealous-mayer-j5oda9` +maintainer's shadow test · **Branch:** `phase-4/b-storage` (stacked on `claude/zealous-mayer-j5oda9`) ## Done @@ -29,6 +29,35 @@ maintainer's shadow test · **Branch:** `claude/zealous-mayer-j5oda9` - ADR-0013 "Battery as a controlled asset" **proposed** (prerequisite of WP10). - Review fixes: in progress (`fix(...)` commits on the branch). +- **Phase 4 – PR 4-B (WP3, storage and statistics; branch `phase-4/b-storage`, stacked on 4-A)**: + - `/data/timeseries.db` (own migrations and `user_version`; excluded from backups together with + its WAL/SHM and `timeseries.db.corrupt*`): `measurement` 48 h and `measurement_1m` 30 days + (primary key `(ts, key)`: about 5x fewer WAL bytes per flush than `(key, ts)`), `measurement_15m` + unlimited with a `derived` flag, `forecast` versioned by issue time (kept 45 days after the + slot), `price` 400 days, `meta`. `TimeseriesStore`: batched appends, series round trips, + `latest_forecast_before`, nightly job 03:30 local (wakes at most hourly, follows the time + zone): downsample (1-minute tier only for complete quarter hours), retention, `VACUUM` only + when worth it, `size_report`. + - Corruption handling: only "not a database"/"malformed"/a failing `quick_check` move the file + aside (with WAL/SHM, `timeseries.db.corrupt-`, newest two kept); locked, full, I/O or + permission errors and a file with a newer schema run SunCore without time series, nothing is + moved or deleted. + - Rebuild of 28 days of site history from `recorder/statistics_during_period` (period `hour`, + explicit units) as `derived` quarter hours (`suncore/ha/statistics.py`, + `suncore/app/history.py`), until it succeeded once (marker in `timeseries.db`), back-off 1, 5, + 15, then 60 min; `unknown_command` is final for the process. + - Controller measurements through a batched writer (flush every 60 s and at shutdown, only + OK-quality values; the tick hook cannot disturb control). + - `suncore.db` migration 0004 (`plan_snapshot` zlib JSON, 200 rows / 14 days; `learned_param`): + **DB schema 4, earlier versions refuse it – export before updating (release-risk note).** + - `ActionRateLimiter` per `domain.service[#sub-key]` for `call_action_response` (default 30 s, + `ActionRateLimitedError`, follow-up A3 done); boundary rule 7 admits only + `suncore.app.adapters`. + - Fixed: a cancelled `Database.transaction` could leave a transaction open on the shared + connection. + - Follow-ups: `size_report` is not on a diagnostics page yet; a too-new `timeseries.db` is only + logged (a UI hint follows in 4-H). + - **Fix after the first live test (2026-10-16, branch `fix/device-freshness`)**: on the maintainer's HA the SunSpec battery SoC (84 % for 25 min) and the `smartthinq_sensors` power sensors (5 W for a day) never advance `last_reported` for an unchanged value, so SunCore marked them "stale" and the @@ -309,21 +338,6 @@ maintainer's shadow test · **Branch:** `claude/zealous-mayer-j5oda9` - **ADR-0013 must be accepted before WP10** (4-G). Its contract changes (`BatteryAction` `None` = baseline, `BatteryBand.grid_charge_max_w`/`discharge_lock`, `effective_reserve_pct`, `battery_identity`) land at the start of 4-G, after the maintainer accepted it. -- **PR 4-B (WP3, branch `phase-4/b-storage`)**: `/data/timeseries.db` (own migrations; tables - `measurement` 48 h, `measurement_1m` 30 days, `measurement_15m` unlimited with a `derived` flag, - `forecast` versioned by issue time, kept 45 days after the slot, `price` 400 days; - `TimeseriesStore`; nightly job 03:30 local: downsample, retention, VACUUM only when worth it; a - corrupt file is moved aside, a newer one is left alone and SunCore runs without time series). - Rebuild of 28 days of site history from `recorder/statistics_during_period` (period `hour`, - explicit units) once after the first connect when the 15-min table is empty - (`suncore/ha/statistics.py`, `suncore/app/history.py`). Controller measurements recorded through - a batched writer (flush every 60 s, only OK-quality values). `suncore.db` migration 0004 - (`plan_snapshot` zlib JSON, 200 rows / 14 days; `learned_param`): **DB schema 4, earlier versions - refuse it – export before updating.** `ActionRateLimiter` per `domain.service` for - `call_action_response` (default 30 s, `ActionRateLimitedError`, follow-up A3); boundary rule 7 - admits only `suncore.app.adapters`. Fixed: a cancelled `Database.transaction` could leave a - transaction open on the shared connection. `UNVERIFIED`: Supervisor proxy passes `recorder/*`; - `change`/`sum` of real energy counters. - Maintainer questions from 4-A: plan §6 "Offene Fragen aus PR 4-A" (grid-charge schedule precedence, VAT on negative spot prices, export tiers, grid charging on the Kostal). @@ -516,6 +530,8 @@ maintainer's shadow test · **Branch:** `claude/zealous-mayer-j5oda9` | Kostal Modbus external battery management not enabled on the maintainer's inverter; how it is enabled; §3.5 fallback timer behaviour | `research/kostal-plenticore-modbus.md`, ADR-0013 | maintainer enables it and measures; `kostal_modbus` capped at assisted until then | | Creating a modbus hub from the HA UI (else YAML) | `research/ha-websocket-api.md` | maintainer's HA | | Supervisor proxy passes `recorder/*` and `energy/*` commands | `research/ha-websocket-api.md` | first Phase 4 run on HA OS | +| Supervisor proxy forwards `recorder/statistics_during_period` (the 4-B rebuild logs a warning and retries) | `suncore/ha/statistics.py` | first run on HA OS | +| `change`/`sum` rows of real energy counters (the rebuild reads them as verified from source, not tested against live counters); `backup_exclude` accepts the glob `timeseries.db.corrupt*` | `suncore/ha/statistics.py`, `addon/*/config.yaml` | maintainer's HA with an energy counter; a backup on HA OS | | Publication time of next-day prices (Tibber, Nord Pool, ENTSO-E, EPEX) | `research/integrations.md` | observe on the maintainer's HA | | VAT applied to negative spot prices | `suncore/core/tariff.py` | maintainer's bill / supplier terms | | KNN load model performance on aarch64 | plan WP12 | CI arm runner benchmark | From ef3aa20dd758389f1f2db5786ff7263facc1fcc0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 07:07:08 +0000 Subject: [PATCH 11/11] fix(app): release devices before awaiting storage tasks; harden the tick hook and the action call - Runtime.stop cancels the storage tasks (prune, flush, maintenance, history) but awaits them only after ControlStack.shutdown(): a running downsample or retention statement keeps the shielded rollback of its cancelled task waiting, which must not delay the device release (the gate still shuts down before the connection closes). Test with a stuck maintenance task. - ControlStack._record_measurements builds the values inside the try as well. - call_action_response checks the connection before taking a rate-limit slot. - Boundary rule 8: statistics_during_period only in suncore.ha and suncore.sim. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- suncore/app/control.py | 28 +++++++------- suncore/app/runtime.py | 17 +++++++-- suncore/ha/websocket.py | 2 + tests/app/test_timeseries_runtime.py | 55 ++++++++++++++++++++++++++++ tests/ha/test_action_rate_limit.py | 15 ++++++++ tests/safety/test_boundary.py | 14 +++++++ 6 files changed, 113 insertions(+), 18 deletions(-) diff --git a/suncore/app/control.py b/suncore/app/control.py index 8ab3a2d..79f7164 100644 --- a/suncore/app/control.py +++ b/suncore/app/control.py @@ -718,21 +718,21 @@ def _record_measurements( """Hand this tick's values to the recorder; only ``OK`` data counts (a gap otherwise).""" if self._on_measurements is None: return - values: dict[str, float | None] = {} - if flows is not None: - for key, flow in ( - (SITE_GRID_W, flows.grid_w), - (SITE_PV_W, flows.pv_w), - (SITE_BATTERY_W, flows.battery_w), - (SITE_SOC_PCT, flows.battery_soc_pct), - (SITE_HOUSE_W, flows.house_w), - ): - values[key] = flow.value if flow.quality is Quality.OK else None - for device_id, estimate in estimates.items(): - values[device_power_key(device_id)] = ( - estimate.value_w if estimate.quality is Quality.OK else None - ) try: + values: dict[str, float | None] = {} + if flows is not None: + for key, flow in ( + (SITE_GRID_W, flows.grid_w), + (SITE_PV_W, flows.pv_w), + (SITE_BATTERY_W, flows.battery_w), + (SITE_SOC_PCT, flows.battery_soc_pct), + (SITE_HOUSE_W, flows.house_w), + ): + values[key] = flow.value if flow.quality is Quality.OK else None + for device_id, estimate in estimates.items(): + values[device_power_key(device_id)] = ( + estimate.value_w if estimate.quality is Quality.OK else None + ) self._on_measurements(now, values) except Exception: log.exception("control.measurement_sink_failed") diff --git a/suncore/app/runtime.py b/suncore/app/runtime.py index a7b6c68..8b82efb 100644 --- a/suncore/app/runtime.py +++ b/suncore/app/runtime.py @@ -501,23 +501,32 @@ async def stop(self) -> None: for unsubscribe in self._unsubscribers: unsubscribe() self._unsubscribers.clear() - for task in ( - self._connect_task, + control_tasks = (self._connect_task, self._tz_task) + storage_tasks = ( self._prune_task, - self._tz_task, self._flush_task, self._maintenance_task, self._history_task, - ): + ) + for task in (*control_tasks, *storage_tasks): if task is not None: task.cancel() + for task in control_tasks: + if task is not None: with contextlib.suppress(asyncio.CancelledError, Exception): await task self._connect_task = self._prune_task = self._tz_task = None self._flush_task = self._maintenance_task = self._history_task = None + # Hardware release comes first: a running downsample or retention statement keeps the + # (shielded) rollback of its cancelled task waiting, and a Pi can need seconds for it. + # The storage tasks are awaited only after the devices are released. async with self._stack_lock: if self.stack is not None: await self.stack.shutdown() + for task in storage_tasks: + if task is not None: + with contextlib.suppress(asyncio.CancelledError, Exception): + await task await self.catalogue.stop() await self.connection.close() await self.recorder.flush() diff --git a/suncore/ha/websocket.py b/suncore/ha/websocket.py index 6a80ed9..6a2d47a 100644 --- a/suncore/ha/websocket.py +++ b/suncore/ha/websocket.py @@ -497,6 +497,8 @@ async def call_action_response( if not is_read_only_action(domain, service): msg = f"{domain}.{service} is not a read-only action" raise ActionNotAllowedError(msg) + if self._ws is None or self._ws.closed: # before a limiter slot is taken + raise NotConnectedError("not connected to Home Assistant") key = action_rate_key(domain, service, rate_key) if not await self._action_limiter.acquire(key, min_interval_s, max_wait_s=max_wait_s): raise ActionRateLimitedError(key, self._action_limiter.retry_after(key)) diff --git a/tests/app/test_timeseries_runtime.py b/tests/app/test_timeseries_runtime.py index 00dbaa5..7cfd5f7 100644 --- a/tests/app/test_timeseries_runtime.py +++ b/tests/app/test_timeseries_runtime.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import contextlib from collections.abc import AsyncIterator, Callable from datetime import UTC, datetime, timedelta from pathlib import Path @@ -630,3 +631,57 @@ async def hang(_rows: Any) -> int: assert recorder.pending == 2 # back in the buffer, to be written by the final flush del store.append_measurements assert await recorder.flush() == 2 + + +async def test_devices_are_released_before_storage_tasks_are_awaited( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + """A storage task that is slow to cancel (its rollback waits behind a long SQL statement) + must not delay the hardware release of the control stack.""" + runtime, _sleep, _conn = gated + stack = await idle(runtime) + hold = asyncio.Event() + release = asyncio.Event() + in_maintain = asyncio.Event() + shut_down = asyncio.Event() + + async def stubborn(_now: datetime) -> None: + in_maintain.set() + try: + await hold.wait() + except asyncio.CancelledError: + await release.wait() # the shielded rollback behind the running statement + raise + + store_of(runtime).maintain = stubborn # type: ignore[method-assign,assignment] + runtime._maintenance_task.cancel() # type: ignore[union-attr] + with contextlib.suppress(asyncio.CancelledError): + await runtime._maintenance_task # type: ignore[misc] + runtime._maintenance_task = asyncio.create_task(runtime._maintenance_loop()) + await in_maintain.wait() + original = stack.shutdown + + async def shutdown() -> None: + await original() + shut_down.set() + + stack.shutdown = shutdown # type: ignore[method-assign] + stopping = asyncio.create_task(runtime.stop()) + async with asyncio.timeout(5): + await shut_down.wait() # the release ran although the storage task is still stuck + assert not stopping.done() + release.set() + await stopping + + +async def test_a_failure_while_building_the_values_does_not_break_the_tick( + gated: tuple[Runtime, GatedSleep, RecordingConnection], +) -> None: + runtime, _sleep, _conn = gated + stack = await idle(runtime) + await until(lambda: stack.cache.get("sensor.sim_house_power") is not None) + stack._on_measurements = lambda _t, _v: None + stack.estimates = None # type: ignore[assignment] # not a mapping: .items() would raise + with structlog.testing.capture_logs() as logs: + stack._record_measurements(runtime.clock(), None, None) # type: ignore[arg-type] + assert any(e["event"] == "control.measurement_sink_failed" for e in logs) diff --git a/tests/ha/test_action_rate_limit.py b/tests/ha/test_action_rate_limit.py index 62a2e7e..2bdf5a3 100644 --- a/tests/ha/test_action_rate_limit.py +++ b/tests/ha/test_action_rate_limit.py @@ -279,3 +279,18 @@ async def test_the_fake_connection_is_limited_like_the_real_one() -> None: await fake.call_action_response("tibber", "get_prices") finally: await fake.close() + + +async def test_not_connected_does_not_consume_a_limiter_slot(ha: FakeHA, clock: FakeClock) -> None: + from suncore.ha.connection import NotConnectedError # noqa: PLC0415 + + connection = WebSocketConnection( + DirectTransport.from_base_url(ha.base_url, TOKEN), + clock=clock.monotonic, + sleep=clock.sleep, + ) + with pytest.raises(NotConnectedError): + await connection.call_action_response("tibber", "get_prices") + assert connection._action_limiter.retry_after("tibber.get_prices") == 0.0 + with pytest.raises(NotConnectedError): # and not "rate limited" the second time + await connection.call_action_response("tibber", "get_prices") diff --git a/tests/safety/test_boundary.py b/tests/safety/test_boundary.py index 49dacf5..28daa62 100644 --- a/tests/safety/test_boundary.py +++ b/tests/safety/test_boundary.py @@ -34,6 +34,9 @@ the one adapter wiring module ``suncore.app.adapters`` (Phase 4); no other part of ``suncore.app`` may use it. ``READ_ONLY_ACTIONS`` is the fixed allowlist. +8. ``statistics_during_period`` (read-only recorder statistics) is used only in ``suncore.ha``, + and ``suncore.sim`` (``suncore.app.history`` goes through ``suncore.ha.statistics``). + The scan is robust to packages that are still empty (``suncore/ha`` until WP3) because it only looks at the files that exist. """ @@ -64,6 +67,7 @@ API_FORBIDDEN = (SAFETY, HA, "suncore.devices") WRITE_CAPABILITY_NAMES = frozenset({"ServiceCaller", "ServiceCallerProvider"}) SAFETY_FORBIDDEN_IMPORTS = ("suncore.core.config", "suncore.storage") +STATISTICS_OWNERS = ("suncore.ha", "suncore.sim") ACTION_RESPONSE_OWNERS = ("suncore.ha", "suncore.sim", "suncore.app.adapters") @@ -336,6 +340,16 @@ def test_call_action_response_is_confined(path: Path) -> None: ) +@pytest.mark.parametrize("path", SOURCES, ids=lambda p: str(p.relative_to(ROOT))) +def test_statistics_during_period_is_confined(path: Path) -> None: + """Rule 8: the recorder statistics read lives in ha (+ the fake) only.""" + module = module_name(path) + if any(in_package(module, owner) for owner in STATISTICS_OWNERS): + return + offenders = fmt(path, attribute_refs(parse(path), "statistics_during_period")) + assert not offenders, f"statistics_during_period referenced outside ha/sim: {offenders}" + + def test_action_response_owner_is_the_adapter_module_not_the_whole_app() -> None: """Rule 7 admits ``suncore.app.adapters`` only: its siblings in ``suncore.app`` stay out.""" assert "suncore.app" not in ACTION_RESPONSE_OWNERS