diff --git a/docs/STATUS.md b/docs/STATUS.md index 378b77f..c19975d 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -3,10 +3,10 @@ Read this first in every session, then `CLAUDE.md` (once it exists) and the ADRs relevant to your task. Keep it short; history lives in git. -**Last update:** 2026-10-19 · **Phase:** 4 in progress – PR 4-A contracts, PR 4-B storage, PR 4-D PV -forecast; Phase 3 (3-A `phase-3/a-safety`, 3-B `phase-3/b-controller`, 3-C `phase-3/c-ui`) -implemented, waiting for the maintainer's shadow test · **Branch:** `phase-4/d-pv-forecast` -(stacked on `phase-4/b-storage`) +**Last update:** 2026-10-22 · **Phase:** 4 in progress – PR 4-A contracts, PR 4-B storage, PR 4-D PV +forecast, PR 4-E load forecast and risk model; Phase 3 (3-A `phase-3/a-safety`, 3-B +`phase-3/b-controller`, 3-C `phase-3/c-ui`) implemented, waiting for the maintainer's shadow test · +**Branch:** `phase-4/e-load-risk` (stacked on `phase-4/d-pv-forecast`) ## Done @@ -97,6 +97,56 @@ implemented, waiting for the maintainer's shadow test · **Branch:** `phase-4/d- - Follow-ups: the weights come from the *uncalibrated* sources' MAE (a calibrated source is under-weighted until a calibrated accuracy is scored); no UI yet (4-H). +- **Phase 4 – PR 4-E (WP6 load forecast, WP7 risk model; branch `phase-4/e-load-risk`, stacked on + 4-D)**: + - WP6 `suncore/forecast/load.py` + `load_service.py`: base load = `site.house_w` minus every + configured device's `device..power_w` (the history rebuild also fetches the devices' power + sensors; unmeasured quarter hours count 0 W, are reported per device and are left out of a day + type's profile once it has 7 fully measured days, else kept and flagged; negative rests + clipped); profiles per day type (workday/weekend/holiday) × 96 wall-clock quarter hours, + exponential smoothing (half-life 21 days, 56 days look-back), clipped z-score (3 × max(MAD + scale, ½ median)), shrinkage of thin day types (holiday → weekend → all non-holiday days), + kernel smoothing; optional temperature term (heating/cooling degrees 15/22 °C, bounded 0–300 + W/K, active after 14 days with a recorded temperature, offset ≤ 50 % of the day's mean, only for + forecast days with ≥ 20 h of temperatures); residual P10/P90 per time-of-day block; cold start + (< 3 days: flat at the 7-day mean, P10 0.5×, P90 2×), `no_history` without data. Day types: the + `workday` sensor's state for today + `days_offset`, `workday.check_date(d − days_offset)` for + other dates (Home Assistant adds the offset; ≤ 12 calls per refresh, answers cached in + `learned_param` `load_profile/day_types` with a fingerprint of entity, `workdays`, `excludes` + and offset, dropped when it changes), else the weekday rule. Refresh every 15 min, refit + hourly/after the history rebuild/nightly, stored hourly as kind `load` source `load_profile`, + accuracy (all quarter hours) in the nightly job; `GET /api/v1/forecast/load?hours=48`; the + controlled devices' histories are kept for 4-F. Benchmark (n5): a refit (read 56 days of house, + 3 devices and temperature from SQLite + fit) takes 118–163 ms on the session's amd64 container + (4 vCPU), the forecast itself about 1 ms; aarch64 not measured yet. + - Config: `forecast.load.workday` / `forecast.load.weather` (additive in `forecast` v2, domain + checked; the wizard form carries them through, UI in 4-H). `workday.check_date` joined + `READ_ONLY_ACTIONS`. The weather entity's `temperature` is recorded as + `weather.temperature_c` (HA keeps no statistics of weather entities). + - WP7 `suncore/forecast/risk.py` + `risk_learning.py`: windows `rest_of_night` (before + sunrise), `rest_of_today`, `night`, `next_day`, `bridge` (night + next day, one quantile) and + `day_and_night` (coming daytime + the night after it) anchored on today's sunset from the PV + forecast; weather class from the PV clear-sky index; net energy quantile `learned` (relative + errors per window × class, pooled per window when thin) from 30 days with samples, else + `source_quantiles` (two-piece normal, PV and load independent, slot quantiles summed) or + `default_band` (PV forecast without a band: default relative band per weather class); + `evening_target_pct`, `morning_buffer_pct` (one quantile over `day_and_night` against the + median night need), floor `core.battery.effective_reserve_pct` (ADR-0013), `prefer_full` = + max SoC, manual limits and max SoC always win; `goals.battery_target` is a floor of the + evening target, its time is ignored there (timed constraint in 4-F; maintainer 2026-10-22). + Reasons `planner.battery.morning_buffer` / `.evening_target` (en/de, naming the risk level). + Nightly samples from the issues at sunrise + 1 h and sunset − 1 h against measured PV and + base load, only for windows both forecasts cover, without clipped daytime quarter hours, + kept 365 days (`learned_param` `net_error`). `GET /api/v1/forecast/risk`. Hypothesis tests + (monotone in risk level, in PV and load uncertainty and on the learned path, limits incl. + the site min SoC) and the four scenarios. + - Review fixes (M1–M5, m2–m8, n1–n6) on 2026-10-22/23, see the plan's deviations. + - Demo: deterministic workday sensor and weather entity (`binary_sensor.sim_workday_sensor`, + `weather.sim_home`, both actions answered), battery model 10 kWh / 5 kW (still observe only). + - Decisions and deviations: plan §"Deviations during implementation" (PR 4-E). + - Follow-ups: the planner (4-F) consumes `morning_buffer_pct`/`evening_target_pct` and the + device histories; no UI yet (4-H); ML load model in shadow (4-I). + - **Fix after the first live test (2026-10-16, branch `fix/device-freshness`)**: on the maintainer's HA the SunSpec battery SoC (84 % for 25 min) and the `smartthinq_sensors` power sensors (5 W for a day) never advance `last_reported` for an unchanged value, so SunCore marked them "stale" and the @@ -593,6 +643,10 @@ implemented, waiting for the maintainer's shadow test · **Branch:** `phase-4/d- | Supervisor proxy forwards `energy/solar_forecast` | `suncore/ha/` (`fetch_solar_forecast`) | first run on HA OS | | PV calibration constants (classes kt 0.3/0.65, shrink k = 16, decay 0.98, factor 0.5–1.5) and the clipping thresholds (plateau ±0.3 % for ≥ 1 h on ≥ 2 days; export within max(3 %, 50 W)) are reasoned, not fitted to real data | `suncore/forecast/pv/` | 30+ days of the maintainer's data | | Solve times on Raspberry Pi 4 | ADR-0003 | Phase 4 benchmark | +| `weather.get_forecasts` `hourly` support of the maintainer's weather integration (the `temperature_unit` attribute and the conversion of the forecast to it are verified from source, `research/ha-websocket-api.md`) | `suncore/app/adapters.py` (`HaLoadReader`) | maintainer's HA | +| `workday.check_date` over the Supervisor proxy and with the maintainer's Workday configuration (the `days_offset` handling is verified from source; a changed country/holiday list is not visible in the attributes, so the day-type cache only resets on a changed entity, `workdays`, `excludes` or offset) | `suncore/app/adapters.py`, `suncore/forecast/load_service.py` | maintainer maps a Workday sensor | +| Load and risk constants (half-life 21 days, outlier z 3, shrink 3 days, 7 clean days, degree-day bases 15/22 °C, coverage 75 %/90 %, 20 h temperature days, anchors sunrise + 1 h / sunset − 1 h, sunrise threshold 2 %/20 W) and the default PV bands per weather class (clear 0.85/1.10, mixed 0.55/1.35, overcast 0.35/1.70) are reasoned, not fitted | `suncore/forecast/load.py`, `suncore/forecast/risk.py`, `risk_learning.py` | 30+ days of the maintainer's data | +| Known limitation (n6): the load history and the risk samples are not re-derived when the device list changes - a device added later is only subtracted from the base load from the day its power is recorded (or rebuilt), and older risk samples keep the old base load | `suncore/forecast/load_service.py`, `risk_learning.py` | accepted for Phase 4; revisit with 4-I | ## Environment notes diff --git a/docs/plans/phase-4.md b/docs/plans/phase-4.md index d0a850d..74a9db8 100644 --- a/docs/plans/phase-4.md +++ b/docs/plans/phase-4.md @@ -410,6 +410,47 @@ shadow and assisted run is the acceptance test. Open-Meteo Solar Forecast, generic template). - `BatteryAction` is part of the 4-A contracts but rejected by the gate until WP10 (ADR-0011 addendum); its semantics are proposed in ADR-0013. +- PR 4-E (WP6, WP7): + - The optional `workday` binary sensor and `weather` entity of the load forecast live in + `forecast.load` (additive in `forecast` v2, no migration; the wizard carries them through, + the UI is 4-H). `workday.check_date` joined the read-only action allowlist (verified in + `research/ha-websocket-api.md` "Workday"). + - Home Assistant keeps no long-term statistics of a weather entity, so the temperature term + learns from SunCore's own recording of the entity's `temperature` attribute (key + `weather.temperature_c`) and becomes active after 14 days with a temperature. + - The risk model has a fourth window `bridge` (night + next day, sunset to sunset): the + evening target takes **one** quantile of it instead of adding the night's and the next + day's quantiles (which would count caution twice, §0). + - Learnt errors are relative to the window's forecast energy (PV P50 + load P50), so a short + `rest_of_today` and a long one, a summer and a winter day share one distribution. Samples + are taken from the forecasts as they stood at sunrise + 1 h (`rest_of_today`, + `day_and_night`) and sunset − 1 h (`night`, `next_day`, `bridge`) and kept 365 days (review + m3: fixed 06:00/18:00 anchors meant different situations in winter and summer). A window is + learnt only if the PV and load forecasts cover it (≥ 90 %); without a PV forecast only the + night; daytime windows with a clipped quarter hour (the 4-D clipping flags) are dropped. + - Before sunrise the rest of the night is its own window `rest_of_night` (review m2). + - Morning buffer (review m7, replaces the first rule with a comfort factor 2): **one** quantile + over `day_and_night` (the coming daytime and the night after it) gives the margin left after + both are covered; `share = clamp(1 − margin / median night need, 0, 1)` and + `X = floor + share × (evening target − floor)`. The evening target is only the upper bound + of X, so no caution is counted twice. + - Floor of buffer and target: `core.battery.effective_reserve_pct` (ADR-0013 decision 3; + review M4), never below the site battery's minimum SoC. + - A PV forecast without a band (P10 = P50) gets a default relative band per weather class + (`default_band`, `UNVERIFIED` constants) until the learnt distribution takes over (review + M5). + - `goals.battery_target` is a floor of the evening target; its **time** is ignored there, the + timed constraint follows in the planner (4-F) – confirmed by the maintainer on 2026-10-22. + A Monday–Friday holiday counts as a weekend day while few holidays were seen (shrinkage + towards the weekend profile) – also confirmed. + - Workday offset (review M1): Home Assistant evaluates the sensor and `workday.check_date` + for the date plus `days_offset` (`workday/entity.py`); SunCore places the state on + today + offset and asks `check_date(d − offset)`. Cached answers carry a fingerprint + (entity, `workdays`, `excludes`, `days_offset`) and are dropped when it changes (m6). + - Devices in the base load (review M2): the history rebuild also fetches the devices' power + sensors (`mean`, W); quarter hours with an unmeasured device are left out of a day type's + profile once it has 7 fully measured days, before that they are kept and flagged. + - The temperature term shifts a forecast day only with ≥ 20 hours of temperatures (m5). ## 7. Out of scope for Phase 4 diff --git a/docs/research/ha-websocket-api.md b/docs/research/ha-websocket-api.md index 0c340f3..d58fac2 100644 --- a/docs/research/ha-websocket-api.md +++ b/docs/research/ha-websocket-api.md @@ -299,9 +299,18 @@ files `homeassistant/components/weather/{services.py,services.yaml,const.py,enti string in practice, `ATTR_FORECAST_TIME = "datetime"`). `temperature` is set to `null` when the entity delivered none. Keys the integration does not provide are simply absent: the adapter must treat every key but `datetime` as - optional. The temperature unit is **not** in the response; read - `unit_of_measurement`-like info from the entity's `temperature_unit` state - attribute (not read in this pass, `UNVERIFIED`). + optional. The temperature unit is **not** in the response; it is the + entity's `temperature_unit` state attribute. Verified 2026-10-23 at the same + commit: `weather/entity.py` `state_attributes` always sets + `temperature_unit = self._temperature_unit` (line 525, key `temperature_unit` + from `weather/const.py:86/104`) and `_convert_forecast` converts every + forecast temperature to the same `_temperature_unit` (lines 608–634); values + are `UnitOfTemperature` (`°C`, `°F`, `K`, `homeassistant/const.py:587`). +- **Workday offset** (`workday/entity.py:103–115`, same commit): `check_date` + returns `date_is_workday(check_date)`, which evaluates + `adjusted_date = check_date + timedelta(days=days_offset)`; the sensor state + is the same function for today. SunCore asks `check_date(d - days_offset)` + for date `d` and places the state on `today + days_offset`. - Whether `hourly` is offered depends on the integration (`supported_features` bit `FORECAST_HOURLY = 2`); not every weather entity has it. Fall back to `daily` (a coarser temperature profile) or no temperature term. @@ -921,8 +930,8 @@ native range, `select` still validates against `options`); `switch.turn_on/off/t take no fields. Still `UNVERIFIED`: the Supervisor proxy forwarding of `recorder/*` and -`energy/*` commands; the temperature unit of a `weather.get_forecasts` response -(taken from the entity's attributes, not read); the vendor meaning of the +`energy/*` commands (the temperature unit of a `weather.get_forecasts` response +was verified on 2026-10-23, see "Weather forecasts"); the vendor meaning of the Forecast.Solar `watt_hours_period` keys (start vs end of the period, API unreachable); the HA error for an unknown Modbus hub name; whether a plain Modbus hub can be added in the UI. diff --git a/frontend/src/lib/api/openapi.json b/frontend/src/lib/api/openapi.json index 642e270..2df094b 100644 --- a/frontend/src/lib/api/openapi.json +++ b/frontend/src/lib/api/openapi.json @@ -1369,6 +1369,25 @@ "title": "ControlState", "type": "object" }, + "DayType": { + "enum": [ + "workday", + "weekend", + "holiday" + ], + "title": "DayType", + "type": "string" + }, + "DayTypeSource": { + "description": "Where the day type of a date came from.", + "enum": [ + "workday_sensor", + "check_date", + "weekday_rule" + ], + "title": "DayTypeSource", + "type": "string" + }, "Decision": { "additionalProperties": false, "description": "The safety gate's verdict on an intent, as stored in the decision log.", @@ -2409,6 +2428,31 @@ "title": "EnergyPrefsOut", "type": "object" }, + "EnergyView": { + "additionalProperties": false, + "description": "P10/P50/P90 energy (Wh) of a window (the slot quantiles summed).", + "properties": { + "p10_wh": { + "title": "P10 Wh", + "type": "number" + }, + "p50_wh": { + "title": "P50 Wh", + "type": "number" + }, + "p90_wh": { + "title": "P90 Wh", + "type": "number" + } + }, + "required": [ + "p10_wh", + "p50_wh", + "p90_wh" + ], + "title": "EnergyView", + "type": "object" + }, "EntityCandidateOut": { "additionalProperties": false, "description": "An entity offered for a role.\n\n``device_class`` and ``unit`` are empty strings when unknown. ``score`` is 0 and\n``criteria`` empty for the unranked Phase 1 list. ``value`` is the raw state in ``unit``\n(no conversion, no sign inversion).", @@ -3000,7 +3044,7 @@ }, "ForecastConfig": { "additionalProperties": false, - "description": "PV forecast sources (schema v2: several sources, blending, learning settings).\n\nIndependent of the tariff so that the wizard can store the choice without prices.\nForecast uncertainty is handled by the learned risk model and the battery section's\n``risk`` level, not by a flat discount.", + "description": "PV forecast sources (schema v2: several sources, blending, learning settings) and the\noptional inputs of the load forecast (``load``, added in Phase 4 WP6 without a version\nbump: an absent ``load`` is the default, so every stored v2 document stays valid).\n\nIndependent of the tariff so that the wizard can store the choice without prices.\nForecast uncertainty is handled by the learned risk model and the battery section's\n``risk`` level, not by a flat discount.", "properties": { "learning": { "$ref": "#/components/schemas/LearningSettings", @@ -3009,6 +3053,10 @@ "min_history_days": 30 } }, + "load": { + "$ref": "#/components/schemas/LoadForecastConfig", + "default": {} + }, "schema_version": { "default": 2, "title": "Schema Version", @@ -3796,6 +3844,273 @@ } ] }, + "LoadDayView": { + "additionalProperties": false, + "description": "The day type (and forecast mean temperature) of one local date of the forecast.", + "properties": { + "date": { + "format": "date", + "title": "Date", + "type": "string" + }, + "day_type": { + "$ref": "#/components/schemas/DayType" + }, + "source": { + "$ref": "#/components/schemas/DayTypeSource" + }, + "temperature_c": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Temperature C" + } + }, + "required": [ + "date", + "day_type", + "source", + "temperature_c" + ], + "title": "LoadDayView", + "type": "object" + }, + "LoadForecastConfig": { + "additionalProperties": false, + "description": "Optional inputs of the load forecast (Phase 4 WP6; additive in ``forecast`` v2).\n\n``workday``: a ``binary_sensor`` of the ``workday`` integration (``on`` = today is a\nworkday); with it the day types come from the user's own holiday calendar (today from the\nstate, other days through ``workday.check_date``), without it from the weekday (Mon-Fri\nworkday, Sat/Sun weekend, no holidays). ``weather``: a ``weather`` entity whose outside\ntemperature (state attribute ``temperature``) is recorded and whose hourly forecast\n(``weather.get_forecasts``) drives the temperature term of the load profile.", + "properties": { + "weather": { + "anyOf": [ + { + "$ref": "#/components/schemas/EntityRef" + }, + { + "type": "null" + } + ] + }, + "workday": { + "anyOf": [ + { + "$ref": "#/components/schemas/EntityRef" + }, + { + "type": "null" + } + ] + } + }, + "title": "LoadForecastConfig", + "type": "object" + }, + "LoadForecastView": { + "additionalProperties": false, + "description": "The base load forecast (W, P10/P50/P90 on the 15-minute grid from the current slot).\n\nThe base load is the house consumption without the configured (controllable) devices; their\ndraw is planned separately. ``quality``: ``ok`` (profiles from history), ``cold_start``\n(fewer than 3 days: a flat line at the recent mean with a wide band) or ``no_history``\n(nothing measured yet: no forecast).", + "properties": { + "clamped_slots": { + "title": "Clamped Slots", + "type": "integer" + }, + "controlled_devices": { + "items": { + "type": "string" + }, + "title": "Controlled Devices", + "type": "array" + }, + "days": { + "items": { + "$ref": "#/components/schemas/LoadDayView" + }, + "title": "Days", + "type": "array" + }, + "days_of_history": { + "title": "Days Of History", + "type": "integer" + }, + "devices_unmeasured": { + "additionalProperties": { + "type": "integer" + }, + "title": "Devices Unmeasured", + "type": "object" + }, + "fitted_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Fitted At" + }, + "generated_at": { + "format": "date-time", + "title": "Generated At", + "type": "string" + }, + "hours": { + "title": "Hours", + "type": "integer" + }, + "p10_w": { + "items": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "title": "P10 W", + "type": "array" + }, + "p50_w": { + "items": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "title": "P50 W", + "type": "array" + }, + "p90_w": { + "items": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "title": "P90 W", + "type": "array" + }, + "quality": { + "$ref": "#/components/schemas/LoadQuality" + }, + "source_id": { + "title": "Source Id", + "type": "string" + }, + "start": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Start" + }, + "step_s": { + "exclusiveMinimum": 0.0, + "title": "Step S", + "type": "integer" + }, + "temperature": { + "$ref": "#/components/schemas/LoadTemperatureView" + }, + "unmeasured_in_profile": { + "items": { + "$ref": "#/components/schemas/DayType" + }, + "title": "Unmeasured In Profile", + "type": "array" + } + }, + "required": [ + "generated_at", + "hours", + "source_id", + "quality", + "days_of_history", + "fitted_at", + "start", + "step_s", + "p10_w", + "p50_w", + "p90_w", + "days", + "temperature", + "controlled_devices", + "devices_unmeasured", + "clamped_slots", + "unmeasured_in_profile" + ], + "title": "LoadForecastView", + "type": "object" + }, + "LoadQuality": { + "enum": [ + "ok", + "cold_start", + "no_history" + ], + "title": "LoadQuality", + "type": "string" + }, + "LoadTemperatureView": { + "additionalProperties": false, + "description": "The temperature term of the load profile.\n\n``reason``: ``active``, ``not_configured`` (no weather entity), ``too_few_days`` (fewer\nthan 14 days with a recorded temperature) or ``no_spread`` (the temperatures varied too\nlittle to learn anything).", + "properties": { + "active": { + "title": "Active", + "type": "boolean" + }, + "cooling_w_per_k": { + "title": "Cooling W Per K", + "type": "number" + }, + "days": { + "title": "Days", + "type": "integer" + }, + "heating_w_per_k": { + "title": "Heating W Per K", + "type": "number" + }, + "reason": { + "enum": [ + "active", + "not_configured", + "too_few_days", + "no_spread" + ], + "title": "Reason", + "type": "string" + } + }, + "required": [ + "active", + "reason", + "days", + "heating_w_per_k", + "cooling_w_per_k" + ], + "title": "LoadTemperatureView", + "type": "object" + }, "Mode": { "additionalProperties": false, "description": "One mode of a mode-controlled device and the power it draws.", @@ -5070,6 +5385,242 @@ "title": "RiskLevel", "type": "string" }, + "RiskTargetView": { + "additionalProperties": false, + "description": "A computed state of charge with its structured reason.", + "properties": { + "reason": { + "$ref": "#/components/schemas/Reason" + }, + "soc_pct": { + "title": "Soc Pct", + "type": "number" + }, + "source": { + "enum": [ + "learned", + "source_quantiles", + "default_band" + ], + "title": "Source", + "type": "string" + } + }, + "required": [ + "soc_pct", + "reason", + "source" + ], + "title": "RiskTargetView", + "type": "object" + }, + "RiskView": { + "additionalProperties": false, + "description": "The current morning buffer and evening target and what they rest on (Phase 4 WP7).\n\n``status``: ``ok``; ``no_battery_model`` (the ``battery`` section has no model yet: the\nwindows are shown, the targets are ``null``); ``no_forecast`` (no load forecast yet).\n``distribution_source``: ``learned`` once ``min_history_days`` days of measured net errors\nexist, else ``source_quantiles`` (the PV sources' P10/P90 and the load residuals).", + "properties": { + "bridge_next_day": { + "title": "Bridge Next Day", + "type": "boolean" + }, + "distribution_source": { + "enum": [ + "learned", + "source_quantiles", + "default_band" + ], + "title": "Distribution Source", + "type": "string" + }, + "evening_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Evening At" + }, + "evening_target": { + "anyOf": [ + { + "$ref": "#/components/schemas/RiskTargetView" + }, + { + "type": "null" + } + ] + }, + "generated_at": { + "format": "date-time", + "title": "Generated At", + "type": "string" + }, + "history_days": { + "title": "History Days", + "type": "integer" + }, + "min_history_days": { + "title": "Min History Days", + "type": "integer" + }, + "morning_buffer": { + "anyOf": [ + { + "$ref": "#/components/schemas/RiskTargetView" + }, + { + "type": "null" + } + ] + }, + "pv_forecast": { + "title": "Pv Forecast", + "type": "boolean" + }, + "quantile": { + "title": "Quantile", + "type": "number" + }, + "risk": { + "$ref": "#/components/schemas/RiskLevel" + }, + "samples": { + "additionalProperties": { + "type": "integer" + }, + "title": "Samples", + "type": "object" + }, + "status": { + "enum": [ + "ok", + "no_battery_model", + "no_forecast" + ], + "title": "Status", + "type": "string" + }, + "strategy": { + "$ref": "#/components/schemas/BatteryStrategy" + }, + "windows": { + "items": { + "$ref": "#/components/schemas/RiskWindowView" + }, + "title": "Windows", + "type": "array" + } + }, + "required": [ + "generated_at", + "status", + "risk", + "quantile", + "strategy", + "bridge_next_day", + "pv_forecast", + "distribution_source", + "history_days", + "min_history_days", + "samples", + "evening_at", + "windows", + "morning_buffer", + "evening_target" + ], + "title": "RiskView", + "type": "object" + }, + "RiskWindow": { + "enum": [ + "rest_of_night", + "rest_of_today", + "night", + "next_day", + "bridge", + "day_and_night" + ], + "title": "RiskWindow", + "type": "string" + }, + "RiskWindowView": { + "additionalProperties": false, + "description": "One window of the risk model and its net energy (PV - base load) at the risk quantile.", + "properties": { + "end": { + "format": "date-time", + "title": "End", + "type": "string" + }, + "load": { + "$ref": "#/components/schemas/EnergyView" + }, + "net_p50_wh": { + "title": "Net P50 Wh", + "type": "number" + }, + "net_quantile_wh": { + "title": "Net Quantile Wh", + "type": "number" + }, + "pooled": { + "title": "Pooled", + "type": "boolean" + }, + "pv": { + "$ref": "#/components/schemas/EnergyView" + }, + "samples": { + "title": "Samples", + "type": "integer" + }, + "source": { + "enum": [ + "learned", + "source_quantiles", + "default_band" + ], + "title": "Source", + "type": "string" + }, + "start": { + "format": "date-time", + "title": "Start", + "type": "string" + }, + "weather": { + "enum": [ + "overcast", + "mixed", + "clear", + "night" + ], + "title": "Weather", + "type": "string" + }, + "window": { + "$ref": "#/components/schemas/RiskWindow" + } + }, + "required": [ + "window", + "start", + "end", + "weather", + "pv", + "load", + "net_p50_wh", + "net_quantile_wh", + "source", + "samples", + "pooled" + ], + "title": "RiskWindowView", + "type": "object" + }, "RoleSpec": { "additionalProperties": false, "description": "One entity role of a template.", @@ -7484,7 +8035,7 @@ }, "/api/v1/forecast/accuracy": { "get": { - "description": "MAE and bias per source and horizon bucket (0-6 h, 6-24 h, 24-48 h) over the last 14\ndays, plus the source id ``blended`` (the blended forecast); ``bias_w = actual -\nforecast`` (positive: the source under-forecasts); ``n`` counts 15-minute intervals;\nclipped quarter hours are excluded; computed nightly.", + "description": "MAE and bias per source and horizon bucket (0-6 h, 6-24 h, 24-48 h) over the last 14\ndays, plus the source id ``blended`` (the blended forecast); ``bias_w = actual -\nforecast`` (positive: the source under-forecasts); ``n`` counts 15-minute intervals;\nclipped quarter hours are excluded; computed nightly. The source id ``load_profile`` is\nthe base load forecast (scored on every quarter hour, not only in daylight).", "operationId": "forecast_accuracy_api_v1_forecast_accuracy_get", "responses": { "200": { @@ -7501,6 +8052,49 @@ "summary": "Forecast Accuracy" } }, + "/api/v1/forecast/load": { + "get": { + "description": "The base load forecast (house consumption without the configured devices; W,\nP10/P50/P90 on the 15-minute grid from the current quarter hour), learnt from the\nhistory per day type (workday/weekend/holiday) with an optional temperature term; its\nquality (``ok``, ``cold_start``, ``no_history``), the day type of every forecast date\nand where it came from, and how many quarter hours lacked a device measurement.", + "operationId": "forecast_load_api_v1_forecast_load_get", + "parameters": [ + { + "in": "query", + "name": "hours", + "required": false, + "schema": { + "default": 48, + "maximum": 96, + "minimum": 1, + "title": "Hours", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LoadForecastView" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationErrorOut" + } + } + }, + "description": "Validation error" + } + }, + "summary": "Forecast Load" + } + }, "/api/v1/forecast/pv": { "get": { "description": "Per source the latest good P10/P50/P90 in W from the current quarter hour on, with\nits freshness, and ``blended``: the forecast the planner uses (sources bias-calibrated\nwhere active, blended by inverse 14-day MAE), its weights per horizon bucket, the\ncalibration state per source, and the blend without and with every learnt\ncalibration for comparison.", @@ -7544,6 +8138,25 @@ "summary": "Forecast Pv" } }, + "/api/v1/forecast/risk": { + "get": { + "description": "The risk model now: the windows ``rest_of_today``, ``night``, ``next_day`` and\n``bridge`` (night + next day) with PV, base load and net energy (P50 and at the risk\nlevel's quantile), the morning buffer (\"battery first until X %\") and the evening\ntarget with their structured reasons, and whether the error distribution is\n``learned`` (from ``min_history_days`` days of measured net errors) or still taken from\nthe ``source_quantiles``. Targets are ``null`` until the ``battery`` section has a\nmodel.", + "operationId": "forecast_risk_api_v1_forecast_risk_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RiskView" + } + } + }, + "description": "Successful Response" + } + }, + "summary": "Forecast Risk" + } + }, "/api/v1/info": { "get": { "operationId": "info_api_v1_info_get", diff --git a/frontend/src/lib/api/schema.d.ts b/frontend/src/lib/api/schema.d.ts index 414ab59..b34605e 100644 --- a/frontend/src/lib/api/schema.d.ts +++ b/frontend/src/lib/api/schema.d.ts @@ -381,7 +381,8 @@ export interface paths { * @description MAE and bias per source and horizon bucket (0-6 h, 6-24 h, 24-48 h) over the last 14 * days, plus the source id ``blended`` (the blended forecast); ``bias_w = actual - * forecast`` (positive: the source under-forecasts); ``n`` counts 15-minute intervals; - * clipped quarter hours are excluded; computed nightly. + * clipped quarter hours are excluded; computed nightly. The source id ``load_profile`` is + * the base load forecast (scored on every quarter hour, not only in daylight). */ get: operations["forecast_accuracy_api_v1_forecast_accuracy_get"]; put?: never; @@ -392,6 +393,30 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/forecast/load": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Forecast Load + * @description The base load forecast (house consumption without the configured devices; W, + * P10/P50/P90 on the 15-minute grid from the current quarter hour), learnt from the + * history per day type (workday/weekend/holiday) with an optional temperature term; its + * quality (``ok``, ``cold_start``, ``no_history``), the day type of every forecast date + * and where it came from, and how many quarter hours lacked a device measurement. + */ + get: operations["forecast_load_api_v1_forecast_load_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/forecast/pv": { parameters: { query?: never; @@ -416,6 +441,32 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/forecast/risk": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Forecast Risk + * @description The risk model now: the windows ``rest_of_today``, ``night``, ``next_day`` and + * ``bridge`` (night + next day) with PV, base load and net energy (P50 and at the risk + * level's quantile), the morning buffer ("battery first until X %") and the evening + * target with their structured reasons, and whether the error distribution is + * ``learned`` (from ``min_history_days`` days of measured net errors) or still taken from + * the ``source_quantiles``. Targets are ``null`` until the ``battery`` section has a + * model. + */ + get: operations["forecast_risk_api_v1_forecast_risk_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/info": { parameters: { query?: never; @@ -1224,6 +1275,17 @@ export interface components { /** Ts */ ts: string | null; }; + /** + * DayType + * @enum {string} + */ + DayType: "workday" | "weekend" | "holiday"; + /** + * DayTypeSource + * @description Where the day type of a date came from. + * @enum {string} + */ + DayTypeSource: "workday_sensor" | "check_date" | "weekday_rule"; /** * Decision * @description The safety gate's verdict on an intent, as stored in the decision log. @@ -1557,6 +1619,18 @@ export interface components { /** Suggestions */ suggestions: components["schemas"]["SuggestionOut"][]; }; + /** + * EnergyView + * @description P10/P50/P90 energy (Wh) of a window (the slot quantiles summed). + */ + EnergyView: { + /** P10 Wh */ + p10_wh: number; + /** P50 Wh */ + p50_wh: number; + /** P90 Wh */ + p90_wh: number; + }; /** * EntityCandidateOut * @description An entity offered for a role. @@ -1768,7 +1842,9 @@ export interface components { }; /** * ForecastConfig - * @description PV forecast sources (schema v2: several sources, blending, learning settings). + * @description PV forecast sources (schema v2: several sources, blending, learning settings) and the + * optional inputs of the load forecast (``load``, added in Phase 4 WP6 without a version + * bump: an absent ``load`` is the default, so every stored v2 document stays valid). * * Independent of the tariff so that the wizard can store the choice without prices. * Forecast uncertainty is handled by the learned risk model and the battery section's @@ -1782,6 +1858,8 @@ export interface components { * } */ learning: components["schemas"]["LearningSettings"]; + /** @default {} */ + load: components["schemas"]["LoadForecastConfig"]; /** * Schema Version * @default 2 @@ -2071,6 +2149,112 @@ export interface components { min_history_days: number; }; LiveMessage: components["schemas"]["SnapshotMessage"] | components["schemas"]["FlowsMessage"] | components["schemas"]["HealthMessage"] | components["schemas"]["DecisionsMessage"] | components["schemas"]["ControlMessage"]; + /** + * LoadDayView + * @description The day type (and forecast mean temperature) of one local date of the forecast. + */ + LoadDayView: { + /** + * Date + * Format: date + */ + date: string; + day_type: components["schemas"]["DayType"]; + source: components["schemas"]["DayTypeSource"]; + /** Temperature C */ + temperature_c: number | null; + }; + /** + * LoadForecastConfig + * @description Optional inputs of the load forecast (Phase 4 WP6; additive in ``forecast`` v2). + * + * ``workday``: a ``binary_sensor`` of the ``workday`` integration (``on`` = today is a + * workday); with it the day types come from the user's own holiday calendar (today from the + * state, other days through ``workday.check_date``), without it from the weekday (Mon-Fri + * workday, Sat/Sun weekend, no holidays). ``weather``: a ``weather`` entity whose outside + * temperature (state attribute ``temperature``) is recorded and whose hourly forecast + * (``weather.get_forecasts``) drives the temperature term of the load profile. + */ + LoadForecastConfig: { + weather?: components["schemas"]["EntityRef"] | null; + workday?: components["schemas"]["EntityRef"] | null; + }; + /** + * LoadForecastView + * @description The base load forecast (W, P10/P50/P90 on the 15-minute grid from the current slot). + * + * The base load is the house consumption without the configured (controllable) devices; their + * draw is planned separately. ``quality``: ``ok`` (profiles from history), ``cold_start`` + * (fewer than 3 days: a flat line at the recent mean with a wide band) or ``no_history`` + * (nothing measured yet: no forecast). + */ + LoadForecastView: { + /** Clamped Slots */ + clamped_slots: number; + /** Controlled Devices */ + controlled_devices: string[]; + /** Days */ + days: components["schemas"]["LoadDayView"][]; + /** Days Of History */ + days_of_history: number; + /** Devices Unmeasured */ + devices_unmeasured: { + [key: string]: number; + }; + /** Fitted At */ + fitted_at: string | null; + /** + * Generated At + * Format: date-time + */ + generated_at: string; + /** Hours */ + hours: number; + /** P10 W */ + p10_w: (number | null)[]; + /** P50 W */ + p50_w: (number | null)[]; + /** P90 W */ + p90_w: (number | null)[]; + quality: components["schemas"]["LoadQuality"]; + /** Source Id */ + source_id: string; + /** Start */ + start: string | null; + /** Step S */ + step_s: number; + temperature: components["schemas"]["LoadTemperatureView"]; + /** Unmeasured In Profile */ + unmeasured_in_profile: components["schemas"]["DayType"][]; + }; + /** + * LoadQuality + * @enum {string} + */ + LoadQuality: "ok" | "cold_start" | "no_history"; + /** + * LoadTemperatureView + * @description The temperature term of the load profile. + * + * ``reason``: ``active``, ``not_configured`` (no weather entity), ``too_few_days`` (fewer + * than 14 days with a recorded temperature) or ``no_spread`` (the temperatures varied too + * little to learn anything). + */ + LoadTemperatureView: { + /** Active */ + active: boolean; + /** Cooling W Per K */ + cooling_w_per_k: number; + /** Days */ + days: number; + /** Heating W Per K */ + heating_w_per_k: number; + /** + * Reason + * @enum {string} + */ + reason: "active" | "not_configured" | "too_few_days" | "no_spread"; + }; /** * Mode * @description One mode of a mode-controlled device and the power it draws. @@ -2608,6 +2792,110 @@ export interface components { * @enum {string} */ RiskLevel: "cautious" | "balanced" | "bold"; + /** + * RiskTargetView + * @description A computed state of charge with its structured reason. + */ + RiskTargetView: { + reason: components["schemas"]["Reason"]; + /** Soc Pct */ + soc_pct: number; + /** + * Source + * @enum {string} + */ + source: "learned" | "source_quantiles" | "default_band"; + }; + /** + * RiskView + * @description The current morning buffer and evening target and what they rest on (Phase 4 WP7). + * + * ``status``: ``ok``; ``no_battery_model`` (the ``battery`` section has no model yet: the + * windows are shown, the targets are ``null``); ``no_forecast`` (no load forecast yet). + * ``distribution_source``: ``learned`` once ``min_history_days`` days of measured net errors + * exist, else ``source_quantiles`` (the PV sources' P10/P90 and the load residuals). + */ + RiskView: { + /** Bridge Next Day */ + bridge_next_day: boolean; + /** + * Distribution Source + * @enum {string} + */ + distribution_source: "learned" | "source_quantiles" | "default_band"; + /** Evening At */ + evening_at: string | null; + evening_target: components["schemas"]["RiskTargetView"] | null; + /** + * Generated At + * Format: date-time + */ + generated_at: string; + /** History Days */ + history_days: number; + /** Min History Days */ + min_history_days: number; + morning_buffer: components["schemas"]["RiskTargetView"] | null; + /** Pv Forecast */ + pv_forecast: boolean; + /** Quantile */ + quantile: number; + risk: components["schemas"]["RiskLevel"]; + /** Samples */ + samples: { + [key: string]: number; + }; + /** + * Status + * @enum {string} + */ + status: "ok" | "no_battery_model" | "no_forecast"; + strategy: components["schemas"]["BatteryStrategy"]; + /** Windows */ + windows: components["schemas"]["RiskWindowView"][]; + }; + /** + * RiskWindow + * @enum {string} + */ + RiskWindow: "rest_of_night" | "rest_of_today" | "night" | "next_day" | "bridge" | "day_and_night"; + /** + * RiskWindowView + * @description One window of the risk model and its net energy (PV - base load) at the risk quantile. + */ + RiskWindowView: { + /** + * End + * Format: date-time + */ + end: string; + load: components["schemas"]["EnergyView"]; + /** Net P50 Wh */ + net_p50_wh: number; + /** Net Quantile Wh */ + net_quantile_wh: number; + /** Pooled */ + pooled: boolean; + pv: components["schemas"]["EnergyView"]; + /** Samples */ + samples: number; + /** + * Source + * @enum {string} + */ + source: "learned" | "source_quantiles" | "default_band"; + /** + * Start + * Format: date-time + */ + start: string; + /** + * Weather + * @enum {string} + */ + weather: "overcast" | "mixed" | "clear" | "night"; + window: components["schemas"]["RiskWindow"]; + }; /** * RoleSpec * @description One entity role of a template. @@ -4044,6 +4332,37 @@ export interface operations { }; }; }; + forecast_load_api_v1_forecast_load_get: { + parameters: { + query?: { + hours?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["LoadForecastView"]; + }; + }; + /** @description Validation error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ValidationErrorOut"]; + }; + }; + }; + }; forecast_pv_api_v1_forecast_pv_get: { parameters: { query?: { @@ -4075,6 +4394,26 @@ export interface operations { }; }; }; + forecast_risk_api_v1_forecast_risk_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RiskView"]; + }; + }; + }; + }; info_api_v1_info_get: { parameters: { query?: never; diff --git a/frontend/src/lib/i18n/locales/de.json b/frontend/src/lib/i18n/locales/de.json index 60d6159..38d31b2 100644 --- a/frontend/src/lib/i18n/locales/de.json +++ b/frontend/src/lib/i18n/locales/de.json @@ -860,5 +860,9 @@ "api.validation.import_formula_all_in": "Der Tarif {template_id} liefert bereits Endpreise inklusive Gebühren und Steuern; eine Preisformel würde sie doppelt zählen.", "tariff.import.tou_stored": "Ein Zeittarif (HT/NT) ist gespeichert. Er lässt sich in einer späteren Version hier bearbeiten; Speichern lässt ihn unverändert, solange du keine andere Art wählst.", "safety.battery_not_supported": "Batteriesteuerung ist noch nicht verfügbar; es wurde nichts geschrieben.", - "action.battery": "Batterie-Limits setzen" + "action.battery": "Batterie-Limits setzen", + "api.validation.forecast_workday_domain": "{entity_id} ist kein Binärsensor. Wähle den Binärsensor der Workday-Integration.", + "api.validation.forecast_weather_domain": "{entity_id} ist keine Wetter-Entität.", + "planner.battery.morning_buffer": "Akku zuerst bis {soc_pct, number, ::.0} %: {cause, select, prefer_full {du willst den Akku so voll wie möglich} forecast_poor {bei der Risikostufe {risk, select, cautious {vorsichtig} balanced {ausgewogen} other {mutig}} deckt die Solarprognose heute und die Nacht nicht mit Reserve} forecast_uncertain {die Solarprognose ist unsicher} manual_floor {dein Mindest-Morgenpuffer} other {die Solarprognose lässt reichlich Überschuss}} (übrig nach heute und der Nacht bei dieser Risikostufe: {surplus_kwh, number, ::.0} kWh).", + "planner.battery.evening_target": "Abendziel {soc_pct, number, ::.0} % (Risikostufe {risk, select, cautious {vorsichtig} balanced {ausgewogen} other {mutig}}): {cause, select, prefer_full {du willst den Akku so voll wie möglich} next_day_deficit {die Nacht ({night_kwh, number, ::.0} kWh) und ein schwacher nächster Tag ({deficit_kwh, number, ::.0} kWh mehr) brauchen es} user_target {dein Ziel-Ladestand} max_soc {begrenzt auf den maximalen Ladestand} other {die Nacht braucht {night_kwh, number, ::.0} kWh}}." } diff --git a/frontend/src/lib/i18n/locales/en.json b/frontend/src/lib/i18n/locales/en.json index e6e777e..0a81b1d 100644 --- a/frontend/src/lib/i18n/locales/en.json +++ b/frontend/src/lib/i18n/locales/en.json @@ -860,5 +860,9 @@ "api.validation.import_formula_all_in": "The tariff {template_id} already delivers end prices including fees and taxes; a price formula would count them twice.", "tariff.import.tou_stored": "A time-of-use tariff (HT/NT) is stored. It can be edited here in a later version; saving keeps it unchanged unless you choose another kind.", "safety.battery_not_supported": "Battery control is not available yet; nothing was written.", - "action.battery": "Set battery limits" + "action.battery": "Set battery limits", + "api.validation.forecast_workday_domain": "{entity_id} is not a binary sensor. Choose the binary sensor of the Workday integration.", + "api.validation.forecast_weather_domain": "{entity_id} is not a weather entity.", + "planner.battery.morning_buffer": "Battery first until {soc_pct, number, ::.0} %: {cause, select, prefer_full {you want the battery as full as possible} forecast_poor {at the {risk, select, cautious {cautious} balanced {balanced} other {bold}} risk level the solar forecast does not cover today and the night with a margin} forecast_uncertain {the solar forecast is uncertain} manual_floor {your minimum morning buffer} other {the solar forecast leaves plenty of surplus}} (left after today and the night at that risk level: {surplus_kwh, number, ::.0} kWh).", + "planner.battery.evening_target": "Evening target {soc_pct, number, ::.0} % ({risk, select, cautious {cautious} balanced {balanced} other {bold}} risk level): {cause, select, prefer_full {you want the battery as full as possible} next_day_deficit {the night ({night_kwh, number, ::.0} kWh) and a weak next day ({deficit_kwh, number, ::.0} kWh more) need it} user_target {your target state of charge} max_soc {capped at the maximum state of charge} other {the night needs {night_kwh, number, ::.0} kWh}}." } diff --git a/frontend/src/lib/wizard/forecastForm.ts b/frontend/src/lib/wizard/forecastForm.ts index 91c3bfb..0a27ac2 100644 --- a/frontend/src/lib/wizard/forecastForm.ts +++ b/frontend/src/lib/wizard/forecastForm.ts @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // Forecast section (v2) <-> form: the PV forecast sources (templates plus the entities of their -// roles). Independent of the tariff, so the choice can be stored without prices. Weights and -// learning settings have no form fields yet (Phase 4 WP10); they are carried through unchanged. +// roles). Independent of the tariff, so the choice can be stored without prices. Weights, learning +// settings and the load forecast inputs (`load`: workday sensor, weather entity) have no form +// fields yet (Phase 4 WP11); they are carried through unchanged. import type { ForecastConfig } from '../api/types'; import { emptyRef, fromRef, toRef, type RefField } from './forms'; @@ -15,16 +16,18 @@ export interface ForecastForm { /** Stored weighting per template id (carried through). */ weights: Record>; learning: ForecastConfig['learning'] | null; + load: ForecastConfig['load'] | null; } export function emptyForecastForm(): ForecastForm { - return { templateIds: [], entities: {}, weights: {}, learning: null }; + return { templateIds: [], entities: {}, weights: {}, learning: null, load: null }; } export function forecastFormFromConfig(cfg: ForecastConfig | null): ForecastForm { if (!cfg) return emptyForecastForm(); const form = emptyForecastForm(); form.learning = cfg.learning; + form.load = cfg.load ?? null; for (const source of cfg.sources ?? []) { const id = source.template_id; form.templateIds.push(id); @@ -55,6 +58,7 @@ export function forecastFromForm(form: ForecastForm): ForecastConfig { schema_version: 2, sources, learning: form.learning ?? { min_history_days: 30, load_ml_mode: 'shadow' }, + load: form.load ?? { workday: null, weather: null }, }; } diff --git a/frontend/src/lib/wizard/forms.test.ts b/frontend/src/lib/wizard/forms.test.ts index c42f4c0..2df1808 100644 --- a/frontend/src/lib/wizard/forms.test.ts +++ b/frontend/src/lib/wizard/forms.test.ts @@ -142,12 +142,14 @@ describe('forecast form', () => { }, ], learning: { min_history_days: 14, load_ml_mode: 'off' }, + load: { workday: ref('binary_sensor.workday_sensor'), weather: null }, }; expect(forecastFromForm(forecastFormFromConfig(cfg))).toEqual(cfg); expect(forecastFromForm(forecastFormFromConfig(null))).toEqual({ schema_version: 2, sources: [], learning, + load: { workday: null, weather: null }, }); }); it('gives each source the entities of its own roles', () => { @@ -177,6 +179,7 @@ describe('forecast form', () => { }, ], learning, + load: { workday: null, weather: null }, }; const form = forecastFormFromConfig(cfg); expect(form.entities['solcast.pv']!.today).not.toBe(form.entities['forecast_solar.pv']!.today); diff --git a/suncore/api/app.py b/suncore/api/app.py index 9e6e152..ff32019 100644 --- a/suncore/api/app.py +++ b/suncore/api/app.py @@ -86,7 +86,13 @@ from suncore.core.decisions import Decision, DecisionKind from suncore.core.site import SiteConfig from suncore.core.wizard import PreviewTooLargeError -from suncore.forecast.views import MAX_VIEW_HOURS, ForecastAccuracyView, PvForecastView +from suncore.forecast.views import ( + MAX_VIEW_HOURS, + ForecastAccuracyView, + LoadForecastView, + PvForecastView, + RiskView, +) from suncore.storage.errors import ( ConfigCorruptError, HistoryEntryNotFoundError, @@ -271,7 +277,8 @@ async def decisions( def _register_forecast(app: FastAPI, services: Services) -> None: - """PV forecast and its measured accuracy (Phase 4 WP5; read only, no UI yet).""" + """PV and load forecasts and their measured accuracy (Phase 4 WP5/WP6; read only, no UI + yet).""" @app.get(f"{API_PREFIX}/forecast/pv", response_model=PvForecastView) async def forecast_pv( @@ -289,9 +296,32 @@ async def forecast_accuracy() -> ForecastAccuracyView: """MAE and bias per source and horizon bucket (0-6 h, 6-24 h, 24-48 h) over the last 14 days, plus the source id ``blended`` (the blended forecast); ``bias_w = actual - forecast`` (positive: the source under-forecasts); ``n`` counts 15-minute intervals; - clipped quarter hours are excluded; computed nightly.""" + clipped quarter hours are excluded; computed nightly. The source id ``load_profile`` is + the base load forecast (scored on every quarter hour, not only in daylight).""" return await services.forecast_accuracy() + @app.get(f"{API_PREFIX}/forecast/load", response_model=LoadForecastView) + async def forecast_load( + hours: int = Query(default=48, ge=1, le=MAX_VIEW_HOURS), + ) -> LoadForecastView: + """The base load forecast (house consumption without the configured devices; W, + P10/P50/P90 on the 15-minute grid from the current quarter hour), learnt from the + history per day type (workday/weekend/holiday) with an optional temperature term; its + quality (``ok``, ``cold_start``, ``no_history``), the day type of every forecast date + and where it came from, and how many quarter hours lacked a device measurement.""" + return await services.load_forecast(hours) + + @app.get(f"{API_PREFIX}/forecast/risk", response_model=RiskView) + async def forecast_risk() -> RiskView: + """The risk model now: the windows ``rest_of_today``, ``night``, ``next_day`` and + ``bridge`` (night + next day) with PV, base load and net energy (P50 and at the risk + level's quantile), the morning buffer ("battery first until X %") and the evening + target with their structured reasons, and whether the error distribution is + ``learned`` (from ``min_history_days`` days of measured net errors) or still taken from + the ``source_quantiles``. Targets are ``null`` until the ``battery`` section has a + model.""" + return await services.forecast_risk() + def _register_control(app: FastAPI, services: Services) -> None: """Control state, emergency stop, overrides, proposals and operating modes (WP9).""" diff --git a/suncore/api/services.py b/suncore/api/services.py index 12ea9cb..405ac2a 100644 --- a/suncore/api/services.py +++ b/suncore/api/services.py @@ -22,7 +22,12 @@ from suncore.core.detection import DetectedEntity, EnergySuggestions from suncore.core.site import SiteConfig from suncore.core.wizard import CheckItem, PreviewItem -from suncore.forecast.views import ForecastAccuracyView, PvForecastView +from suncore.forecast.views import ( + ForecastAccuracyView, + LoadForecastView, + PvForecastView, + RiskView, +) from suncore.storage import HistoryEntry from suncore.templates import TemplateCatalog @@ -174,7 +179,18 @@ async def pv_forecast(self, hours: int) -> PvForecastView: async def forecast_accuracy(self) -> ForecastAccuracyView: """MAE and bias (actual - forecast) per source and horizon bucket from the nightly job, - including the blended forecast (source id ``blended``).""" + including the blended forecast (source id ``blended``) and the load forecast (source id + ``load_profile``).""" + ... + + async def load_forecast(self, hours: int) -> LoadForecastView: + """The base load forecast for the next ``hours`` (W, P10/P50/P90) with its day types, + temperature term and quality (Phase 4 WP6).""" + ... + + async def forecast_risk(self) -> RiskView: + """The current morning buffer and evening target with their reasons, the windows of + the net energy and the source of the error distribution (Phase 4 WP7).""" ... # --- control (Phase 3 WP9) --------------------------------------------------------- diff --git a/suncore/api/validation.py b/suncore/api/validation.py index 0992c43..5eacd41 100644 --- a/suncore/api/validation.py +++ b/suncore/api/validation.py @@ -114,6 +114,8 @@ "suncore.battery_entities_need_adapter": "api.validation.battery_entities_need_adapter", "suncore.forecast_auto_weight": "api.validation.forecast_auto_weight", "suncore.forecast_manual_weight": "api.validation.forecast_manual_weight", + "suncore.forecast_weather_domain": "api.validation.forecast_weather_domain", + "suncore.forecast_workday_domain": "api.validation.forecast_workday_domain", "suncore.grid_charge_needs_limit": "api.validation.grid_charge_needs_limit", "suncore.import_formula_dynamic_only": "api.validation.import_formula_dynamic_only", "suncore.import_tou_extra": "api.validation.import_tou_extra", diff --git a/suncore/app/adapters.py b/suncore/app/adapters.py index df34ff4..661bf28 100644 --- a/suncore/app/adapters.py +++ b/suncore/app/adapters.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""Composition of the tariff and forecast adapters (Phase 4 WP4/WP5). +"""Composition of the tariff and forecast adapters (Phase 4 WP4/WP5/WP6). This module builds the price and forecast sources from the configured templates and hands them the Home Assistant connection. It is the only module of ``suncore.app`` that may call @@ -12,15 +12,28 @@ extra Home Assistant call), the Energy dashboard's ``energy/solar_forecast`` through ``suncore.ha.fetch_solar_forecast`` (rule 9 keeps the command inside ``suncore.ha``) and the read-only action ``solcast_solar.query_forecast_data``. + +The load forecast reads through :class:`HaLoadReader`: the ``workday`` binary sensor's state and +the weather entity's ``temperature`` attribute from the entity cache, and the read-only actions +``workday.check_date`` (per date) and ``weather.get_forecasts`` (``type: hourly``). """ from __future__ import annotations from collections.abc import Awaitable, Callable, Mapping -from datetime import datetime +from datetime import date, datetime, tzinfo from typing import Any from suncore.core.config import ForecastConfig +from suncore.forecast.accuracy import LearnedParams +from suncore.forecast.load import ( + WorkdayState, + parse_check_date, + parse_hourly_temperatures, + parse_workday_sensor, + temperature_c, +) +from suncore.forecast.load_service import LoadData, LoadForecastService, LoadInputs from suncore.forecast.pv.factory import build_sources from suncore.forecast.pv.service import ( BlendContext, @@ -87,4 +100,103 @@ def build_pv_forecast_service( ) -__all__ = ["HaPvDataReader", "build_pv_forecast_service"] +class HaLoadReader: + """The :class:`~suncore.forecast.load_service.LoadHaReader` backed by Home Assistant. + + Shapes verified in ``docs/research/ha-websocket-api.md`` ("Workday", "Weather forecasts"): + the weather entity's ``temperature_unit`` attribute is the unit of its ``temperature`` and of + every ``weather.get_forecasts`` item (both converted to it by Home Assistant). + """ + + def __init__(self, connection: HAConnection, cache: EntityCache) -> None: + self._connection = connection + self._cache = cache + + def _state(self, entity_id: str) -> tuple[str, Mapping[str, Any]] | None: + entry = self._cache.get(entity_id) + if entry is None or entry.state.state in UNAVAILABLE_STATES: + return None + return entry.state.state, entry.state.attributes + + def workday_state(self, entity_id: str) -> WorkdayState | None: + current = self._state(entity_id) + return None if current is None else parse_workday_sensor(entity_id, *current) + + async def check_workday(self, entity_id: str, day: date, *, min_interval_s: float) -> bool: + response = await self._connection.call_action_response( + "workday", + "check_date", + {"check_date": day.isoformat()}, + target={"entity_id": entity_id}, + min_interval_s=min_interval_s, + rate_key=day.isoformat(), + ) + workday = parse_check_date(response, entity_id) + if workday is None: + msg = f"workday.check_date: no answer for {entity_id}" + raise ValueError(msg) + return workday + + def _unit(self, entity_id: str) -> str | None: + current = self._state(entity_id) + unit = None if current is None else current[1].get("temperature_unit") + return unit if isinstance(unit, str) else None + + def outside_temperature(self, entity_id: str) -> float | None: + current = self._state(entity_id) + if current is None: + return None + return temperature_c(current[1].get("temperature"), self._unit(entity_id)) + + async def hourly_temperatures( + self, entity_id: str, *, min_interval_s: float + ) -> list[tuple[datetime, float]]: + response = await self._connection.call_action_response( + "weather", + "get_forecasts", + {"type": "hourly"}, + target={"entity_id": entity_id}, + min_interval_s=min_interval_s, + rate_key=entity_id, + ) + return parse_hourly_temperatures(response, entity_id, self._unit(entity_id)) + + +def build_load_forecast_service( + config: ForecastConfig | None, + device_ids: tuple[str, ...], + *, + data: LoadData | None, + learned: LearnedParams | None, + connection: HAConnection, + cache: EntityCache, + clock: Callable[[], datetime], + sleep: Callable[[float], Awaitable[None]], + tz: Callable[[], tzinfo], +) -> LoadForecastService | None: + """The load forecast of the site; ``None`` without ``timeseries.db`` (no history).""" + if data is None: + return None + load = None if config is None else config.load + inputs = LoadInputs( + device_ids=device_ids, + workday_entity=None if load is None or load.workday is None else load.workday.entity_id, + weather_entity=None if load is None or load.weather is None else load.weather.entity_id, + ) + return LoadForecastService( + data, + learned, + clock=clock, + sleep=sleep, + tz=tz, + inputs=inputs, + reader=HaLoadReader(connection, cache), + ) + + +__all__ = [ + "HaLoadReader", + "HaPvDataReader", + "build_load_forecast_service", + "build_pv_forecast_service", +] diff --git a/suncore/app/history.py b/suncore/app/history.py index 1773019..d334f66 100644 --- a/suncore/app/history.py +++ b/suncore/app/history.py @@ -8,9 +8,12 @@ PV as the sum of the arrays, the house from its sensor or from ``pv + grid - battery``. The rows are stored as *derived* quarter hours. -Controlled devices are not rebuilt: their power comes from their own power models, which have -no statistic to read. ``UNVERIFIED``: whether the Supervisor proxy forwards ``recorder/*``; -an error is reported to the caller, which logs it and carries on without history. +Controlled devices with a power sensor (``power_model.power_role`` mapped) are rebuilt too, from +the same statistics (``mean``, W) into ``device..power_w``, so the load forecast can take +their draw out of the base load from the first day (PR 4-E review M2). A device without a power +sensor (nominal or counter-derived model) has no statistic; its quarter hours stay unmeasured. +``UNVERIFIED``: whether the Supervisor proxy forwards ``recorder/*``; an error is reported to +the caller, which logs it and carries on without history. """ from __future__ import annotations @@ -21,12 +24,14 @@ import structlog +from suncore.core.devices import DeviceConfig from suncore.core.measurements import ( SITE_BATTERY_W, SITE_GRID_W, SITE_HOUSE_W, SITE_PV_W, SITE_SOC_PCT, + device_power_key, ) from suncore.core.series import Grid, SeriesUnit, TimeSeries, resample_power from suncore.core.site import SiteConfig @@ -148,6 +153,22 @@ def balance(*v: float) -> float: return {key: series for key, series in named.items() if series is not None} +_DEVICE_PREFIX = "device:" + + +def device_statistic_specs(devices: Sequence[DeviceConfig]) -> list[StatisticSpec]: + """The power sensor of every device that maps one (``power_model.power_role``).""" + specs: list[StatisticSpec] = [] + for device in devices: + role = device.power_model.power_role + ref = None if role is None else device.entities.get(role) + if ref is not None: + specs.append( + StatisticSpec(f"{_DEVICE_PREFIX}{device.id}", ref.entity_id, invert=ref.invert_sign) + ) + return specs + + async def rebuild_history( connection: HAConnection, store: TimeseriesStore, @@ -155,18 +176,23 @@ async def rebuild_history( now: datetime, *, days: int = REBUILD_DAYS, + devices: Sequence[DeviceConfig] = (), ) -> RebuildResult: - """Fetch the site's hourly statistics and store them as derived quarter hours. + """Fetch the site's (and the devices' power) hourly statistics and store them as derived + quarter hours. Home Assistant errors propagate. Existing measured rows are never overwritten. """ end = now.astimezone(UTC).replace(minute=0, second=0, microsecond=0) - history = await fetch_quarter_history( - connection, site_statistic_specs(site), end - timedelta(days=days), end - ) + specs = [*site_statistic_specs(site), *device_statistic_specs(devices)] + history = await fetch_quarter_history(connection, specs, end - timedelta(days=days), end) stored: dict[str, int] = {} for key, series in assemble_site_series(site, history.series).items(): stored[key] = await store.store_quarter_hours(key, series, derived=True) + for key, series in history.series.items(): + if key.startswith(_DEVICE_PREFIX): + device_key = device_power_key(key.removeprefix(_DEVICE_PREFIX)) + stored[device_key] = await store.store_quarter_hours(device_key, series, derived=True) result = RebuildResult(stored=stored, missing=history.missing) log.info("history.rebuilt", quarter_hours=result.total, missing=list(result.missing)) return result diff --git a/suncore/app/openapi_stub.py b/suncore/app/openapi_stub.py index 7345f11..aa8dec1 100644 --- a/suncore/app/openapi_stub.py +++ b/suncore/app/openapi_stub.py @@ -13,6 +13,7 @@ from suncore import __version__ from suncore.api.services import EntityQuery, HealthReport, PeerPolicy from suncore.core.balance import FlowSnapshot +from suncore.core.battery import BatterySettings from suncore.core.config import OperationConfig from suncore.core.controlstate import ControlState, DeviceControlState, OverrideCommand from suncore.core.decisions import Decision, DecisionKind, OperatingMode @@ -20,7 +21,10 @@ from suncore.core.site import SiteConfig from suncore.core.wizard import CheckItem, PreviewItem from suncore.forecast.accuracy import accuracy_view -from suncore.forecast.views import ForecastAccuracyView, PvForecastView +from suncore.forecast.load_service import LoadForecastService +from suncore.forecast.risk import ErrorModel +from suncore.forecast.risk_learning import risk_view +from suncore.forecast.views import ForecastAccuracyView, LoadForecastView, PvForecastView, RiskView from suncore.storage import HistoryEntry from suncore.templates import TemplateCatalog @@ -86,6 +90,22 @@ async def pv_forecast(self, hours: int) -> PvForecastView: async def forecast_accuracy(self) -> ForecastAccuracyView: return accuracy_view([], {}) + async def load_forecast(self, hours: int) -> LoadForecastView: + return LoadForecastService(None, None, clock=lambda: datetime.now(UTC)).view(hours) + + async def forecast_risk(self) -> RiskView: + return risk_view( + datetime.now(UTC), + pv=None, + load=None, + settings=BatterySettings(), + user_target_pct=None, + model=ErrorModel(), + min_history_days=30, + tz=UTC, + plant=None, + ) + def control_state(self) -> ControlState: return ControlState( ts=None, diff --git a/suncore/app/runtime.py b/suncore/app/runtime.py index b03dde2..c45d134 100644 --- a/suncore/app/runtime.py +++ b/suncore/app/runtime.py @@ -26,7 +26,7 @@ UnknownRoleError, UnknownTemplateError, ) -from suncore.app.adapters import build_pv_forecast_service +from suncore.app.adapters import build_load_forecast_service, build_pv_forecast_service from suncore.app.cadence import device_cadences, merge_cadences from suncore.app.control import ControlStack from suncore.app.history import rebuild_history @@ -34,6 +34,7 @@ from suncore.app.recorder import FLUSH_INTERVAL_S, MeasurementRecorder from suncore.app.settings import SUPERVISOR_INGRESS_PEER, Settings from suncore.core.balance import FlowSnapshot, compute_flows +from suncore.core.battery import BatterySettings from suncore.core.config import ( DevicesConfig, ForecastConfig, @@ -60,12 +61,21 @@ from suncore.core.site import SiteConfig from suncore.core.wizard import CheckItem, PreviewItem, collect_refs, preview, run_system_check from suncore.forecast.accuracy import accuracy_view, load_accuracy +from suncore.forecast.load import LOAD_SOURCE_ID +from suncore.forecast.load_service import LoadForecastService from suncore.forecast.pv.calibration import load_bias from suncore.forecast.pv.clearsky import plant_geometry -from suncore.forecast.pv.clipping import export_limit_w +from suncore.forecast.pv.clipping import clip_filter, export_limit_w from suncore.forecast.pv.learning import BLENDED_SOURCE_ID, LearningContext, run_nightly from suncore.forecast.pv.service import BlendContext, PvForecastService -from suncore.forecast.views import ForecastAccuracyView, PvForecastView +from suncore.forecast.risk import ErrorModel +from suncore.forecast.risk_learning import learn_net_errors, load_error_model, risk_view +from suncore.forecast.views import ( + ForecastAccuracyView, + LoadForecastView, + PvForecastView, + RiskView, +) from suncore.ha import ( SITE_ROLES, CommandError, @@ -145,6 +155,12 @@ class Runtime: forecast: ForecastConfig | None = None pv_service: PvForecastService | None = None """Refreshes the configured PV forecast sources (``None`` = none configured).""" + load_service: LoadForecastService | None = None + """The base load forecast (``None`` without ``timeseries.db``).""" + _clipped: frozenset[datetime] = frozenset() + """Quarter hours the last PV nightly job flagged as clipped (risk learning skips them).""" + risk_model: ErrorModel = field(default_factory=ErrorModel) + """The learnt net energy errors (WP7), loaded at start and after the nightly job.""" timeseries_db: Database | None = None """``timeseries.db``; ``None`` = unavailable (too new, unopenable): control is unaffected.""" timeseries: TimeseriesStore | None = None @@ -178,6 +194,7 @@ class Runtime: _maintenance_task: asyncio.Task[None] | None = None _history_task: asyncio.Task[None] | None = None _pv_task: asyncio.Task[None] | None = None + _load_task: asyncio.Task[None] | None = None _accuracy_task: asyncio.Future[None] | None = None _history_done: bool = False """The statistics rebuild ran (or was not needed) in this process.""" @@ -210,6 +227,8 @@ async def prepare(self) -> None: self.stack = self._new_stack() self.pv_service = self._new_pv_service() await self._load_pv_learning() + self.load_service = self._new_load_service() + await self._load_risk_model() async def start(self) -> None: """Wire the control stack and the connection and start connecting in the background. @@ -232,6 +251,7 @@ async def start(self) -> None: self._connect_task = asyncio.create_task(self._connect(), name="suncore-ha-connect") self._prune_task = asyncio.create_task(self._prune_loop(), name="suncore-prune") self._start_pv_task() + self._start_load_task() if self.timeseries is not None: self._flush_task = asyncio.create_task(self._flush_loop(), name="suncore-ts-flush") self._maintenance_task = asyncio.create_task( @@ -397,15 +417,76 @@ def _schedule_forecast_accuracy(self) -> None: """Run the accuracy statistics in the background (nightly timing unchanged).""" if ( not self._started - or self.pv_service is None + or (self.pv_service is None and self.load_service is None) or (self._accuracy_task is not None and not self._accuracy_task.done()) ): return self._accuracy_task = asyncio.ensure_future(self.update_forecast_accuracy()) async def update_forecast_accuracy(self) -> None: - """The nightly PV forecast learning: clipping, accuracy of the sources and of the - blend, bias tables (never raises); the results go straight to the service.""" + """The nightly forecast learning (never raises): PV (clipping, accuracy of the sources + and of the blend, bias tables; the results go straight to the service), then the load + forecast (accuracy, refit).""" + await self._update_pv_learning() + await self._update_load_learning() + + async def _update_load_learning(self) -> None: + service = self.load_service + if service is None: + return + try: + await service.nightly(self.clock()) + except Exception: + log.exception("forecast.load.nightly_failed") + return + if self.timeseries is None or self.learned is None: + return + context = self._blend_context() + try: + self.risk_model = await learn_net_errors( + self.timeseries, + self.learned, + self.clock(), + tz=context.tz, + plant=context.plant, + base_reader=service.base_load_series, + is_clipped=clip_filter(self._clipped), + ) + except Exception: + log.exception("forecast.risk.nightly_failed") + + async def _load_risk_model(self) -> None: + if self.learned is None: + return + try: + self.risk_model = (await load_error_model(self.learned)).model + except Exception: + log.exception("forecast.risk.load_failed") + + async def forecast_risk(self) -> RiskView: + """Morning buffer, evening target and their windows as the planner would get them.""" + now = self.clock() + stored = await self.config.get("battery") + settings = stored if isinstance(stored, BatterySettings) else BatterySettings() + target = None if self.goals is None else self.goals.battery_target + blended = None if self.pv_service is None else self.pv_service.blended(now) + context = self._blend_context() + return risk_view( + now, + pv=None if blended is None else blended.series, + load=None if self.load_service is None else self.load_service.forecast(now), + settings=settings, + user_target_pct=None if target is None else target.soc_pct, + site_battery=None if self.site is None else self.site.battery, + model=self.risk_model, + min_history_days=( + 30 if self.forecast is None else self.forecast.learning.min_history_days + ), + tz=context.tz, + plant=context.plant, + ) + + async def _update_pv_learning(self) -> None: if self.timeseries is None or self.learned is None or self.pv_service is None: return if self._ha_location is None: @@ -413,7 +494,7 @@ async def update_forecast_accuracy(self) -> None: service = self.pv_service context = self._blend_context() try: - await run_nightly( + result = await run_nightly( self.timeseries, self.learned, service.source_ids, @@ -425,6 +506,7 @@ async def update_forecast_accuracy(self) -> None: except Exception: log.exception("forecast.accuracy_failed") return + self._clipped = result.clipped if self.pv_service is service: await self._load_pv_learning() @@ -487,6 +569,47 @@ async def _load_pv_learning(self) -> None: {k: v for k, v in bias.items() if k in service.source_ids}, ) + # --- load forecast (Phase 4 WP6) ------------------------------------------------ + + def _new_load_service(self) -> LoadForecastService | None: + device_ids = () if self.devices is None else tuple(d.id for d in self.devices.devices) + return build_load_forecast_service( + self.forecast, + device_ids, + data=self.timeseries, + learned=self.learned, + connection=self.connection, + cache=self.cache, + clock=self.clock, + sleep=self.control_sleep, + tz=self._blend_tz, + ) + + def _blend_tz(self) -> tzinfo: + return self._blend_context().tz + + def _start_load_task(self) -> None: + if self.load_service is not None and (self._load_task is None or self._load_task.done()): + self._load_task = asyncio.create_task( + self.load_service.run(), name="suncore-load-forecast" + ) + + async def _rebuild_load_service(self) -> None: + """Replace the load forecast after a ``forecast`` or ``devices`` change.""" + task, self._load_task = self._load_task, None + if task is not None: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + self.load_service = self._new_load_service() + if self._started: + self._start_load_task() + + async def load_forecast(self, hours: int) -> LoadForecastView: + if self.load_service is None: + return LoadForecastService(None, None, clock=self.clock).view(hours) + return self.load_service.view(hours) + def _start_pv_task(self) -> None: if self.pv_service is not None and (self._pv_task is None or self._pv_task.done()): self._pv_task = asyncio.create_task(self.pv_service.run(), name="suncore-pv-forecast") @@ -510,6 +633,8 @@ async def pv_forecast(self, hours: int) -> PvForecastView: async def forecast_accuracy(self) -> ForecastAccuracyView: ids = [] if self.pv_service is None else [*self.pv_service.source_ids, BLENDED_SOURCE_ID] + if self.load_service is not None: + ids.append(LOAD_SOURCE_ID) stored = {} if self.learned is None else await load_accuracy(self.learned, ids) return accuracy_view(ids, stored) @@ -573,8 +698,13 @@ async def _rebuild_history(self) -> None: return try: if await store.get_meta(META_HISTORY_REBUILT) is None: - await rebuild_history(self.connection, store, site, self.clock()) + devices = () if self.devices is None else self.devices.devices + await rebuild_history( + self.connection, store, site, self.clock(), devices=devices + ) await store.set_meta(META_HISTORY_REBUILT, self.clock().isoformat()) + if self.load_service is not None: + self.load_service.request_refit() # the profiles can use it now except asyncio.CancelledError: raise except CommandError as exc: @@ -657,6 +787,7 @@ async def stop(self) -> None: self._maintenance_task, self._history_task, self._pv_task, + self._load_task, self._accuracy_task, ) for task in (*control_tasks, *storage_tasks): @@ -668,7 +799,7 @@ async def stop(self) -> None: await task self._connect_task = self._prune_task = self._tz_task = None self._flush_task = self._maintenance_task = self._history_task = None - self._pv_task = None + self._pv_task = self._load_task = None # Hardware release comes first: a running downsample or retention statement keeps the # (shielded) rollback of its cancelled task waiting, and a Pi can need seconds for it. # The storage tasks are awaited only after the devices are released. @@ -874,6 +1005,8 @@ async def _apply_stored( await self._rebuild_stack() if name == "forecast": await self._rebuild_pv_service() + if name in ("forecast", "devices"): + await self._rebuild_load_service() if name == "devices": await self._sync_modes() @@ -1139,6 +1272,7 @@ async def _reload_live_sections(self, *, resubscribe: bool) -> None: """After a restore or import: re-read what the control layer runs on and apply it.""" before = (self.site, self.devices, self.region) forecast_before = self.forecast + devices_before = self.devices await self._load_live_sections() if resubscribe: await self._resubscribe() @@ -1148,6 +1282,8 @@ async def _reload_live_sections(self, *, resubscribe: bool) -> None: self.stack.set_goals(self.goals) if self.forecast != forecast_before: await self._rebuild_pv_service() + if self.forecast != forecast_before or self.devices != devices_before: + await self._rebuild_load_service() await self._sync_modes() def add_change_listener(self, listener: Callable[[], None]) -> Callable[[], None]: diff --git a/suncore/core/actions.py b/suncore/core/actions.py index aa90547..bcb952f 100644 --- a/suncore/core/actions.py +++ b/suncore/core/actions.py @@ -16,6 +16,7 @@ "tibber.get_prices", "solcast_solar.query_forecast_data", "weather.get_forecasts", + "workday.check_date", } ) """``.`` of the actions whose response SunCore may read.""" diff --git a/suncore/core/battery.py b/suncore/core/battery.py index 9a04036..307c770 100644 --- a/suncore/core/battery.py +++ b/suncore/core/battery.py @@ -20,7 +20,7 @@ from suncore.core.decisions import BatteryAction from suncore.core.devices import RoleName, TemplateId -from suncore.core.site import EntityRef +from suncore.core.site import BatteryConfig, EntityRef from suncore.core.tariff import WeeklySchedule from suncore.core.validation import invalid @@ -108,6 +108,21 @@ def grid_charge_pays(self, charge_price: float, later_price: float) -> bool: return later_price > self.min_later_price(charge_price) +def effective_reserve_pct(site_battery: BatteryConfig | None, model: BatteryModel) -> float: + """The one SoC floor of planner, controller and gate (ADR-0013 decision 3). + + ``max(site.battery.min_soc_pct or 0, model.reserve_pct + model.bms_drift_margin_pct)``: the + planning floor of the model, raised to the minimum SoC configured for the site battery if + that is higher (the site value is what the user set in the wizard for the inverter). + """ + site_min = ( + 0.0 + if site_battery is None or site_battery.min_soc_pct is None + else site_battery.min_soc_pct + ) + return max(site_min, model.min_soc_pct) + + @unique class BatteryStrategy(StrEnum): """How the planner uses the battery (plan §1 item 5, answers 2026-10-16).""" @@ -290,4 +305,5 @@ def entity_refs(self) -> list[EntityRef]: "CalibrationCharge", "GridChargePolicy", "RiskLevel", + "effective_reserve_pct", ] diff --git a/suncore/core/config.py b/suncore/core/config.py index 18a5109..fe0bc96 100644 --- a/suncore/core/config.py +++ b/suncore/core/config.py @@ -245,8 +245,50 @@ class LearningSettings(_Frozen): load_ml_mode: Literal["off", "shadow", "auto"] = "shadow" +class LoadForecastConfig(_Frozen): + """Optional inputs of the load forecast (Phase 4 WP6; additive in ``forecast`` v2). + + ``workday``: a ``binary_sensor`` of the ``workday`` integration (``on`` = today is a + workday); with it the day types come from the user's own holiday calendar (today from the + state, other days through ``workday.check_date``), without it from the weekday (Mon-Fri + workday, Sat/Sun weekend, no holidays). ``weather``: a ``weather`` entity whose outside + temperature (state attribute ``temperature``) is recorded and whose hourly forecast + (``weather.get_forecasts``) drives the temperature term of the load profile. + """ + + workday: EntityRef | None = None + weather: EntityRef | None = None + + @field_validator("workday") + @classmethod + def _binary_sensor(cls, value: EntityRef | None) -> EntityRef | None: + if value is not None and not value.entity_id.startswith("binary_sensor."): + raise invalid( + "forecast_workday_domain", + "forecast.load.workday: {entity_id} is not a binary_sensor", + entity_id=value.entity_id, + ) + return value + + @field_validator("weather") + @classmethod + def _weather(cls, value: EntityRef | None) -> EntityRef | None: + if value is not None and not value.entity_id.startswith("weather."): + raise invalid( + "forecast_weather_domain", + "forecast.load.weather: {entity_id} is not a weather entity", + entity_id=value.entity_id, + ) + return value + + def entity_refs(self) -> list[EntityRef]: + return [r for r in (self.workday, self.weather) if r is not None] + + class ForecastConfig(_Frozen): - """PV forecast sources (schema v2: several sources, blending, learning settings). + """PV forecast sources (schema v2: several sources, blending, learning settings) and the + optional inputs of the load forecast (``load``, added in Phase 4 WP6 without a version + bump: an absent ``load`` is the default, so every stored v2 document stays valid). Independent of the tariff so that the wizard can store the choice without prices. Forecast uncertainty is handled by the learned risk model and the battery section's @@ -258,6 +300,7 @@ class ForecastConfig(_Frozen): default_factory=list ) learning: LearningSettings = LearningSettings() + load: LoadForecastConfig = LoadForecastConfig() @model_validator(mode="after") def _unique_templates(self) -> Self: @@ -275,6 +318,8 @@ def entity_refs(self) -> list[EntityRef]: for source in self.sources: for ref in source.entities.values(): refs.setdefault(ref.entity_id, ref) + for ref in self.load.entity_refs(): + refs.setdefault(ref.entity_id, ref) return list(refs.values()) diff --git a/suncore/core/measurements.py b/suncore/core/measurements.py index 42551dd..bc53516 100644 --- a/suncore/core/measurements.py +++ b/suncore/core/measurements.py @@ -17,6 +17,10 @@ SITE_SOC_PCT: Final = "site.soc_pct" SITE_HOUSE_W: Final = "site.house_w" +WEATHER_TEMPERATURE_C: Final = "weather.temperature_c" +"""Outside temperature (°C) of the ``forecast.load.weather`` entity, recorded by the load +forecast for its temperature term (not a site power, never part of the balance).""" + SITE_KEYS: Final[tuple[str, ...]] = ( SITE_GRID_W, SITE_PV_W, @@ -46,6 +50,7 @@ def is_percent_key(key: str) -> bool: "SITE_KEYS", "SITE_PV_W", "SITE_SOC_PCT", + "WEATHER_TEMPERATURE_C", "device_power_key", "is_percent_key", ] diff --git a/suncore/forecast/accuracy.py b/suncore/forecast/accuracy.py index 13c5db3..7c296bc 100644 --- a/suncore/forecast/accuracy.py +++ b/suncore/forecast/accuracy.py @@ -34,7 +34,7 @@ import json import time -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta from typing import Any, Final, Literal, Protocol @@ -70,6 +70,10 @@ ClipFilter = Callable[[datetime], bool] """``is_clipped(slot_start)``: whether the measured PV of that quarter hour was limited.""" +ForecastKind = Literal["pv", "load", "price"] +ActualReader = Callable[[TimeRange], Awaitable[TimeSeries]] +"""Reads the measured quantity (15-minute means, W) a forecast is scored against.""" + SCHEMA_VERSION: Final = 1 NS_FORECAST_ACCURACY: Final = "forecast_accuracy" """``learned_param`` namespace (equal to ``suncore.storage.learned.NS_FORECAST_ACCURACY``; the @@ -147,11 +151,13 @@ def quarter_errors( actual: TimeSeries, *, is_clipped: ClipFilter = no_clipping, + daylight_only: bool = True, ) -> list[float]: """``actual - forecast`` per 15-minute daylight interval (forecast or measurement above 0 W) both series know and that is not clipped. ``forecast_p50`` is resampled onto the grid of ``actual`` (must be the 15-minute grid). + ``daylight_only=False`` (the load forecast) keeps the intervals where both are zero too. """ if not len(actual) or not len(forecast_p50): return [] @@ -164,7 +170,7 @@ def quarter_errors( ): if measured is None or predicted is None or is_clipped(begin): continue - if measured <= 0 and predicted <= 0: + if daylight_only and measured <= 0 and predicted <= 0: continue # night: both zero says nothing about the forecast errors.append(measured - predicted) return errors @@ -177,12 +183,24 @@ async def compute_accuracy( *, window_days: int = WINDOW_DAYS, is_clipped: ClipFilter = no_clipping, + kind: ForecastKind = "pv", + actual_reader: ActualReader | None = None, ) -> list[AccuracyRecord]: - """The records of one source for the last ``window_days`` complete days (buckets with data).""" + """The records of one source for the last ``window_days`` complete days (buckets with data). + + ``kind`` and ``actual_reader`` select what is scored: by default the PV forecast against + ``site.pv_w`` on daylight intervals; the load forecast passes ``kind="load"`` and a reader + of the measured base load, and then every interval counts (a night is load too). + """ end = floor_to_step(now, PLANNING_STEP) start = end - timedelta(days=window_days) - actual = await data.read_series(SITE_PV_W, TimeRange(start=start, end=end), "15m", SeriesUnit.W) - issues = await data.forecast_issues(source_id, "pv", start - BUCKETS[-1][2]) + window = TimeRange(start=start, end=end) + if actual_reader is None: + actual = await data.read_series(SITE_PV_W, window, "15m", SeriesUnit.W) + else: + actual = await actual_reader(window) + daylight_only = kind == "pv" + issues = await data.forecast_issues(source_id, kind, start - BUCKETS[-1][2]) sums = {bucket: _Sums() for bucket, _, _ in BUCKETS} for index, issued in enumerate(issues): following = issues[index + 1] if index + 1 < len(issues) else None @@ -202,13 +220,15 @@ async def compute_accuracy( first = min(lo for _, lo, _ in ranges) last = max(hi for _, _, hi in ranges) forecast = await data.read_forecast( - source_id, "pv", issued, TimeRange(start=first - timedelta(hours=1), end=last) + source_id, kind, issued, TimeRange(start=first - timedelta(hours=1), end=last) ) if forecast is None: continue for bucket, lo, hi in ranges: measured = _slice(actual, lo, hi) - for error in quarter_errors(forecast.p50, measured, is_clipped=is_clipped): + for error in quarter_errors( + forecast.p50, measured, is_clipped=is_clipped, daylight_only=daylight_only + ): sums[bucket].add(error) return [ AccuracyRecord( @@ -329,6 +349,8 @@ async def update_accuracy( *, window_days: int = WINDOW_DAYS, is_clipped: ClipFilter = no_clipping, + kind: ForecastKind = "pv", + actual_reader: ActualReader | None = None, ) -> dict[str, list[AccuracyRecord]]: """The nightly job: recompute and store every source; one failing source does not stop the others (it is logged and keeps last night's result).""" @@ -337,7 +359,13 @@ async def update_accuracy( began = time.perf_counter() # runtime for the log only, never for the statistics try: records = await compute_accuracy( - data, source_id, now, window_days=window_days, is_clipped=is_clipped + data, + source_id, + now, + window_days=window_days, + is_clipped=is_clipped, + kind=kind, + actual_reader=actual_reader, ) await save_accuracy(learned, source_id, records, now=now, window_days=window_days) except Exception: @@ -359,7 +387,9 @@ async def update_accuracy( "NS_FORECAST_ACCURACY", "WINDOW_DAYS", "AccuracyData", + "ActualReader", "ClipFilter", + "ForecastKind", "LearnedParams", "StoredAccuracy", "accuracy_view", diff --git a/suncore/forecast/days.py b/suncore/forecast/days.py new file mode 100644 index 0000000..a6fefd4 --- /dev/null +++ b/suncore/forecast/days.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Local days and grid slices shared by the forecast statistics (PV, load, risk). + +A local day is ``[local midnight, next local midnight)`` in absolute time, so a DST day has 23 +or 25 hours. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, date, datetime, time, timedelta, tzinfo + +from suncore.core.series import TimeSeries + + +@dataclass(frozen=True, slots=True) +class LocalDay: + day: date + start: datetime + """Local midnight, in UTC.""" + end: datetime + + +def local_midnight(moment: datetime, tz: tzinfo) -> datetime: + local = moment.astimezone(tz) + return datetime.combine(local.date(), time(), tzinfo=tz) + + +def local_days(start: datetime, end: datetime, tz: tzinfo) -> list[LocalDay]: + """The complete local days inside ``[start, end)`` (DST days have 23 or 25 hours).""" + first = start.astimezone(tz).date() + if datetime.combine(first, time(), tzinfo=tz) < start: + first += timedelta(days=1) + days: list[LocalDay] = [] + current = first + while True: + # in UTC: Python subtracts two datetimes of the same zone on the wall clock (24 h even + # on a DST day), so the bounds are kept in absolute time + begin = datetime.combine(current, time(), tzinfo=tz).astimezone(UTC) + finish = datetime.combine(current + timedelta(days=1), time(), tzinfo=tz).astimezone(UTC) + if finish > end: + return days + days.append(LocalDay(current, begin, finish)) + current += timedelta(days=1) + + +def slice_series(series: TimeSeries, begin: datetime, end: datetime) -> TimeSeries: + """The grid-aligned part of ``series`` in ``[begin, end)`` (clamped to its extent).""" + step = series.step + first = max(0, (begin - series.start) // step) + last = max(first, min(len(series), -(-(end - series.start) // step))) + return TimeSeries( + start=series.start + step * first, + step=step, + values=series.values[first:last], + unit=series.unit, + ) + + +__all__ = ["LocalDay", "local_days", "local_midnight", "slice_series"] diff --git a/suncore/forecast/load.py b/suncore/forecast/load.py new file mode 100644 index 0000000..fde1b90 --- /dev/null +++ b/suncore/forecast/load.py @@ -0,0 +1,833 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Load forecast: base load profiles per day type, temperature term, residual quantiles +(Phase 4 WP6). Pure functions; the service (:mod:`suncore.forecast.load_service`) reads the +history and writes the result. + +**Base load** = the house consumption (``site.house_w``, which *includes* every device in the +house: ``house = pv + grid - battery``) minus the draw of every configured (controllable) device +(``device..power_w``). A device is subtracted exactly once; a quarter hour without a device +measurement counts that device as 0 W (the base load then still contains its draw) and is +counted in ``unmeasured_slots`` (the history rebuilt from Home Assistant's statistics has device +rows only for devices with a power sensor). The fit leaves such quarter hours out of a day +type's profile once it has ``MIN_CLEAN_DAYS`` (7) fully measured days; before that they are kept +and the day type is flagged (``unmeasured_kept``). A negative difference (sensor mismatch) is +clipped to 0 W and counted. +Controlled devices are forecast by the planner from their own models, never here. + +**Profile** per day type (:class:`DayType`) and local quarter hour of the day (96 slots, wall +clock: on a 25-hour day the repeated hour is averaged, on a 23-hour day the skipped slots simply +get no sample). Every complete local day with at least ``DAY_COVERAGE`` of its quarter hours +known contributes one sample per slot, weighted by exponential smoothing +(``0.5 ** (age_days / HALF_LIFE_DAYS)``, half-life 21 days). Robust against outliers: the +samples of one cell are clipped into ``median +- OUTLIER_Z * scale`` with the robust scale +``1.4826 * MAD``, at least half the median (clipped z-score), before the weighted mean. A day +type with few days is shrunk towards a fallback with +``(n * own + SHRINK_DAYS * fallback) / (n + SHRINK_DAYS)``: workday and weekend towards the +profile of all non-holiday days, holiday towards the (shrunk) weekend profile - a holiday looks +like a Sunday until enough holidays were seen. The profile is finally smoothed along the day +with the kernel ``(1/4, 1/2, 1/4)``. + +**Temperature term** (only with a configured ``weather`` entity): the daily residual (mean base +load of the day minus the mean of its profile) is regressed on heating and cooling degrees of +the day's mean outside temperature, ``HDD = max(0, 15 °C - T)`` and ``CDD = max(0, T - 22 °C)``, +by recency-weighted least squares, centred on the weighted mean degrees of the fitting days (so +an average day gets no offset), coefficients bounded to ``[0, MAX_W_PER_K]``. It is active from +``TEMPERATURE_MIN_DAYS`` days with a temperature and a spread of at least +``TEMPERATURE_MIN_SPREAD_K`` in one of the two regressors; the offset of a forecast day is +bounded to ``TEMPERATURE_MAX_SHARE`` of that day's mean profile. + +**Quantiles**: the in-sample residuals (actual - profile - temperature offset) per time-of-day +block (:class:`SlotBlock`) give the 10 % and 90 % quantiles, added to the P50 (pooled over all +blocks when a block has fewer than ``RESIDUAL_MIN_SAMPLES``). In-sample residuals are slightly +optimistic; the risk model learns the real error of the net energy separately (WP7). + +**Cold start**: with fewer than ``COLD_START_MIN_DAYS`` usable days the forecast is a flat line +at the mean of the last ``COLD_START_RECENT_DAYS`` days' known values with a wide band +(P10 = 0.5 x, P90 = 2 x), quality ``cold_start``; without any value there is no forecast +(``no_history``). The 28 days rebuilt from Home Assistant's statistics (PR 4-B) make the +profile usable from the first day. + +**Day types**: ``workday`` binary sensor ``on`` = workday; ``off`` on Monday-Friday = holiday, +on Saturday/Sunday = weekend (the sensor cannot tell the two apart, research note "Workday"); +without the sensor the weekday rule applies (Mon-Fri workday, Sat/Sun weekend, never holiday). +A sensor with ``days_offset`` describes today + offset, and ``workday.check_date(d)`` answers for +``d + offset``; the service compensates (:class:`WorkdayState`). +""" + +from __future__ import annotations + +import json +import math +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import UTC, date, datetime, timedelta, tzinfo +from enum import StrEnum, unique +from typing import Any, Final, Literal + +from suncore.core.series import ( + PLANNING_STEP, + Grid, + QuantileSeries, + SeriesUnit, + TimeSeries, +) +from suncore.forecast.days import local_days, local_midnight +from suncore.forecast.stats import ( + clip_outliers, + median, + recency_weight, + weighted_mean, + weighted_quantile, +) + +LOAD_SOURCE_ID: Final = "load_profile" +"""Source id of the profile forecast in ``timeseries.db`` and the accuracy statistics.""" + +SLOTS_PER_DAY: Final = 96 +HALF_LIFE_DAYS: Final = 21.0 +HISTORY_DAYS: Final = 56 +"""Look-back of the fit (eight weeks; older days weigh less than 16 %).""" +DAY_COVERAGE: Final = 0.75 +OUTLIER_Z: Final = 3.0 +OUTLIER_FLOOR_W: Final = 20.0 +OUTLIER_FLOOR_SHARE: Final = 0.5 +"""Lowest robust scale of a cell: ``max(OUTLIER_FLOOR_W, OUTLIER_FLOOR_SHARE * |median|)``. A +cell of near-identical values (MAD about 0) would otherwise clip a real level change (a new +fridge, a heating season) onto the old median for weeks.""" +SHRINK_DAYS: Final = 3.0 +SMOOTHING_KERNEL: Final = (0.25, 0.5, 0.25) +COLD_START_MIN_DAYS: Final = 3 +MIN_CLEAN_DAYS: Final = 7 +"""Fully measured days (every device measured) a day type needs before quarter hours with an +unmeasured device are left out of its profile.""" +COLD_START_RECENT_DAYS: Final = 7 +COLD_START_LOW: Final = 0.5 +COLD_START_HIGH: Final = 2.0 +RESIDUAL_MIN_SAMPLES: Final = 48 +LOW_QUANTILE: Final = 0.1 +HIGH_QUANTILE: Final = 0.9 +HEATING_BASE_C: Final = 15.0 +COOLING_BASE_C: Final = 22.0 +TEMPERATURE_MIN_DAYS: Final = 14 +TEMPERATURE_MIN_SPREAD_K: Final = 1.0 +TEMPERATURE_DAY_COVERAGE: Final = 0.5 +MAX_W_PER_K: Final = 300.0 +TEMPERATURE_MAX_SHARE: Final = 0.5 +MIN_TEMPERATURE_HOURS: Final = 6 +"""Hours with a temperature a day needs for a daily mean (default of the helper).""" +FORECAST_TEMPERATURE_HOURS: Final = 20 +"""Hours with a temperature (recorded or forecast) a *forecast* day needs before the temperature +term shifts it: a partial day's mean (e.g. only the warm afternoon) would bias the offset.""" + + +@unique +class DayType(StrEnum): + WORKDAY = "workday" + WEEKEND = "weekend" + HOLIDAY = "holiday" + + +@unique +class DayTypeSource(StrEnum): + """Where the day type of a date came from.""" + + WORKDAY_SENSOR = "workday_sensor" + """The state of the ``workday`` binary sensor (today only).""" + CHECK_DATE = "check_date" + """The read-only action ``workday.check_date``.""" + WEEKDAY_RULE = "weekday_rule" + """Mon-Fri workday, Sat/Sun weekend (no sensor, or it did not answer).""" + + +@dataclass(frozen=True, slots=True) +class DayInfo: + day_type: DayType + source: DayTypeSource + + +@unique +class LoadQuality(StrEnum): + OK = "ok" + COLD_START = "cold_start" + NO_HISTORY = "no_history" + + +@unique +class SlotBlock(StrEnum): + """Time-of-day block the residual quantiles are pooled in (local wall clock).""" + + NIGHT = "night" + """00:00-06:00""" + MORNING = "morning" + """06:00-10:00""" + MIDDAY = "midday" + """10:00-16:00""" + EVENING = "evening" + """16:00-22:00""" + LATE = "late" + """22:00-24:00""" + + +_BLOCK_EDGES: Final = ((24, SlotBlock.NIGHT), (40, SlotBlock.MORNING), (64, SlotBlock.MIDDAY)) + + +def slot_block(slot: int) -> SlotBlock: + """Block of a quarter-hour index (0 = 00:00-00:15 local).""" + for edge, block in _BLOCK_EDGES: + if slot < edge: + return block + return SlotBlock.EVENING if slot < 88 else SlotBlock.LATE + + +def weekday_rule(day: date) -> DayType: + return DayType.WORKDAY if day.weekday() < 5 else DayType.WEEKEND + + +def day_type_from_workday(day: date, workday: bool) -> DayType: + """``workday`` as the sensor/action says; ``off`` on Mon-Fri is a holiday.""" + if workday: + return DayType.WORKDAY + return DayType.HOLIDAY if day.weekday() < 5 else DayType.WEEKEND + + +def local_slot(moment: datetime, tz: tzinfo) -> tuple[date, int]: + """Local date and quarter-hour index (0..95) of an instant.""" + local = moment.astimezone(tz) + return local.date(), local.hour * 4 + local.minute // 15 + + +# --- base load ------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class BaseLoad: + """The base load history and how much of it lacks device measurements.""" + + series: TimeSeries + unmeasured_slots: Mapping[str, int] = field(default_factory=dict) + """Device id -> quarter hours (with a house value) without a device measurement (0 W + assumed, so the base load still contains that device's draw).""" + clamped_slots: int = 0 + """Quarter hours where the devices exceeded the house (clipped to 0 W).""" + unmeasured: tuple[bool, ...] = () + """Per slot of ``series``: a device had no measurement (its draw is inside the value).""" + + +def base_load(house: TimeSeries, devices: Mapping[str, TimeSeries]) -> BaseLoad: + """``house - sum(devices)`` per slot (W); every device series must share the house grid.""" + for device_id, series in devices.items(): + if not series.same_grid(house): + msg = f"base_load: device {device_id} is not on the house grid" + raise ValueError(msg) + unmeasured = dict.fromkeys(devices, 0) + clamped = 0 + values: list[float | None] = [] + mask: list[bool] = [] + for i, total in enumerate(house.values): + if total is None: + values.append(None) + mask.append(False) + continue + draw = 0.0 + missing = False + for device_id, series in devices.items(): + value = series.values[i] + if value is None: + unmeasured[device_id] += 1 + missing = True + else: + draw += max(0.0, value) + rest = total - draw + if rest < 0: + clamped += 1 + rest = 0.0 + values.append(rest) + mask.append(missing) + series = house.model_copy(update={"values": tuple(values), "unit": SeriesUnit.W}) + return BaseLoad( + series=series, + unmeasured_slots={k: v for k, v in unmeasured.items() if v}, + clamped_slots=clamped, + unmeasured=tuple(mask), + ) + + +# --- day grids ----------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class DayValues: + """One complete local day of a 15-minute series by wall-clock slot.""" + + day: date + slots: tuple[float | None, ...] + """96 entries; ``None`` = unknown or the slot does not exist that day (spring forward).""" + coverage: float + """Known share of the slots that exist that day.""" + + def known(self) -> list[tuple[int, float]]: + return [(i, v) for i, v in enumerate(self.slots) if v is not None] + + +def day_values(series: TimeSeries, tz: tzinfo, until: datetime) -> list[DayValues]: + """The complete local days of ``series`` that end at or before ``until``.""" + if not len(series): + return [] + days = local_days(series.start, min(series.end, until), tz) + out: list[DayValues] = [] + for day in days: + sums = [0.0] * SLOTS_PER_DAY + counts = [0] * SLOTS_PER_DAY + exists = [False] * SLOTS_PER_DAY + moment = day.start + while moment < day.end: + _, slot = local_slot(moment, tz) + exists[slot] = True + index = (moment - series.start) // series.step + if 0 <= index < len(series): + value = series.values[index] + if value is not None: + sums[slot] += value + counts[slot] += 1 + moment += PLANNING_STEP + slots = tuple(s / c if c else None for s, c in zip(sums, counts, strict=True)) + existing = sum(exists) + known = sum(1 for c in counts if c) + out.append(DayValues(day.day, slots, known / existing if existing else 0.0)) + return out + + +# --- temperature --------------------------------------------------------------------------- + +TemperatureReason = Literal["active", "not_configured", "too_few_days", "no_spread"] + + +def degree_days(temperature_c: float) -> tuple[float, float]: + """``(heating, cooling)`` degrees of a daily mean temperature.""" + return max(0.0, HEATING_BASE_C - temperature_c), max(0.0, temperature_c - COOLING_BASE_C) + + +@dataclass(frozen=True, slots=True) +class TemperatureTerm: + """The fitted temperature dependence of the daily base load (W per kelvin).""" + + active: bool + reason: TemperatureReason + days: int = 0 + heating_w_per_k: float = 0.0 + cooling_w_per_k: float = 0.0 + mean_hdd: float = 0.0 + mean_cdd: float = 0.0 + + def offset_w(self, temperature_c: float | None) -> float: + """Offset of a day with mean ``temperature_c`` against an average fitting day.""" + if not self.active or temperature_c is None: + return 0.0 + hdd, cdd = degree_days(temperature_c) + return self.heating_w_per_k * (hdd - self.mean_hdd) + self.cooling_w_per_k * ( + cdd - self.mean_cdd + ) + + +def _slope(xs: Sequence[float], ys: Sequence[float], ws: Sequence[float]) -> float: + """Weighted least-squares slope of centred data (0 when ``xs`` has no spread).""" + sxx = sum(w * x * x for x, w in zip(xs, ws, strict=True)) + if sxx <= 1e-12: + return 0.0 + return sum(w * x * y for x, y, w in zip(xs, ys, ws, strict=True)) / sxx + + +def _clamp_coef(value: float) -> float: + return min(MAX_W_PER_K, max(0.0, value)) + + +def fit_temperature(samples: Sequence[tuple[float, float, float]]) -> TemperatureTerm: + """Fit the term from ``(daily residual W, daily mean °C, weight)`` samples.""" + if len(samples) < TEMPERATURE_MIN_DAYS: + return TemperatureTerm(active=False, reason="too_few_days", days=len(samples)) + ws = [w for _, _, w in samples] + hdd = [degree_days(t)[0] for _, t, _ in samples] + cdd = [degree_days(t)[1] for _, t, _ in samples] + res = [r for r, _, _ in samples] + mh, mc, mr = weighted_mean(hdd, ws), weighted_mean(cdd, ws), weighted_mean(res, ws) + xh = [h - mh for h in hdd] + xc = [c - mc for c in cdd] + y = [r - mr for r in res] + + def spread(xs: Sequence[float]) -> float: + return math.sqrt(weighted_mean([x * x for x in xs], ws)) + + has_h = spread(xh) >= TEMPERATURE_MIN_SPREAD_K + has_c = spread(xc) >= TEMPERATURE_MIN_SPREAD_K + if not has_h and not has_c: + return TemperatureTerm(active=False, reason="no_spread", days=len(samples)) + h = c = 0.0 + if has_h and has_c: + shh = sum(w * a * a for a, w in zip(xh, ws, strict=True)) + scc = sum(w * a * a for a, w in zip(xc, ws, strict=True)) + shc = sum(w * a * b for a, b, w in zip(xh, xc, ws, strict=True)) + shy = sum(w * a * b for a, b, w in zip(xh, y, ws, strict=True)) + scy = sum(w * a * b for a, b, w in zip(xc, y, ws, strict=True)) + det = shh * scc - shc * shc + if det > 1e-9: + h = (shy * scc - scy * shc) / det + c = (scy * shh - shy * shc) / det + if det <= 1e-9 or h < 0 or c < 0: # fall back to the stronger single regressor + h1, c1 = _slope(xh, y, ws), _slope(xc, y, ws) + h, c = (h1, 0.0) if abs(h1) * spread(xh) >= abs(c1) * spread(xc) else (0.0, c1) + elif has_h: + h = _slope(xh, y, ws) + else: + c = _slope(xc, y, ws) + return TemperatureTerm( + active=True, + reason="active", + days=len(samples), + heating_w_per_k=_clamp_coef(h), + cooling_w_per_k=_clamp_coef(c), + mean_hdd=mh, + mean_cdd=mc, + ) + + +# --- model ----------------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class LoadModel: + """A fitted load profile model (or the cold-start/no-history placeholder).""" + + quality: LoadQuality + days: int + """Complete local days that went into the fit.""" + profiles: Mapping[DayType, tuple[float, ...]] = field(default_factory=dict) + day_counts: Mapping[DayType, int] = field(default_factory=dict) + residuals: Mapping[SlotBlock, tuple[float, float]] = field(default_factory=dict) + """``(q10, q90)`` offsets (W) per block, ``q10 <= 0 <= q90``.""" + flat_w: float | None = None + """Cold start: the flat level (W).""" + temperature: TemperatureTerm = TemperatureTerm(active=False, reason="not_configured") + unmeasured_kept: frozenset[DayType] = frozenset() + """Day types whose profile still contains quarter hours with an unmeasured device (fewer + than ``MIN_CLEAN_DAYS`` fully measured days of that type).""" + + def day_mean(self, day_type: DayType) -> float: + profile = self.profiles.get(day_type) + return sum(profile) / len(profile) if profile else 0.0 + + def quantiles( + self, slot: int, day_type: DayType, temperature_c: float | None + ) -> tuple[float, float, float] | None: + """``(p10, p50, p90)`` W of one slot; ``None`` without a model.""" + if self.quality is LoadQuality.NO_HISTORY or ( + self.quality is LoadQuality.COLD_START and self.flat_w is None + ): + return None + if self.quality is LoadQuality.COLD_START: + assert self.flat_w is not None + return COLD_START_LOW * self.flat_w, self.flat_w, COLD_START_HIGH * self.flat_w + profile = self.profiles[day_type] + bound = TEMPERATURE_MAX_SHARE * self.day_mean(day_type) + offset = min(bound, max(-bound, self.temperature.offset_w(temperature_c))) + p50 = max(0.0, profile[slot] + offset) + q10, q90 = self.residuals.get(slot_block(slot), (0.0, 0.0)) + return max(0.0, min(p50, p50 + q10)), p50, max(p50, p50 + q90) + + +def _cell_mean(samples: Sequence[tuple[float, float]]) -> float: + values = [v for v, _ in samples] + floor = max(OUTLIER_FLOOR_W, OUTLIER_FLOOR_SHARE * abs(median(values))) + clipped = clip_outliers(values, OUTLIER_Z, floor) + return weighted_mean(clipped, [w for _, w in samples]) + + +def _smooth(profile: Sequence[float]) -> tuple[float, ...]: + n = len(profile) + a, b, c = SMOOTHING_KERNEL + return tuple( + a * profile[(i - 1) % n] + b * profile[i] + c * profile[(i + 1) % n] for i in range(n) + ) + + +def _shrink( + cells: Sequence[list[tuple[float, float]]], fallback: Sequence[float] +) -> tuple[float, ...]: + out: list[float] = [] + for slot, samples in enumerate(cells): + if not samples: + out.append(fallback[slot]) + continue + n = len(samples) + out.append((n * _cell_mean(samples) + SHRINK_DAYS * fallback[slot]) / (n + SHRINK_DAYS)) + return tuple(out) + + +def _cold_start( + base: TimeSeries, now: datetime, days: int, *, temperature_configured: bool +) -> LoadModel: + term = TemperatureTerm( + active=False, reason="too_few_days" if temperature_configured else "not_configured" + ) + since = now - timedelta(days=COLD_START_RECENT_DAYS) + recent = [v for begin, _, v in base.slots() if v is not None and since <= begin < now] + if not recent: + return LoadModel(quality=LoadQuality.NO_HISTORY, days=days, temperature=term) + return LoadModel( + quality=LoadQuality.COLD_START, + days=days, + flat_w=sum(recent) / len(recent), + temperature=term, + ) + + +def _profiles( + days: Sequence[DayValues], weights: Mapping[date, float], kinds: Mapping[date, DayType] +) -> dict[DayType, tuple[float, ...]]: + pooled: list[list[tuple[float, float]]] = [[] for _ in range(SLOTS_PER_DAY)] + by_type: dict[DayType, list[list[tuple[float, float]]]] = { + t: [[] for _ in range(SLOTS_PER_DAY)] for t in DayType + } + for d in days: + for slot, value in d.known(): + if kinds[d.day] is not DayType.HOLIDAY: # holidays are special, not "any day" + pooled[slot].append((value, weights[d.day])) + by_type[kinds[d.day]][slot].append((value, weights[d.day])) + pooled_mean = [_cell_mean(cell) if cell else math.nan for cell in pooled] + known = [v for v in pooled_mean if not math.isnan(v)] + level = sum(known) / len(known) if known else 0.0 + base_profile = tuple(level if math.isnan(v) else v for v in pooled_mean) + profiles: dict[DayType, tuple[float, ...]] = {} + profiles[DayType.WORKDAY] = _shrink(by_type[DayType.WORKDAY], base_profile) + profiles[DayType.WEEKEND] = _shrink(by_type[DayType.WEEKEND], base_profile) + profiles[DayType.HOLIDAY] = _shrink(by_type[DayType.HOLIDAY], profiles[DayType.WEEKEND]) + return {t: _smooth(p) for t, p in profiles.items()} + + +def _daily_temperatures(temperature: TimeSeries, tz: tzinfo, until: datetime) -> dict[date, float]: + out: dict[date, float] = {} + for d in day_values(temperature, tz, until): + known = [v for _, v in d.known()] + if known and d.coverage >= TEMPERATURE_DAY_COVERAGE: + out[d.day] = sum(known) / len(known) + return out + + +def _temperature_term( + days: Sequence[DayValues], + weights: Mapping[date, float], + kinds: Mapping[date, DayType], + profiles: Mapping[DayType, tuple[float, ...]], + daily_temp: Mapping[date, float], +) -> TemperatureTerm: + daily: list[tuple[float, float, float]] = [] + for d in days: + if d.day not in daily_temp: + continue + pairs = d.known() + profile = profiles[kinds[d.day]] + residual = sum(v - profile[s] for s, v in pairs) / len(pairs) + daily.append((residual, daily_temp[d.day], weights[d.day])) + return fit_temperature(daily) + + +def _residuals( + model: LoadModel, + days: Sequence[DayValues], + weights: Mapping[date, float], + kinds: Mapping[date, DayType], + daily_temp: Mapping[date, float], +) -> dict[SlotBlock, tuple[float, float]]: + blocks: dict[SlotBlock, list[tuple[float, float]]] = {b: [] for b in SlotBlock} + for d in days: + temp = daily_temp.get(d.day) + for slot, value in d.known(): + fitted = model.quantiles(slot, kinds[d.day], temp) + assert fitted is not None + blocks[slot_block(slot)].append((value - fitted[1], weights[d.day])) + everything = [s for samples in blocks.values() for s in samples] + residuals: dict[SlotBlock, tuple[float, float]] = {} + for block, samples in blocks.items(): + use = samples if len(samples) >= RESIDUAL_MIN_SAMPLES else everything + if not use: + residuals[block] = (0.0, 0.0) + continue + values, ws = [v for v, _ in use], [w for _, w in use] + q10 = weighted_quantile(values, ws, LOW_QUANTILE) + q90 = weighted_quantile(values, ws, HIGH_QUANTILE) + residuals[block] = (min(0.0, q10), max(0.0, q90)) + return residuals + + +def _drop_unmeasured( + days: list[DayValues], + *, + kinds: Mapping[date, DayType], + base: TimeSeries, + unmeasured: Sequence[bool], + tz: tzinfo, + until: datetime, +) -> tuple[list[DayValues], frozenset[DayType]]: + """Remove the quarter hours with an unmeasured device from the days of every day type that + has at least ``MIN_CLEAN_DAYS`` fully measured days; the other types keep them (flagged).""" + if not any(unmeasured): + return days, frozenset() + masked_values = tuple( + None if i < len(unmeasured) and unmeasured[i] else v for i, v in enumerate(base.values) + ) + masked = { + d.day: d for d in day_values(base.model_copy(update={"values": masked_values}), tz, until) + } + clean = dict.fromkeys(DayType, 0) + for d in days: + if masked[d.day].slots == d.slots: + clean[kinds[d.day]] += 1 + kept = frozenset( + t + for t, n in clean.items() + if n < MIN_CLEAN_DAYS + and any(kinds[d.day] is t and masked[d.day].slots != d.slots for d in days) + ) + out = [d if kinds[d.day] in kept else masked[d.day] for d in days] + return [d for d in out if d.known()], kept + + +def fit_load_model( + base: TimeSeries, + *, + now: datetime, + tz: tzinfo, + day_types: Mapping[date, DayType], + temperature: TimeSeries | None = None, + temperature_configured: bool = False, + half_life_days: float = HALF_LIFE_DAYS, + unmeasured: Sequence[bool] = (), +) -> LoadModel: + """Fit the profiles (module doc) from the base load history up to ``now``. + + ``day_types`` maps history dates to their type (missing dates: weekday rule); + ``temperature`` is the recorded outside temperature (°C, 15 minutes); ``unmeasured`` flags + the slots of ``base`` where a device had no measurement (:attr:`BaseLoad.unmeasured`). + """ + today = local_midnight(now, tz).astimezone(UTC) + days = [d for d in day_values(base, tz, today) if d.coverage >= DAY_COVERAGE] + if len(days) < COLD_START_MIN_DAYS: + return _cold_start(base, now, len(days), temperature_configured=temperature_configured) + today_date = now.astimezone(tz).date() + kinds = {d.day: day_types.get(d.day, weekday_rule(d.day)) for d in days} + days, kept = _drop_unmeasured( + days, kinds=kinds, base=base, unmeasured=unmeasured, tz=tz, until=today + ) + weights = {d.day: recency_weight((today_date - d.day).days, half_life_days) for d in days} + profiles = _profiles(days, weights, kinds) + counts = {t: sum(1 for d in days if kinds[d.day] is t) for t in DayType} + term = TemperatureTerm(active=False, reason="not_configured") + daily_temp: dict[date, float] = {} + if temperature_configured: + if temperature is not None: + daily_temp = _daily_temperatures(temperature, tz, today) + term = _temperature_term(days, weights, kinds, profiles, daily_temp) + model = LoadModel( + quality=LoadQuality.OK, + days=len(days), + profiles=profiles, + day_counts=counts, + temperature=term, + ) + residuals = _residuals(model, days, weights, kinds, daily_temp) + return LoadModel( + quality=LoadQuality.OK, + days=len(days), + profiles=profiles, + day_counts=counts, + residuals=residuals, + temperature=term, + unmeasured_kept=kept, + ) + + +def forecast_load( + model: LoadModel, + grid: Grid, + tz: tzinfo, + day_types: Mapping[date, DayType], + daily_temperature: Mapping[date, float] | None = None, +) -> QuantileSeries | None: + """P10/P50/P90 of the base load (W) on ``grid``; ``None`` without a model.""" + temps = daily_temperature or {} + lows: list[float | None] = [] + mids: list[float | None] = [] + highs: list[float | None] = [] + for begin in grid.starts(): + day, slot = local_slot(begin, tz) + q = model.quantiles(slot, day_types.get(day, weekday_rule(day)), temps.get(day)) + if q is None: + return None + lows.append(q[0]) + mids.append(q[1]) + highs.append(q[2]) + + def series(values: list[float | None]) -> TimeSeries: + return TimeSeries(start=grid.start, step=grid.step, values=tuple(values), unit=SeriesUnit.W) + + return QuantileSeries(p10=series(lows), p50=series(mids), p90=series(highs)) + + +# --- Home Assistant shapes (parsed here, called by the adapter) ---------------------------- + + +_TO_CELSIUS: Final[dict[str | None, Callable[[float], float]]] = { + None: lambda t: t, + "°C": lambda t: t, + "C": lambda t: t, + "°F": lambda t: (t - 32.0) * 5.0 / 9.0, + "F": lambda t: (t - 32.0) * 5.0 / 9.0, + "K": lambda t: t - 273.15, +} + + +def temperature_c(value: object, unit: str | None) -> float | None: + """A temperature in °C from a Home Assistant value and its unit (``°C``/``°F``/``K``). + + ``None`` for a missing/non-numeric value or an unknown unit (never guessed). A weather + entity always reports ``temperature_unit`` (``homeassistant/components/weather/entity.py`` + ``state_attributes``, core ``5cfe9aad``; values ``UnitOfTemperature``); a missing unit is + read as °C. + """ + if isinstance(value, bool) or not isinstance(value, int | float | str): + return None + try: + number = float(value) + except ValueError: + return None + if not math.isfinite(number): + return None + convert = _TO_CELSIUS.get(unit) + return None if convert is None else convert(number) + + +def parse_hourly_temperatures( + response: Mapping[str, Any], entity_id: str, unit: str | None +) -> list[tuple[datetime, float]]: + """``(time, °C)`` points of a ``weather.get_forecasts`` response for one entity. + + Shape (``docs/research/ha-websocket-api.md`` "Weather forecasts"): + ``{entity_id: {"forecast": [{"datetime": iso, "temperature": float | null, ...}]}}``; + only ``datetime`` is guaranteed. Items without a time zone, a parsable time or a + temperature are skipped. + """ + body = response.get(entity_id) + items = body.get("forecast") if isinstance(body, Mapping) else None + if not isinstance(items, list): + return [] + points: list[tuple[datetime, float]] = [] + for item in items: + if not isinstance(item, Mapping): + continue + raw = item.get("datetime") + if not isinstance(raw, str): + continue + try: + moment = datetime.fromisoformat(raw) + except ValueError: + continue + if moment.tzinfo is None or moment.utcoffset() is None: + continue + value = temperature_c(item.get("temperature"), unit) + if value is not None: + points.append((moment.astimezone(UTC), value)) + points.sort(key=lambda p: p[0]) + return points + + +def daily_mean_temperatures( + points: Iterable[tuple[datetime, float]], tz: tzinfo, *, min_hours: int = MIN_TEMPERATURE_HOURS +) -> dict[date, float]: + """Mean temperature per local date: points are averaged per local hour first (so a + 15-minute measurement and an hourly forecast weigh alike), then over the hours; a date with + fewer than ``min_hours`` hours is left out.""" + hours: dict[tuple[date, int], list[float]] = {} + for moment, value in points: + local = moment.astimezone(tz) + hours.setdefault((local.date(), local.hour), []).append(value) + per_day: dict[date, list[float]] = {} + for (day, _), values in hours.items(): + per_day.setdefault(day, []).append(sum(values) / len(values)) + return {d: sum(v) / len(v) for d, v in per_day.items() if len(v) >= min_hours} + + +def parse_check_date(response: Mapping[str, Any], entity_id: str) -> bool | None: + """``workday.check_date`` response ``{entity_id: {"workday": bool}}`` -> the bool.""" + body = response.get(entity_id) + value = body.get("workday") if isinstance(body, Mapping) else None + return value if isinstance(value, bool) else None + + +@dataclass(frozen=True, slots=True) +class WorkdayState: + """The ``workday`` binary sensor as SunCore reads it. + + ``workday`` is the state (``on`` = workday) of the date ``today + offset``: Home Assistant + evaluates the sensor and ``workday.check_date`` for the *adjusted* date (``days_offset`` + added, ``homeassistant/components/workday/entity.py`` ``date_is_workday``), so the sensor + describes today + offset and ``check_date(d)`` describes ``d + offset``. ``fingerprint`` + identifies the configuration the answers belong to (cache key). + """ + + workday: bool + offset: int + fingerprint: str + + +def parse_workday_sensor( + entity_id: str, state: str, attributes: Mapping[str, Any] +) -> WorkdayState | None: + """The sensor's state and configuration; ``None`` if the state is not ``on``/``off`` or + ``days_offset`` is not an integer (the offset could not be compensated).""" + offset = attributes.get("days_offset", 0) + if isinstance(offset, bool) or not isinstance(offset, int): + return None + if state not in ("on", "off"): + return None + config = { + "entity_id": entity_id, + "workdays": attributes.get("workdays"), + "excludes": attributes.get("excludes"), + "days_offset": offset, + } + fingerprint = json.dumps(config, sort_keys=True, default=str) + return WorkdayState(workday=state == "on", offset=offset, fingerprint=fingerprint) + + +__all__ = [ + "COLD_START_MIN_DAYS", + "FORECAST_TEMPERATURE_HOURS", + "HALF_LIFE_DAYS", + "HISTORY_DAYS", + "LOAD_SOURCE_ID", + "BaseLoad", + "DayInfo", + "DayType", + "DayTypeSource", + "DayValues", + "LoadModel", + "LoadQuality", + "SlotBlock", + "TemperatureReason", + "TemperatureTerm", + "WorkdayState", + "base_load", + "daily_mean_temperatures", + "day_type_from_workday", + "day_values", + "degree_days", + "fit_load_model", + "fit_temperature", + "forecast_load", + "local_slot", + "parse_check_date", + "parse_hourly_temperatures", + "parse_workday_sensor", + "slot_block", + "temperature_c", + "weekday_rule", +] diff --git a/suncore/forecast/load_service.py b/suncore/forecast/load_service.py new file mode 100644 index 0000000..eaeb938 --- /dev/null +++ b/suncore/forecast/load_service.py @@ -0,0 +1,615 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The load forecast service: history, day types, temperature, refresh, store (Phase 4 WP6). + +* **Fit** (on start, then at most hourly and after the nightly job): read the last + ``HISTORY_DAYS`` of ``site.house_w`` and every configured device's ``device..power_w`` + (15-minute means), form the base load (:func:`suncore.forecast.load.base_load`), resolve the + day types and fit :class:`~suncore.forecast.load.LoadModel`. The device histories are kept + for the planner (4-F) - controlled devices are never part of the base load. +* **Day types** with a mapped ``workday`` binary sensor: the date ``today + days_offset`` from + its state, every other date ``d`` from the read-only action ``workday.check_date`` asked for + ``d - days_offset`` (Home Assistant evaluates both for the date plus the offset; rate limited + per date, at most ``MAX_CHECKS_PER_RUN`` calls per refresh, forecast days first, then the + history newest first). The answers are cached in ``learned_param`` (namespace + ``load_profile``, key ``day_types``) with a fingerprint of the sensor's entity id, + ``workdays``, ``excludes`` and ``days_offset``; a changed fingerprint drops the cache (a changed + country or holiday list is not visible in the attributes - remap the sensor to reset). A date + without an answer (no sensor, its state unavailable, the action failed or is rate limited) + uses the weekday rule and is asked again later. +* **Temperature** with a mapped ``weather`` entity: its ``temperature`` attribute is recorded + with every refresh (key ``weather.temperature_c``, °C) - Home Assistant keeps no long-term + statistics of a weather entity, so the term learns from SunCore's own recording and becomes + active after 14 days; the hourly forecast (``weather.get_forecasts``) gives the forecast + days' mean temperatures (today combines the recorded hours with the forecast ones). +* **Refresh** every 15 minutes: the forecast from the current slot for 48 h; it is stored in + ``timeseries.db`` (kind ``load``, source ``load_profile``) at most hourly. +* **Nightly**: accuracy (MAE and bias = actual - forecast per horizon bucket, all quarter + hours) of ``load_profile`` against the measured base load, then a refit. + +Storage and Home Assistant are reached only through protocols (the composition root passes +the time-series store, the learned-parameter store and an adapter); time and sleeping are +injected. Failures are logged and never raised out of the loop. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, date, datetime, timedelta, tzinfo +from typing import Any, Final, Literal, Protocol + +import structlog + +from suncore.core.forecast import AccuracyRecord +from suncore.core.measurements import SITE_HOUSE_W, WEATHER_TEMPERATURE_C, device_power_key +from suncore.core.series import ( + PLANNING_STEP, + QuantileSeries, + SeriesUnit, + TimeRange, + TimeSeries, + floor_to_step, + planning_grid, +) +from suncore.forecast.accuracy import AccuracyData, LearnedParams, update_accuracy +from suncore.forecast.days import local_midnight +from suncore.forecast.load import ( + FORECAST_TEMPERATURE_HOURS, + HISTORY_DAYS, + LOAD_SOURCE_ID, + BaseLoad, + DayInfo, + DayType, + DayTypeSource, + LoadModel, + LoadQuality, + WorkdayState, + base_load, + daily_mean_temperatures, + day_type_from_workday, + fit_load_model, + forecast_load, + weekday_rule, +) +from suncore.forecast.views import ( + MAX_VIEW_HOURS, + LoadDayView, + LoadForecastView, + LoadTemperatureView, +) + +log = structlog.get_logger(__name__) + +REFRESH_INTERVAL_S: Final = 900.0 +STORE_INTERVAL_S: Final = 3600.0 +REFIT_INTERVAL_S: Final = 3600.0 +WEATHER_INTERVAL_S: Final = 3600.0 +HORIZON: Final = timedelta(hours=48) +MAX_CHECKS_PER_RUN: Final = 12 +CHECK_DATE_INTERVAL_S: Final = 6 * 3600.0 +"""Rate-limit interval of ``workday.check_date`` per date (each date has its own slot).""" +NS_LOAD_PROFILE: Final = "load_profile" +"""``learned_param`` namespace (equal to ``suncore.storage.learned.NS_LOAD_PROFILE``).""" +DAY_TYPES_KEY: Final = "day_types" +DAY_TYPES_SCHEMA: Final = 1 + + +class LoadData(AccuracyData, Protocol): + """The part of ``TimeseriesStore`` the load forecast uses.""" + + async def store_forecast( + self, + source_id: str, + kind: Literal["pv", "load", "price"], + issued_at: datetime, + forecast: QuantileSeries | TimeSeries, + ) -> int: ... + + async def append_measurements( + self, rows: Iterable[tuple[str, datetime, float | None]] + ) -> int: ... + + +class LoadHaReader(Protocol): + """What the load forecast reads from Home Assistant (implemented in ``suncore.app``).""" + + def workday_state(self, entity_id: str) -> WorkdayState | None: + """The ``workday`` sensor's state, offset and configuration; ``None`` if unknown.""" + ... + + async def check_workday(self, entity_id: str, day: date, *, min_interval_s: float) -> bool: + """``workday.check_date`` for ``day`` (Home Assistant answers for ``day + offset``); + raises on any failure (rate limit included).""" + ... + + def outside_temperature(self, entity_id: str) -> float | None: + """The weather entity's current temperature in °C (``None``: unknown).""" + ... + + async def hourly_temperatures( + self, entity_id: str, *, min_interval_s: float + ) -> list[tuple[datetime, float]]: + """The hourly forecast temperatures (°C); raises on failure.""" + ... + + +@dataclass(frozen=True, slots=True) +class LoadInputs: + """Which entities and devices the service uses (from ``forecast.load`` and ``devices``).""" + + device_ids: tuple[str, ...] = () + workday_entity: str | None = None + weather_entity: str | None = None + + +class LoadForecastService: + """Fits, refreshes, stores and serves the base load forecast.""" + + def __init__( + self, + data: LoadData | None, + learned: LearnedParams | None, + *, + clock: Callable[[], datetime], + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + tz: Callable[[], tzinfo] = lambda: UTC, + inputs: LoadInputs | None = None, + reader: LoadHaReader | None = None, + refresh_interval_s: float = REFRESH_INTERVAL_S, + store_interval_s: float = STORE_INTERVAL_S, + refit_interval_s: float = REFIT_INTERVAL_S, + weather_interval_s: float = WEATHER_INTERVAL_S, + horizon: timedelta = HORIZON, + history_days: int = HISTORY_DAYS, + ) -> None: + self._data = data + self._learned = learned + self._clock = clock + self._sleep = sleep + self._tz = tz + self._inputs = inputs or LoadInputs() + self._reader = reader + self._refresh_interval_s = refresh_interval_s + self._store_interval = timedelta(seconds=store_interval_s) + self._refit_interval = timedelta(seconds=refit_interval_s) + self._weather_interval = timedelta(seconds=weather_interval_s) + self._horizon = horizon + self._history_days = history_days + self._model: LoadModel | None = None + self._fitted_at: datetime | None = None + self._base: BaseLoad | None = None + self._devices: dict[str, TimeSeries] = {} + self._cache: dict[date, DayInfo] = {} + self._fingerprint: str | None = None + """The workday configuration the cached answers belong to (m6).""" + self._cache_loaded = False + self._weather_points: list[tuple[datetime, float]] = [] + self._weather_at: datetime | None = None + self._observed: list[tuple[datetime, float]] = [] + """Today's recorded outside temperatures (from the fit's read plus every refresh).""" + self._last_stored_at: datetime | None = None + self._lock = asyncio.Lock() + + # --- inputs --------------------------------------------------------------------- + + @property + def inputs(self) -> LoadInputs: + return self._inputs + + @property + def model(self) -> LoadModel | None: + return self._model + + @property + def device_histories(self) -> Mapping[str, TimeSeries]: + """The controlled devices' measured power (15 minutes, W) of the fit window.""" + return dict(self._devices) + + # --- day types ------------------------------------------------------------------ + + def day_info(self, day: date, today: date | None = None) -> DayInfo: + """The day type of ``day`` and where it came from (module doc).""" + entity = self._inputs.workday_entity + if entity is not None and self._reader is not None: + live = self._reader.workday_state(entity) + if ( + live is not None + and today is not None + and day == today + timedelta(days=live.offset) + ): + return DayInfo( + day_type_from_workday(day, live.workday), DayTypeSource.WORKDAY_SENSOR + ) + if live is None or live.fingerprint == self._fingerprint: + cached = self._cache.get(day) + if cached is not None: + return cached + return DayInfo(weekday_rule(day), DayTypeSource.WEEKDAY_RULE) + + def day_types(self, days: Iterable[date], today: date) -> dict[date, DayType]: + return {d: self.day_info(d, today).day_type for d in days} + + async def _load_cache(self) -> None: + if self._cache_loaded or self._learned is None: + self._cache_loaded = True + return + self._cache_loaded = True + try: + for entry in await self._learned.in_namespace(NS_LOAD_PROFILE): + if entry.key == DAY_TYPES_KEY and entry.schema_version == DAY_TYPES_SCHEMA: + self._cache.update(_decode_days(entry.payload)) + fingerprint = entry.payload.get("fingerprint") + self._fingerprint = fingerprint if isinstance(fingerprint, str) else None + except Exception: + log.exception("forecast.load.day_types_unreadable") + + async def _save_cache(self, today: date) -> None: + if self._learned is None: + return + oldest = today - timedelta(days=self._history_days + 7) + self._cache = {d: v for d, v in self._cache.items() if d >= oldest} + payload = { + "fingerprint": self._fingerprint, + "days": { + d.isoformat(): [v.day_type.value, v.source.value] + for d, v in sorted(self._cache.items()) + }, + } + try: + await self._learned.put( + NS_LOAD_PROFILE, DAY_TYPES_KEY, payload, schema_version=DAY_TYPES_SCHEMA + ) + except Exception: + log.exception("forecast.load.day_types_store_failed") + + async def _resolve(self, wanted: Sequence[date], today: date) -> None: + """Ask ``workday.check_date`` for dates without a cached answer (bounded per run).""" + entity = self._inputs.workday_entity + if entity is None or self._reader is None: + return + state = self._reader.workday_state(entity) + if state is None: + return # without the state the offset is unknown: no answer can be placed + changed = False + if state.fingerprint != self._fingerprint: # another sensor or configuration + if self._cache: + log.info("forecast.load.day_types_reset") + self._cache.clear() + self._fingerprint = state.fingerprint + changed = True + observed = today + timedelta(days=state.offset) + info = DayInfo(day_type_from_workday(observed, state.workday), DayTypeSource.WORKDAY_SENSOR) + if self._cache.get(observed) != info: + self._cache[observed] = info + changed = True + asked = 0 + for day in wanted: + if day in self._cache or asked >= MAX_CHECKS_PER_RUN: + continue + asked += 1 + try: + # Home Assistant answers for the date plus the sensor's offset: ask d - offset + workday = await self._reader.check_workday( + entity, day - timedelta(days=state.offset), min_interval_s=CHECK_DATE_INTERVAL_S + ) + except asyncio.CancelledError: + raise + except Exception as exc: + log.info("forecast.load.check_date_failed", day=day.isoformat(), error=repr(exc)) + break # rate limited or unsupported: the weekday rule until the next run + self._cache[day] = DayInfo( + day_type_from_workday(day, workday), DayTypeSource.CHECK_DATE + ) + changed = True + if changed: + await self._save_cache(today) + + # --- history ---------------------------------------------------------------------- + + async def base_load_series(self, window: TimeRange) -> TimeSeries: + """The measured base load (W, 15 minutes) in ``window`` (house minus devices).""" + result, _ = await self._read_base(window) + return result.series + + async def _read_base(self, window: TimeRange) -> tuple[BaseLoad, dict[str, TimeSeries]]: + assert self._data is not None + house = await self._data.read_series(SITE_HOUSE_W, window, "15m", SeriesUnit.W) + devices = { + device_id: await self._data.read_series( + device_power_key(device_id), window, "15m", SeriesUnit.W + ) + for device_id in self._inputs.device_ids + } + return base_load(house, devices), devices + + async def refit(self, now: datetime | None = None) -> LoadModel | None: + """Read the history and fit the model (kept on failure).""" + if self._data is None: + return None + moment = self._clock() if now is None else now + tz = self._tz() + await self._load_cache() + today = moment.astimezone(tz).date() + end = floor_to_step(moment, PLANNING_STEP) + start = local_midnight(moment, tz).astimezone(UTC) - timedelta(days=self._history_days) + window = TimeRange(start=start, end=end) + history_days = [today - timedelta(days=i) for i in range(1, self._history_days + 1)] + await self._resolve([*self._forecast_days(moment, tz), *history_days], today) + try: + base, devices = await self._read_base(window) + temperature = None + if self._inputs.weather_entity is not None: + temperature = await self._data.read_series( + WEATHER_TEMPERATURE_C, window, "15m", SeriesUnit.CELSIUS + ) + model = fit_load_model( + base.series, + now=moment, + tz=tz, + day_types=self.day_types(history_days, today), + temperature=temperature, + temperature_configured=self._inputs.weather_entity is not None, + unmeasured=base.unmeasured, + ) + except asyncio.CancelledError: + raise + except Exception: + log.exception("forecast.load.fit_failed") + return self._model + self._model, self._fitted_at = model, moment + self._base, self._devices = base, devices + if temperature is not None: + midnight = local_midnight(moment, tz) + live = [(t, v) for t, v in self._observed if t >= end] # not aggregated yet + self._observed = [ + (begin, v) + for begin, _, v in temperature.slots() + if v is not None and begin >= midnight + ] + live + log.info( + "forecast.load.fitted", + quality=model.quality.value, + days=model.days, + temperature=model.temperature.reason, + unmeasured=sum(base.unmeasured_slots.values()), + ) + return model + + # --- temperature ------------------------------------------------------------------ + + async def _record_temperature(self, now: datetime) -> None: + entity = self._inputs.weather_entity + if entity is None or self._reader is None or self._data is None: + return + value = self._reader.outside_temperature(entity) + if value is None: + return + self._observed.append((now, value)) + try: + await self._data.append_measurements([(WEATHER_TEMPERATURE_C, now, value)]) + except asyncio.CancelledError: + raise + except Exception: + log.exception("forecast.load.temperature_store_failed") + + async def _update_weather(self, now: datetime) -> None: + entity = self._inputs.weather_entity + if entity is None or self._reader is None: + return + if self._weather_at is not None and now - self._weather_at < self._weather_interval: + return + try: + points = await self._reader.hourly_temperatures( + entity, min_interval_s=self._weather_interval.total_seconds() + ) + except asyncio.CancelledError: + raise + except Exception as exc: + log.info("forecast.load.weather_failed", error=repr(exc)) + return + self._weather_at = now + if points: + self._weather_points = points + + def daily_temperatures(self, now: datetime) -> dict[date, float]: + """Mean temperature per forecast date: today from the recorded hours plus the forecast + hours, later dates from the forecast alone; only dates with at least + ``FORECAST_TEMPERATURE_HOURS`` hours (others get no temperature offset).""" + tz = self._tz() + today = now.astimezone(tz).date() + observed = [(t, v) for t, v in self._observed if t.astimezone(tz).date() == today] + upcoming = [(t, v) for t, v in self._weather_points if t >= floor_to_step(now)] + return daily_mean_temperatures( + [*observed, *upcoming], tz, min_hours=FORECAST_TEMPERATURE_HOURS + ) + + # --- refresh ---------------------------------------------------------------------- + + def request_refit(self) -> None: + """Refit at the next refresh (e.g. after the history was rebuilt).""" + self._fitted_at = None + + def _forecast_days(self, now: datetime, tz: tzinfo) -> list[date]: + first = now.astimezone(tz).date() + last = (now + self._horizon).astimezone(tz).date() + return [first + timedelta(days=i) for i in range((last - first).days + 1)] + + def forecast(self, now: datetime | None = None) -> QuantileSeries | None: + """The base load forecast from the current slot for ``horizon`` (``None``: no model).""" + model = self._model + if model is None: + return None + moment = self._clock() if now is None else now + tz = self._tz() + grid = planning_grid(moment, self._horizon) + today = moment.astimezone(tz).date() + types = self.day_types(self._forecast_days(moment, tz), today) + temps = self.daily_temperatures(moment) if model.temperature.active else {} + return forecast_load(model, grid, tz, types, temps) + + async def refresh(self) -> QuantileSeries | None: + """One refresh (record, refit if due, weather, forecast, store); never raises.""" + async with self._lock: + now = self._clock() + try: + await self._record_temperature(now) + tz = self._tz() + if ( + self._fitted_at is None + or self._model is None + or self._model.quality is not LoadQuality.OK # little data: refit cheaply + or now - self._fitted_at >= self._refit_interval + or self._fitted_at.astimezone(tz).date() != now.astimezone(tz).date() + ): + await self.refit(now) + else: + await self._resolve(self._forecast_days(now, tz), now.astimezone(tz).date()) + await self._update_weather(now) + forecast = self.forecast(now) + if forecast is not None: + await self._maybe_store(forecast, now) + except asyncio.CancelledError: + raise + except Exception: + log.exception("forecast.load.refresh_failed") + return None + return forecast + + async def _maybe_store(self, forecast: QuantileSeries, now: datetime) -> None: + if self._data is None: + return + if self._last_stored_at is not None and now - self._last_stored_at < self._store_interval: + return + try: + await self._data.store_forecast(LOAD_SOURCE_ID, "load", now, forecast) + except asyncio.CancelledError: + raise + except Exception: + log.exception("forecast.load.store_failed") + return + self._last_stored_at = now + + async def run(self) -> None: + """Refresh now, then every ``refresh_interval_s``.""" + while True: + await self.refresh() + await self._sleep(self._refresh_interval_s) + + # --- nightly ------------------------------------------------------------------------ + + async def nightly(self, now: datetime | None = None) -> list[AccuracyRecord]: + """Accuracy of ``load_profile`` (stored in ``forecast_accuracy``) and a refit.""" + moment = self._clock() if now is None else now + records: list[AccuracyRecord] = [] + if self._data is not None and self._learned is not None: + results = await update_accuracy( + self._data, + self._learned, + [LOAD_SOURCE_ID], + moment, + kind="load", + actual_reader=self.base_load_series, + ) + records = results.get(LOAD_SOURCE_ID, []) + async with self._lock: + await self.refit(moment) + return records + + # --- view ---------------------------------------------------------------------------- + + def view(self, hours: int = 48) -> LoadForecastView: + now = self._clock() + hours = min(max(hours, 1), MAX_VIEW_HOURS) + tz = self._tz() + model = self._model + forecast = self.forecast(now) + count = hours * 4 + p10: list[float | None] = [] + p50: list[float | None] = [] + p90: list[float | None] = [] + start = None + if forecast is not None: + start = forecast.start + p10 = list(forecast.p10.values[:count]) + p50 = list(forecast.p50.values[:count]) + p90 = list(forecast.p90.values[:count]) + today = now.astimezone(tz).date() + temps = self.daily_temperatures(now) + last = (now + timedelta(hours=hours)).astimezone(tz).date() + days = [ + LoadDayView( + date=d, + day_type=(info := self.day_info(d, today)).day_type, + source=info.source, + temperature_c=None if d not in temps else round(temps[d], 2), + ) + for d in self._forecast_days(now, tz) + if d <= last + ] + term = model.temperature if model is not None else None + base = self._base + return LoadForecastView( + generated_at=now, + hours=hours, + source_id=LOAD_SOURCE_ID, + quality=LoadQuality.NO_HISTORY if model is None else model.quality, + days_of_history=0 if model is None else model.days, + fitted_at=self._fitted_at, + start=start, + step_s=int(PLANNING_STEP.total_seconds()), + p10_w=p10, + p50_w=p50, + p90_w=p90, + days=days, + temperature=LoadTemperatureView( + active=term is not None and term.active, + reason=( + "not_configured" + if self._inputs.weather_entity is None + else ( + term.reason + if term is not None and term.reason != "not_configured" + else "too_few_days" + ) + ), + days=0 if term is None else term.days, + heating_w_per_k=0.0 if term is None else round(term.heating_w_per_k, 3), + cooling_w_per_k=0.0 if term is None else round(term.cooling_w_per_k, 3), + ), + controlled_devices=list(self._inputs.device_ids), + devices_unmeasured={} if base is None else dict(base.unmeasured_slots), + clamped_slots=0 if base is None else base.clamped_slots, + unmeasured_in_profile=( + [] if model is None else sorted(model.unmeasured_kept, key=lambda t: t.value) + ), + ) + + +def _decode_days(payload: Mapping[str, Any]) -> dict[date, DayInfo]: + out: dict[date, DayInfo] = {} + raw = payload.get("days") + if not isinstance(raw, Mapping): + return out + for key, value in raw.items(): + try: + day = date.fromisoformat(str(key)) + kind, source = value + out[day] = DayInfo(DayType(kind), DayTypeSource(source)) + except (ValueError, TypeError): + log.warning("forecast.load.day_type_unreadable", day=str(key)) + return out + + +__all__ = [ + "CHECK_DATE_INTERVAL_S", + "DAY_TYPES_KEY", + "HORIZON", + "MAX_CHECKS_PER_RUN", + "NS_LOAD_PROFILE", + "REFRESH_INTERVAL_S", + "LoadData", + "LoadForecastService", + "LoadHaReader", + "LoadInputs", +] diff --git a/suncore/forecast/pv/learning.py b/suncore/forecast/pv/learning.py index 75f98d8..dae553a 100644 --- a/suncore/forecast/pv/learning.py +++ b/suncore/forecast/pv/learning.py @@ -20,7 +20,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass -from datetime import UTC, date, datetime, time, timedelta, tzinfo +from datetime import date, datetime, timedelta, tzinfo from typing import Final, Literal, Protocol import structlog @@ -43,6 +43,7 @@ LearnedParams, update_accuracy, ) +from suncore.forecast.days import LocalDay, local_days, local_midnight, slice_series from suncore.forecast.pv.calibration import ( BiasTable, learn_day, @@ -81,55 +82,13 @@ class LearningContext: export_limit_w: float | None -@dataclass(frozen=True, slots=True) -class LocalDay: - day: date - start: datetime - """Local midnight, in UTC.""" - end: datetime - - @dataclass(frozen=True, slots=True) class NightlyResult: accuracy: Mapping[str, Sequence[AccuracyRecord]] bias: Mapping[str, BiasTable] clipped_slots: int - - -def local_midnight(moment: datetime, tz: tzinfo) -> datetime: - local = moment.astimezone(tz) - return datetime.combine(local.date(), time(), tzinfo=tz) - - -def local_days(start: datetime, end: datetime, tz: tzinfo) -> list[LocalDay]: - """The complete local days inside ``[start, end)`` (DST days have 23 or 25 hours).""" - first = start.astimezone(tz).date() - if datetime.combine(first, time(), tzinfo=tz) < start: - first += timedelta(days=1) - days: list[LocalDay] = [] - current = first - while True: - # in UTC: Python subtracts two datetimes of the same zone on the wall clock (24 h even - # on a DST day), so the bounds are kept in absolute time - begin = datetime.combine(current, time(), tzinfo=tz).astimezone(UTC) - finish = datetime.combine(current + timedelta(days=1), time(), tzinfo=tz).astimezone(UTC) - if finish > end: - return days - days.append(LocalDay(current, begin, finish)) - current += timedelta(days=1) - - -def slice_series(series: TimeSeries, begin: datetime, end: datetime) -> TimeSeries: - """The grid-aligned part of ``series`` in ``[begin, end)`` (clamped to its extent).""" - step = series.step - first = max(0, (begin - series.start) // step) - last = max(first, min(len(series), -(-(end - series.start) // step))) - return TimeSeries( - start=series.start + step * first, - step=step, - values=series.values[first:last], - unit=series.unit, - ) + clipped: frozenset[datetime] = frozenset() + """The flagged quarter hours (slot starts), for the risk model's learning.""" async def day_start_forecast( @@ -258,7 +217,9 @@ async def run_nightly( days=len(days), clipped_slots=len(clipped), ) - return NightlyResult(accuracy=accuracy, bias=learnt, clipped_slots=len(clipped)) + return NightlyResult( + accuracy=accuracy, bias=learnt, clipped_slots=len(clipped), clipped=clipped + ) __all__ = [ diff --git a/suncore/forecast/risk.py b/suncore/forecast/risk.py new file mode 100644 index 0000000..5ab134f --- /dev/null +++ b/suncore/forecast/risk.py @@ -0,0 +1,665 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Risk model of the net energy forecast; morning buffer and evening target (Phase 4 WP7). + +**Net energy** of a window = PV energy - base load energy (Wh), both forecast and measured; the +**net error** is ``actual net - forecast net`` (the one sign convention of SunCore's forecast +statistics, :class:`~suncore.core.forecast.AccuracyRecord`). Taking the quantile of the *net* +error, instead of P10 PV and P90 load separately, avoids counting caution twice (plan §0). + +**Windows** (:class:`RiskWindow`), anchored on today's sunset ("the evening"): + +* ``rest_of_night``: now -> today's sunrise (only before sunrise, else empty); +* ``rest_of_today``: max(now, today's sunrise) -> the evening (empty after sunset); +* ``night``: the evening -> the next sunrise (after sunset: now -> the next sunrise); +* ``next_day``: that sunrise -> the following sunset; +* ``bridge``: the evening -> the following sunset (night + next day, one quantile, used for + next-day bridging so the two parts are not each taken at the risk quantile); +* ``day_and_night``: the coming daytime and the night after it, as one window for the morning + buffer - before the evening ``now -> the next sunrise``, after it ``next_day`` plus the + following night. + +Sunrise and sunset come from the PV forecast itself: the first and the last quarter hour of a +local day whose P50 exceeds ``max(DAYLIGHT_MIN_W, DAYLIGHT_SHARE * the day's peak)``; a day +without PV in the forecast uses ``FALLBACK_SUNRISE``/``FALLBACK_SUNSET`` local time. + +**Weather class** of a daytime window: the class (:class:`~suncore.forecast.pv.clearsky +.WeatherClass`, the PV calibration's clear-sky index of the forecast) that carries the largest +share of the window's forecast PV energy; the nights are their own class ``night``; ``bridge`` +and ``day_and_night`` take the class of their day part. + +**Distribution** (:func:`net_energy_quantile`) for a quantile ``q`` (the risk level's: cautious +0.1, balanced 0.25, bold 0.5): + +``learned`` + From ``min_history_days`` (30) days with samples on: the empirical quantile of the stored + *relative* errors ``error / scale`` of the same window kind and weather class (at least + ``MIN_CLASS_SAMPLES``; else the window kind over all classes, at least + ``MIN_WINDOW_SAMPLES``), times the window's scale ``max(1 Wh, PV P50 + load P50)``, added + to the forecast net P50. Relative errors transfer between a long and a short + ``rest_of_today`` and between summer and winter days. +``source_quantiles`` + Until then (or for a thin cell): the sources' own quantiles, combined as a two-piece normal + under **independence of PV and load**. Over the window the slot quantiles are summed + (comonotone within each quantity - PV errors of one day move together, the conservative + choice) giving ``PV10, PV50, PV90`` and ``L10, L50, L90``; with ``z = Phi^-1(q)`` and + ``z90 = Phi^-1(0.9) = 1.2816``:: + + q < 0.5: net_q = (PV50 - L50) + z * sqrt(((PV50 - PV10) / z90)^2 + ((L90 - L50) / z90)^2) + q > 0.5: net_q = (PV50 - L50) + z * sqrt(((PV90 - PV50) / z90)^2 + ((L50 - L10) / z90)^2) + + and ``net_0.5 = PV50 - L50``. The lower side combines the PV shortfall with the load + excess, the upper side the reverse. +``default_band`` + The same, but for a window whose PV forecast has **no spread** (a source without + quantiles: P10 = P50 = P90) the PV band is ``DEFAULT_PV_BAND`` of the window's weather class + (relative P10/P90 factors, ``UNVERIFIED`` reasoned constants) - otherwise every risk level + would plan on the median PV until the learnt distribution takes over. + +**Planner-facing targets** (all return a value and a :class:`~suncore.core.decisions.Reason`): + +:func:`evening_target_pct` + ``floor + 100 * need / capacity`` with ``floor`` = + :func:`~suncore.core.battery.effective_reserve_pct` (the site battery's minimum SoC or + reserve + BMS drift margin, whichever is higher) and + ``need = max(-net_q(night), -net_q(bridge) if bridge_next_day) / discharge_efficiency`` + (Wh the battery must deliver; never negative). + Never below the user's target SoC (``goals.battery_target``; its time is not used here, the + timed constraint is the planner's, 4-F), never above ``max_soc_pct``; ``prefer_full`` -> + ``max_soc_pct``. +:func:`morning_buffer_pct` + "Battery first until X %" from **one** quantile: ``M = net_q(day_and_night)`` is the energy + left after the coming daytime *and* the night after it are covered, at the risk quantile; + ``N = max(1 Wh, -net_0.5(night))`` the median night need (a scale, no second quantile). + ``share = clamp(1 - M / N, 0, 1)`` and ``X = floor + share * (target - floor)``: a margin of + at least one more median night lets consumers go first (X = floor), no margin puts the + battery first up to the evening target (X = target, an upper bound only). Never below the + user's manual floor (``BatterySettings.morning_buffer_pct``), capped at ``max_soc_pct``; + ``prefer_full`` -> ``max_soc_pct``. + +Both are monotone: a more cautious risk level or a wider distribution never lowers them (a lower +``q`` lowers every net quantile, which raises the need and lowers ``M``). The default band is a +step: a source that gains a tiny spread falls back to its own (possibly narrower) band. +""" + +from __future__ import annotations + +import math +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, date, datetime, time, timedelta, tzinfo +from enum import StrEnum, unique +from statistics import NormalDist +from typing import Any, Final, Literal + +from suncore.core.battery import ( + BatteryModel, + BatterySettings, + BatteryStrategy, + RiskLevel, + effective_reserve_pct, +) +from suncore.core.decisions import Reason +from suncore.core.series import PLANNING_STEP, QuantileSeries, TimeSeries, floor_to_step +from suncore.core.site import BatteryConfig +from suncore.forecast.pv.clearsky import PlantGeometry, WeatherClass, classify_slots +from suncore.forecast.stats import weighted_quantile + +DAYLIGHT_MIN_W: Final = 20.0 +DAYLIGHT_SHARE: Final = 0.02 +FALLBACK_SUNRISE: Final = time(7, 0) +FALLBACK_SUNSET: Final = time(19, 0) +MIN_CLASS_SAMPLES: Final = 10 +MIN_WINDOW_SAMPLES: Final = 20 +Z90: Final = NormalDist().inv_cdf(0.9) +NIGHT_CLASS: Final = "night" + +REASON_MORNING_BUFFER: Final = "planner.battery.morning_buffer" +REASON_EVENING_TARGET: Final = "planner.battery.evening_target" + +RiskClass = Literal["overcast", "mixed", "clear", "night"] +DistributionSource = Literal["learned", "source_quantiles", "default_band"] +BufferCause = Literal[ + "prefer_full", "forecast_poor", "forecast_uncertain", "forecast_good", "manual_floor" +] +TargetCause = Literal["prefer_full", "night_load", "next_day_deficit", "user_target", "max_soc"] + + +_RISK_CLASS: Final[dict[WeatherClass, RiskClass]] = { + WeatherClass.OVERCAST: "overcast", + WeatherClass.MIXED: "mixed", + WeatherClass.CLEAR: "clear", +} +DEFAULT_PV_BAND: Final[dict[RiskClass, tuple[float, float]]] = { + "clear": (0.85, 1.10), + "mixed": (0.55, 1.35), + "overcast": (0.35, 1.70), + "night": (1.0, 1.0), +} +"""``(P10, P90)`` as factors of the PV P50 for a window without a source band (``UNVERIFIED``: +reasoned, not fitted - replaced by the learnt distribution after ``min_history_days``).""" +_CLASSES: Final[dict[str, RiskClass]] = { + "overcast": "overcast", + "mixed": "mixed", + "clear": "clear", + "night": "night", +} + + +@unique +class RiskWindow(StrEnum): + REST_OF_NIGHT = "rest_of_night" + REST_OF_TODAY = "rest_of_today" + NIGHT = "night" + NEXT_DAY = "next_day" + BRIDGE = "bridge" + DAY_AND_NIGHT = "day_and_night" + + +@dataclass(frozen=True, slots=True) +class EnergyQuantiles: + """P10/P50/P90 of an energy (Wh) over a window (slot quantiles summed).""" + + p10: float + p50: float + p90: float + + @classmethod + def zero(cls) -> EnergyQuantiles: + return cls(0.0, 0.0, 0.0) + + +@dataclass(frozen=True, slots=True) +class NetWindow: + """One window with the forecast PV and load energy and its weather class.""" + + kind: RiskWindow + start: datetime + end: datetime + pv: EnergyQuantiles + load: EnergyQuantiles + weather: RiskClass + + @property + def net_p50(self) -> float: + return self.pv.p50 - self.load.p50 + + @property + def scale_wh(self) -> float: + """Normalisation of the learnt relative errors (at least 1 Wh).""" + return max(1.0, self.pv.p50 + self.load.p50) + + +# --- windows ------------------------------------------------------------------------------- + + +def window_energy(series: QuantileSeries | None, start: datetime, end: datetime) -> EnergyQuantiles: + """Energy (Wh) of ``series`` (W) in ``[start, end)``, partial slots by their overlap share. + + Unknown slots count as 0 Wh (the caller judges coverage); ``None`` series = zero energy. + """ + if series is None or end <= start: + return EnergyQuantiles.zero() + totals = [0.0, 0.0, 0.0] + hours = series.step / timedelta(hours=1) + for i, (begin, stop, _) in enumerate(series.p50.slots()): + overlap = (min(stop, end) - max(begin, start)) / series.step + if overlap <= 0: + continue + for k, part in enumerate((series.p10, series.p50, series.p90)): + value = part.values[i] + if value is not None: + totals[k] += value * hours * overlap + lo, mid, hi = totals + return EnergyQuantiles(min(lo, mid), mid, max(hi, mid)) + + +def daylight_spans(pv: QuantileSeries | None, tz: tzinfo) -> dict[date, tuple[datetime, datetime]]: + """``(sunrise, sunset)`` per local date of the PV forecast's daylight (module doc).""" + if pv is None: + return {} + peaks: dict[date, float] = {} + for begin, _, value in pv.p50.slots(): + if value is not None: + day = begin.astimezone(tz).date() + peaks[day] = max(peaks.get(day, 0.0), value) + spans: dict[date, tuple[datetime, datetime]] = {} + for begin, stop, value in pv.p50.slots(): + if value is None: + continue + day = begin.astimezone(tz).date() + if value <= max(DAYLIGHT_MIN_W, DAYLIGHT_SHARE * peaks[day]): + continue + rise, sets = spans.get(day, (begin, stop)) + spans[day] = (min(rise, begin), max(sets, stop)) + return spans + + +def sun_times( + day: date, spans: Mapping[date, tuple[datetime, datetime]], tz: tzinfo +) -> tuple[datetime, datetime]: + if day in spans: + return spans[day] + rise = datetime.combine(day, FALLBACK_SUNRISE, tzinfo=tz).astimezone(UTC) + sets = datetime.combine(day, FALLBACK_SUNSET, tzinfo=tz).astimezone(UTC) + return rise, sets + + +def window_class( + pv: QuantileSeries | None, + start: datetime, + end: datetime, + *, + plant: PlantGeometry | None, + tz: tzinfo, +) -> RiskClass: + """The weather class with the most forecast PV energy in ``[start, end)``; ``overcast`` + without PV in the window.""" + if pv is None: + return "overcast" + classes = classify_slots(pv.p50, plant=plant, tz=tz) + energy: dict[WeatherClass, float] = {} + for (begin, stop, value), cls in zip(pv.p50.slots(), classes, strict=True): + if cls is None or value is None or stop <= start or begin >= end: + continue + energy[cls] = energy.get(cls, 0.0) + value + if not energy: + return "overcast" + best = max(energy.items(), key=lambda item: item[1])[0] + return _RISK_CLASS[best] + + +def day_windows( + now: datetime, + pv: QuantileSeries | None, + load: QuantileSeries | None, + tz: tzinfo, + *, + plant: PlantGeometry | None = None, +) -> dict[RiskWindow, NetWindow]: + """The windows anchored on today's sunset (module doc).""" + spans = daylight_spans(pv, tz) + today = now.astimezone(tz).date() + sunrise, evening = sun_times(today, spans, tz) + # After today's sunset the evening has passed: ``rest_of_today`` is empty and the night + # window starts now (every window is clipped to the current slot below). + next_rise, next_set = sun_times(today + timedelta(days=1), spans, tz) + third_rise = sun_times(today + timedelta(days=2), spans, tz)[0] + begin = floor_to_step(now, PLANNING_STEP) + out: dict[RiskWindow, NetWindow] = {} + + def add(kind: RiskWindow, start: datetime, end: datetime, weather: RiskClass) -> None: + start = max(start, begin) + end = max(end, start) + out[kind] = NetWindow( + kind=kind, + start=start, + end=end, + pv=window_energy(pv, start, end), + load=window_energy(load, start, end), + weather=weather, + ) + + def daytime(start: datetime, end: datetime) -> RiskClass: + return window_class(pv, start, end, plant=plant, tz=tz) + + add(RiskWindow.REST_OF_NIGHT, begin, sunrise, NIGHT_CLASS) + day_start = max(begin, sunrise) + today_class = daytime(day_start, evening) + add(RiskWindow.REST_OF_TODAY, day_start, evening, today_class) + add(RiskWindow.NIGHT, evening, next_rise, NIGHT_CLASS) + next_class = daytime(next_rise, next_set) + add(RiskWindow.NEXT_DAY, next_rise, next_set, next_class) + add(RiskWindow.BRIDGE, evening, next_set, next_class) + if begin < evening: + add(RiskWindow.DAY_AND_NIGHT, begin, next_rise, today_class) + else: + add(RiskWindow.DAY_AND_NIGHT, next_rise, third_rise, next_class) + return out + + +# --- the error distribution -------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class ErrorSample: + """One measured net energy error of a past window (``error = actual - forecast``).""" + + day: date + """The anchor day (the day of the evening; the ``rest_of_today`` day itself).""" + window: RiskWindow + weather: RiskClass + error_wh: float + scale_wh: float + + @property + def relative(self) -> float: + return self.error_wh / max(1.0, self.scale_wh) + + def to_json(self) -> dict[str, Any]: + return { + "day": self.day.isoformat(), + "window": self.window.value, + "weather": self.weather, + "error_wh": round(self.error_wh, 3), + "scale_wh": round(self.scale_wh, 3), + } + + @classmethod + def from_json(cls, raw: Mapping[str, Any]) -> ErrorSample: + weather = _CLASSES.get(str(raw["weather"])) + if weather is None: + msg = f"unknown weather class {raw['weather']!r}" + raise ValueError(msg) + return cls( + day=date.fromisoformat(str(raw["day"])), + window=RiskWindow(str(raw["window"])), + weather=weather, + error_wh=float(raw["error_wh"]), + scale_wh=float(raw["scale_wh"]), + ) + + +@dataclass(frozen=True, slots=True) +class ErrorModel: + """The learnt samples (all window kinds) and how many distinct days they cover.""" + + samples: tuple[ErrorSample, ...] = () + + @property + def days(self) -> int: + return len({s.day for s in self.samples}) + + def cell(self, window: RiskWindow, weather: RiskClass) -> list[float]: + return [s.relative for s in self.samples if s.window is window and s.weather == weather] + + def pooled(self, window: RiskWindow) -> list[float]: + return [s.relative for s in self.samples if s.window is window] + + +@dataclass(frozen=True, slots=True) +class NetQuantile: + """``value_wh`` = the net energy of a window at quantile ``q``.""" + + value_wh: float + q: float + source: DistributionSource + samples: int + """Learnt samples used (0 for ``source_quantiles``).""" + pooled: bool = False + """``learned`` from the window kind over all weather classes (the class cell was thin).""" + + +def has_pv_band(window: NetWindow) -> bool: + """Whether the window's PV forecast carries a spread (or has no PV at all).""" + pv = window.pv + tiny = 1e-6 * max(1.0, pv.p50) + return pv.p50 <= tiny or pv.p50 - pv.p10 > tiny or pv.p90 - pv.p50 > tiny + + +def with_default_band(window: NetWindow) -> NetWindow: + """The window with ``DEFAULT_PV_BAND`` of its weather class as PV band.""" + low, high = DEFAULT_PV_BAND[window.weather] + p50 = window.pv.p50 + band = EnergyQuantiles(p50 * low, p50, p50 * high) + return NetWindow(window.kind, window.start, window.end, band, window.load, window.weather) + + +def source_net_quantile(window: NetWindow, q: float) -> float: + """The two-piece normal of the module doc (``source_quantiles``).""" + if not 0 < q < 1: + msg = "q must be inside (0, 1)" + raise ValueError(msg) + median = window.net_p50 + if math.isclose(q, 0.5): + return median + z = NormalDist().inv_cdf(q) + pv, load = window.pv, window.load + if q < 0.5: + sigma = math.hypot((pv.p50 - pv.p10) / Z90, (load.p90 - load.p50) / Z90) + else: + sigma = math.hypot((pv.p90 - pv.p50) / Z90, (load.p50 - load.p10) / Z90) + return median + z * sigma + + +def net_energy_quantile( + window: NetWindow, + risk: RiskLevel | float, + model: ErrorModel | None = None, + *, + min_history_days: int = 30, +) -> NetQuantile: + """The net energy (Wh) of ``window`` that is exceeded with the risk level's probability.""" + q = risk.quantile if isinstance(risk, RiskLevel) else float(risk) + if model is not None and model.days >= min_history_days: + values = model.cell(window.kind, window.weather) + pooled = False + if len(values) < MIN_CLASS_SAMPLES: + values, pooled = model.pooled(window.kind), True + if len(values) >= (MIN_WINDOW_SAMPLES if pooled else MIN_CLASS_SAMPLES): + rel = weighted_quantile(values, [1.0] * len(values), q) + return NetQuantile( + value_wh=window.net_p50 + rel * window.scale_wh, + q=q, + source="learned", + samples=len(values), + pooled=pooled, + ) + if not has_pv_band(window): + return NetQuantile( + value_wh=source_net_quantile(with_default_band(window), q), + q=q, + source="default_band", + samples=0, + ) + return NetQuantile( + value_wh=source_net_quantile(window, q), q=q, source="source_quantiles", samples=0 + ) + + +# --- targets ---------------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class SocTarget: + """A computed SoC (%) and why.""" + + soc_pct: float + reason: Reason + source: DistributionSource + + +def _round(value: float) -> float: + return round(value, 1) + + +def evening_target_pct( + battery: BatteryModel, + settings: BatterySettings, + windows: Mapping[RiskWindow, NetWindow], + *, + user_target_pct: float | None = None, + site_battery: BatteryConfig | None = None, + model: ErrorModel | None = None, + min_history_days: int = 30, +) -> SocTarget: + """The SoC (%) the battery should hold at the evening (module doc).""" + cap = battery.max_soc_pct + floor = min(cap, effective_reserve_pct(site_battery, battery)) + risk = settings.risk + night = net_energy_quantile( + windows[RiskWindow.NIGHT], risk, model, min_history_days=min_history_days + ) + night_need = max(0.0, -night.value_wh) / battery.discharge_efficiency + bridge_need = 0.0 + source = night.source + if settings.bridge_next_day and RiskWindow.BRIDGE in windows: + bridge = net_energy_quantile( + windows[RiskWindow.BRIDGE], risk, model, min_history_days=min_history_days + ) + bridge_need = max(0.0, -bridge.value_wh) / battery.discharge_efficiency + if bridge.source != "learned": + source = bridge.source + computed = floor + battery.wh_to_soc(max(night_need, bridge_need)) + user = 0.0 if user_target_pct is None else user_target_pct + cause: TargetCause + if settings.strategy is BatteryStrategy.PREFER_FULL: + value, cause = cap, "prefer_full" + elif max(computed, user) > cap: + value, cause = cap, "max_soc" + elif user >= computed: + value, cause = max(user, floor), "user_target" + else: + value = computed + cause = "next_day_deficit" if bridge_need > night_need else "night_load" + value = min(cap, max(floor, value)) + reason = Reason( + key="planner.battery.evening_target", + args={ + "soc_pct": _round(value), + "cause": cause, + "night_kwh": _round(night_need / 1000), + "deficit_kwh": _round(max(0.0, bridge_need - night_need) / 1000), + "risk": risk.value, + "source": source, + }, + ) + return SocTarget(soc_pct=value, reason=reason, source=source) + + +def morning_buffer_pct( + battery: BatteryModel, + settings: BatterySettings, + windows: Mapping[RiskWindow, NetWindow], + evening_target: float, + *, + site_battery: BatteryConfig | None = None, + model: ErrorModel | None = None, + min_history_days: int = 30, +) -> SocTarget: + """ "Battery first until X %" for the coming daytime (module doc).""" + cap = battery.max_soc_pct + manual = settings.morning_buffer_pct + auto_floor = min(cap, effective_reserve_pct(site_battery, battery)) + floor = min(cap, max(auto_floor, manual or 0.0)) + margin = net_energy_quantile( + windows[RiskWindow.DAY_AND_NIGHT], + settings.risk, + model, + min_history_days=min_history_days, + ) + night_need = max(1.0, -windows[RiskWindow.NIGHT].net_p50) + cause: BufferCause + if settings.strategy is BatteryStrategy.PREFER_FULL: + value, cause = cap, "prefer_full" + else: + target = min(cap, max(floor, evening_target)) + share = min(1.0, max(0.0, 1.0 - margin.value_wh / night_need)) + value = floor + share * (target - floor) + if share >= 1.0: + cause = "forecast_poor" + elif share > 0.0: + cause = "forecast_uncertain" + elif manual is not None and manual > auto_floor: + cause = "manual_floor" + else: + cause = "forecast_good" + value = min(cap, max(floor, value)) + reason = Reason( + key="planner.battery.morning_buffer", + args={ + "soc_pct": _round(value), + "cause": cause, + "surplus_kwh": _round(margin.value_wh / 1000), + "risk": settings.risk.value, + "source": margin.source, + }, + ) + return SocTarget(soc_pct=value, reason=reason, source=margin.source) + + +# --- learning samples ------------------------------------------------------------------------- + + +def _window_slots(series: TimeSeries, start: datetime, end: datetime) -> tuple[int, int, float]: + """``(slots of the window, known slots, energy Wh of the known ones)`` on the series' grid; + a window slot the series does not reach counts as unknown.""" + total = max(0, -(-(end - start) // series.step)) + hours = series.step / timedelta(hours=1) + known = 0 + energy = 0.0 + for begin, _, value in series.slots(): + if start <= begin < end and value is not None: + known += 1 + energy += value * hours + return total, known, energy + + +def coverage(series: TimeSeries, start: datetime, end: datetime) -> float: + """Share of the window ``[start, end)`` whose slots ``series`` knows (a slot outside the + series counts as unknown).""" + total, known, _ = _window_slots(series, start, end) + return known / total if total else 0.0 + + +def measured_energy(series: TimeSeries, start: datetime, end: datetime) -> float: + """Energy (Wh) of the known slots of a W series in ``[start, end)``, scaled up to the whole + window from the known share (the caller requires a high coverage).""" + total, known, energy = _window_slots(series, start, end) + return energy * total / known if known else 0.0 + + +def merge_samples( + old: Iterable[ErrorSample], new: Iterable[ErrorSample], *, keep_days: int, today: date +) -> tuple[ErrorSample, ...]: + """``old`` plus ``new`` (a new sample replaces one of the same day and window), only the + last ``keep_days`` days, ordered by day.""" + merged: dict[tuple[date, RiskWindow], ErrorSample] = {(s.day, s.window): s for s in old} + merged.update({(s.day, s.window): s for s in new}) + oldest = today - timedelta(days=keep_days) + return tuple( + sorted( + (s for s in merged.values() if s.day >= oldest), key=lambda s: (s.day, s.window.value) + ) + ) + + +def samples_view(samples: Sequence[ErrorSample]) -> dict[str, int]: + """Sample count per ``window/weather`` cell (for the API).""" + out: dict[str, int] = {} + for s in samples: + key = f"{s.window.value}/{s.weather}" + out[key] = out.get(key, 0) + 1 + return dict(sorted(out.items())) + + +__all__ = [ + "DEFAULT_PV_BAND", + "MIN_CLASS_SAMPLES", + "MIN_WINDOW_SAMPLES", + "REASON_EVENING_TARGET", + "REASON_MORNING_BUFFER", + "DistributionSource", + "EnergyQuantiles", + "ErrorModel", + "ErrorSample", + "NetQuantile", + "NetWindow", + "RiskClass", + "RiskWindow", + "SocTarget", + "coverage", + "day_windows", + "daylight_spans", + "evening_target_pct", + "has_pv_band", + "measured_energy", + "merge_samples", + "morning_buffer_pct", + "net_energy_quantile", + "samples_view", + "source_net_quantile", + "sun_times", + "window_class", + "window_energy", + "with_default_band", +] diff --git a/suncore/forecast/risk_learning.py b/suncore/forecast/risk_learning.py new file mode 100644 index 0000000..11c4faa --- /dev/null +++ b/suncore/forecast/risk_learning.py @@ -0,0 +1,425 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Nightly learning of the net energy error and the risk read model (Phase 4 WP7). + +**Samples.** For every complete anchor day ``D`` (at most ``BACKFILL_DAYS`` back - the stored +forecasts are kept 45 days - and only once: ``learned_until``), the forecasts as they stood at +two anchors relative to the sun of ``D`` (sunrise and sunset from the PV forecast as it stood at +the start of ``D``, the fallback times without one) are compared with what was measured: + +* at sunrise + ``DAY_ANCHOR_OFFSET`` (1 h): ``rest_of_today`` (anchor -> sunset of ``D``) and + ``day_and_night`` (anchor -> sunrise ``D+1``), the situation of the morning buffer; +* at sunset - ``EVENING_ANCHOR_OFFSET`` (1 h): ``night`` (sunset ``D`` -> sunrise ``D+1``), + ``next_day`` (daytime of ``D+1``) and ``bridge`` (sunset ``D`` -> sunset ``D+1``), the + situation of the evening target. + +Sun-relative anchors keep the situation alike in winter and summer (a fixed 06:00 would lie +before sunrise in winter and hours after it in summer). + +The forecast of a slot is the newest issue at or before the anchor (``latest_forecast_before``) +of the PV forecast the planner uses (source ``blended``) and of the load forecast +(``load_profile``); windows, sunrise/sunset and the weather class are derived from that issue +exactly as live (:func:`suncore.forecast.risk.day_windows`). The measurement is the PV power +(``site.pv_w``) and the base load (house minus devices, through the load forecast's reader). A +window counts only if the measured PV and base load **and** the PV and load forecasts cover at +least ``MIN_COVERAGE`` of its quarter hours; without a PV forecast only the ``night`` (no PV by +definition) is sampled. A daytime window with a quarter hour that the PV clipping detection +(:mod:`suncore.forecast.pv.clipping`, handed in as ``is_clipped``) flagged is dropped: a limited +inverter would read as a forecast error. ``D`` is +processed once ``D+1`` ended (the night ``bridge`` window reaches the sunset of ``D+1``), i.e. +the nightly job at 03:30 learns the day before yesterday. + +The samples (``error = actual net - forecast net`` and the window's scale) are kept for +``KEEP_DAYS`` days in ``learned_param`` (namespace ``net_error``, key ``samples``). +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, date, datetime, time, timedelta, tzinfo +from typing import Any, Final, Literal, Protocol + +import structlog + +from suncore.core.battery import BatterySettings +from suncore.core.measurements import SITE_PV_W +from suncore.core.series import ( + PLANNING_STEP, + QuantileSeries, + SeriesUnit, + TimeRange, + TimeSeries, + floor_to_step, +) +from suncore.core.site import BatteryConfig +from suncore.forecast.accuracy import LearnedParams +from suncore.forecast.load import LOAD_SOURCE_ID +from suncore.forecast.pv.clearsky import PlantGeometry +from suncore.forecast.risk import ( + DistributionSource, + EnergyQuantiles, + ErrorModel, + ErrorSample, + NetWindow, + RiskWindow, + coverage, + day_windows, + daylight_spans, + evening_target_pct, + measured_energy, + merge_samples, + morning_buffer_pct, + net_energy_quantile, + samples_view, + sun_times, +) +from suncore.forecast.views import ( + EnergyView, + RiskTargetView, + RiskView, + RiskWindowView, +) + +log = structlog.get_logger(__name__) + +NS_NET_ERROR: Final = "net_error" +"""``learned_param`` namespace (equal to ``suncore.storage.learned.NS_NET_ERROR``).""" +SAMPLES_KEY: Final = "samples" +SCHEMA_VERSION: Final = 1 +KEEP_DAYS: Final = 365 +BACKFILL_DAYS: Final = 45 +DAY_ANCHOR_OFFSET: Final = timedelta(hours=1) +EVENING_ANCHOR_OFFSET: Final = timedelta(hours=1) +DAYTIME_WINDOWS: Final = frozenset( + {RiskWindow.REST_OF_TODAY, RiskWindow.NEXT_DAY, RiskWindow.BRIDGE, RiskWindow.DAY_AND_NIGHT} +) +MIN_COVERAGE: Final = 0.9 +PV_SOURCE_ID: Final = "blended" +"""The PV forecast the planner uses (``suncore.forecast.pv.learning.BLENDED_SOURCE_ID``).""" + +BaseLoadReader = Callable[[TimeRange], Awaitable[TimeSeries]] +ClipFilter = Callable[[datetime], bool] + + +class RiskData(Protocol): + """The part of ``TimeseriesStore`` the learning reads.""" + + async def read_series( + self, + key: str, + window: TimeRange, + resolution: Literal["1m", "15m"] = "15m", + unit: SeriesUnit | None = None, + *, + include_recent: bool = True, + ) -> TimeSeries: ... + + async def latest_forecast_before( + self, + source_id: str, + kind: Literal["pv", "load", "price"], + before: datetime, + window: TimeRange | None = None, + ) -> QuantileSeries | None: ... + + +@dataclass(frozen=True, slots=True) +class StoredErrors: + model: ErrorModel + learned_until: date | None + + +async def load_error_model(learned: LearnedParams) -> StoredErrors: + """The stored samples; an unreadable entry counts as none (learning starts again).""" + for entry in await learned.in_namespace(NS_NET_ERROR): + if entry.key != SAMPLES_KEY: + continue + if entry.schema_version != SCHEMA_VERSION: + log.warning("forecast.risk.unknown_schema", version=entry.schema_version) + break + try: + return _decode(entry.payload) + except (KeyError, ValueError, TypeError): + log.warning("forecast.risk.samples_unreadable") + break + return StoredErrors(ErrorModel(), None) + + +def _decode(payload: Mapping[str, Any]) -> StoredErrors: + raw = payload["samples"] + if not isinstance(raw, list): + msg = "samples must be a list" + raise TypeError(msg) + until = payload.get("learned_until") + return StoredErrors( + ErrorModel(tuple(ErrorSample.from_json(s) for s in raw)), + None if until is None else date.fromisoformat(str(until)), + ) + + +async def save_error_model(learned: LearnedParams, stored: StoredErrors) -> None: + payload = { + "learned_until": None if stored.learned_until is None else stored.learned_until.isoformat(), + "samples": [s.to_json() for s in stored.model.samples], + } + await learned.put(NS_NET_ERROR, SAMPLES_KEY, payload, schema_version=SCHEMA_VERSION) + + +def _at(day: date, moment: time, tz: tzinfo) -> datetime: + return datetime.combine(day, moment, tzinfo=tz).astimezone(UTC) + + +def no_clipping(_slot_start: datetime) -> bool: + return False + + +def _forecast_covers(window: NetWindow, pv: QuantileSeries | None, load: QuantileSeries) -> bool: + if coverage(load.p50, window.start, window.end) < MIN_COVERAGE: + return False + if pv is None: + return window.kind is RiskWindow.NIGHT + return coverage(pv.p50, window.start, window.end) >= MIN_COVERAGE + + +def _clipped(window: NetWindow, is_clipped: ClipFilter) -> bool: + if window.kind not in DAYTIME_WINDOWS: + return False + moment = window.start + while moment < window.end: + if is_clipped(moment): + return True + moment += PLANNING_STEP + return False + + +def _sample(day: date, window: NetWindow, pv: TimeSeries, base: TimeSeries) -> ErrorSample | None: + if window.end <= window.start: + return None + if ( + coverage(pv, window.start, window.end) < MIN_COVERAGE + or coverage(base, window.start, window.end) < MIN_COVERAGE + ): + return None + actual = measured_energy(pv, window.start, window.end) - measured_energy( + base, window.start, window.end + ) + return ErrorSample( + day=day, + window=window.kind, + weather=window.weather, + error_wh=actual - window.net_p50, + scale_wh=window.scale_wh, + ) + + +async def _issue( + data: RiskData, anchor: datetime, span: TimeRange +) -> tuple[QuantileSeries | None, QuantileSeries] | None: + """``(pv, load)`` as issued before ``anchor``; ``None`` without a load forecast (there is + no net forecast to judge then). A missing PV forecast counts as 0 W, as live.""" + pv = await data.latest_forecast_before(PV_SOURCE_ID, "pv", anchor, span) + load = await data.latest_forecast_before(LOAD_SOURCE_ID, "load", anchor, span) + if load is None: + return None + return pv, load + + +async def learn_day( + data: RiskData, + day: date, + *, + tz: tzinfo, + plant: PlantGeometry | None, + base_reader: BaseLoadReader, + is_clipped: ClipFilter = no_clipping, +) -> list[ErrorSample]: + """The samples of one anchor day (module doc); empty when data is missing.""" + start = _at(day, time(0), tz) + end = _at(day + timedelta(days=2), time(0), tz) + span = TimeRange(start=start - timedelta(hours=1), end=end) + measured = TimeRange(start=start, end=end) + pv_actual = await data.read_series(SITE_PV_W, measured, "15m", SeriesUnit.W) + base = await base_reader(measured) + first = await _issue(data, start, span) + sunrise, sunset = sun_times(day, daylight_spans(None if first is None else first[0], tz), tz) + out: list[ErrorSample] = [] + for anchor, kinds in ( + (sunrise + DAY_ANCHOR_OFFSET, (RiskWindow.REST_OF_TODAY, RiskWindow.DAY_AND_NIGHT)), + ( + sunset - EVENING_ANCHOR_OFFSET, + (RiskWindow.NIGHT, RiskWindow.NEXT_DAY, RiskWindow.BRIDGE), + ), + ): + moment = floor_to_step(anchor, PLANNING_STEP) + issue = await _issue(data, moment, span) + if issue is None: + continue + pv, load = issue + windows = day_windows(moment, pv, load, tz, plant=plant) + for kind in kinds: + window = windows[kind] + if not _forecast_covers(window, pv, load) or _clipped(window, is_clipped): + continue + sample = _sample(day, window, pv_actual, base) + if sample is not None: + out.append(sample) + return out + + +async def learn_net_errors( + data: RiskData, + learned: LearnedParams, + now: datetime, + *, + tz: tzinfo, + plant: PlantGeometry | None, + base_reader: BaseLoadReader, + is_clipped: ClipFilter = no_clipping, + backfill_days: int = BACKFILL_DAYS, +) -> ErrorModel: + """The nightly job: learn every new complete anchor day and store the samples; a failure + is logged and keeps what was stored.""" + stored = await load_error_model(learned) + today = now.astimezone(tz).date() + last = today - timedelta(days=2) + first = today - timedelta(days=backfill_days) + if stored.learned_until is not None: + first = max(first, stored.learned_until + timedelta(days=1)) + new: list[ErrorSample] = [] + day = first + try: + while day <= last: + new.extend( + await learn_day( + data, day, tz=tz, plant=plant, base_reader=base_reader, is_clipped=is_clipped + ) + ) + day += timedelta(days=1) + except Exception: + log.exception("forecast.risk.learning_failed", day=day.isoformat()) + last = day - timedelta(days=1) # keep what was learnt so far + if last < first: + return stored.model + model = ErrorModel(merge_samples(stored.model.samples, new, keep_days=KEEP_DAYS, today=today)) + await save_error_model(learned, StoredErrors(model, last)) + log.info("forecast.risk.learned", days=(last - first).days + 1, samples=len(new)) + return model + + +# --- read model --------------------------------------------------------------------------------- + + +def _energy(q: EnergyQuantiles) -> EnergyView: + return EnergyView(p10_wh=round(q.p10, 1), p50_wh=round(q.p50, 1), p90_wh=round(q.p90, 1)) + + +def risk_view( + now: datetime, + *, + pv: QuantileSeries | None, + load: QuantileSeries | None, + settings: BatterySettings, + user_target_pct: float | None, + site_battery: BatteryConfig | None = None, + model: ErrorModel, + min_history_days: int, + tz: tzinfo, + plant: PlantGeometry | None, +) -> RiskView: + """The API read model: windows, buffer and target as the planner would get them now.""" + base: dict[str, Any] = { + "generated_at": now, + "risk": settings.risk, + "quantile": settings.risk.quantile, + "strategy": settings.strategy, + "bridge_next_day": settings.bridge_next_day, + "pv_forecast": pv is not None, + "history_days": model.days, + "min_history_days": min_history_days, + "samples": samples_view(model.samples), + } + if load is None: + return RiskView( + status="no_forecast", + distribution_source="source_quantiles", + evening_at=None, + windows=[], + morning_buffer=None, + evening_target=None, + **base, + ) + windows = day_windows(now, pv, load, tz, plant=plant) + views: list[RiskWindowView] = [] + for window in windows.values(): + q = net_energy_quantile(window, settings.risk, model, min_history_days=min_history_days) + views.append( + RiskWindowView( + window=window.kind, + start=window.start, + end=window.end, + weather=window.weather, + pv=_energy(window.pv), + load=_energy(window.load), + net_p50_wh=round(window.net_p50, 1), + net_quantile_wh=round(q.value_wh, 1), + source=q.source, + samples=q.samples, + pooled=q.pooled, + ) + ) + night = next(v for v in views if v.window is RiskWindow.NIGHT) + source: DistributionSource = night.source + buffer = target = None + battery = settings.model + if battery is not None: + evening = evening_target_pct( + battery, + settings, + windows, + user_target_pct=user_target_pct, + site_battery=site_battery, + model=model, + min_history_days=min_history_days, + ) + morning = morning_buffer_pct( + battery, + settings, + windows, + evening.soc_pct, + site_battery=site_battery, + model=model, + min_history_days=min_history_days, + ) + source = evening.source + target = RiskTargetView( + soc_pct=round(evening.soc_pct, 1), reason=evening.reason, source=evening.source + ) + buffer = RiskTargetView( + soc_pct=round(morning.soc_pct, 1), reason=morning.reason, source=morning.source + ) + return RiskView( + status="ok" if battery is not None else "no_battery_model", + distribution_source=source, + evening_at=windows[RiskWindow.NIGHT].start, + windows=views, + morning_buffer=buffer, + evening_target=target, + **base, + ) + + +__all__ = [ + "BACKFILL_DAYS", + "DAY_ANCHOR_OFFSET", + "EVENING_ANCHOR_OFFSET", + "KEEP_DAYS", + "NS_NET_ERROR", + "BaseLoadReader", + "RiskData", + "StoredErrors", + "learn_day", + "learn_net_errors", + "load_error_model", + "risk_view", + "save_error_model", +] diff --git a/suncore/forecast/stats.py b/suncore/forecast/stats.py new file mode 100644 index 0000000..a8ba20c --- /dev/null +++ b/suncore/forecast/stats.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Small robust statistics shared by the load forecast and the risk model (Phase 4 WP6/WP7). + +Pure functions on plain floats; no NumPy needed for the sizes involved (a few thousand values). +""" + +from __future__ import annotations + +import math +from collections.abc import Sequence +from typing import Final + +MAD_TO_SIGMA: Final = 1.4826 +"""Scale factor that makes the median absolute deviation a consistent estimate of the standard +deviation of a normal distribution.""" + + +def median(values: Sequence[float]) -> float: + """The median; ``ValueError`` on an empty sequence.""" + if not values: + msg = "median of an empty sequence" + raise ValueError(msg) + ordered = sorted(values) + mid = len(ordered) // 2 + if len(ordered) % 2: + return ordered[mid] + return (ordered[mid - 1] + ordered[mid]) / 2 + + +def robust_scale(values: Sequence[float], center: float) -> float: + """``MAD_TO_SIGMA`` times the median absolute deviation from ``center``.""" + return MAD_TO_SIGMA * median([abs(v - center) for v in values]) + + +def clip_outliers(values: Sequence[float], z: float, floor: float) -> list[float]: + """Clipped z-score: every value is clipped into ``median +- z * scale``. + + ``scale`` is the MAD-based robust standard deviation, at least ``floor`` (a cell whose + values are almost all equal would otherwise clip everything onto the median). + """ + if not values: + return [] + center = median(values) + scale = max(robust_scale(values, center), floor) + lo, hi = center - z * scale, center + z * scale + return [min(hi, max(lo, v)) for v in values] + + +def weighted_mean(values: Sequence[float], weights: Sequence[float]) -> float: + total = sum(weights) + if total <= 0: + msg = "weighted mean needs a positive total weight" + raise ValueError(msg) + return sum(v * w for v, w in zip(values, weights, strict=True)) / total + + +def weighted_quantile(values: Sequence[float], weights: Sequence[float], q: float) -> float: + """The ``q``-quantile of a weighted sample (linear interpolation between the midpoints of + the cumulative weights; equal weights give the usual "Hazen" plotting positions). + + ``q`` in ``[0, 1]``; values beyond the outermost midpoints take the extreme value (the + sample never extrapolates). + """ + if not values: + msg = "quantile of an empty sample" + raise ValueError(msg) + if not 0 <= q <= 1: + msg = "q must be in [0, 1]" + raise ValueError(msg) + pairs: list[tuple[float, float]] = sorted( + ((v, w) for v, w in zip(values, weights, strict=True) if w > 0), key=lambda p: p[0] + ) + if not pairs: + msg = "quantile needs a positive weight" + raise ValueError(msg) + total = sum(w for _, w in pairs) + positions: list[float] = [] + running = 0.0 + for _, w in pairs: + positions.append((running + w / 2) / total) + running += w + if q <= positions[0]: + return pairs[0][0] + if q >= positions[-1]: + return pairs[-1][0] + for i in range(1, len(pairs)): + if q <= positions[i]: + lo, hi = positions[i - 1], positions[i] + share = 0.0 if math.isclose(hi, lo) else (q - lo) / (hi - lo) + return pairs[i - 1][0] + share * (pairs[i][0] - pairs[i - 1][0]) + return pairs[-1][0] # pragma: no cover - q < positions[-1] ends the loop + + +def recency_weight(age_days: float, half_life_days: float) -> float: + """Exponential-smoothing weight of an observation ``age_days`` old: ``0.5 ** (age / h)``.""" + return math.pow(0.5, max(0.0, age_days) / half_life_days) + + +__all__ = [ + "MAD_TO_SIGMA", + "clip_outliers", + "median", + "recency_weight", + "robust_scale", + "weighted_mean", + "weighted_quantile", +] diff --git a/suncore/forecast/views.py b/suncore/forecast/views.py index 1bdffa7..23aa91b 100644 --- a/suncore/forecast/views.py +++ b/suncore/forecast/views.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""Read models of the PV forecast and its accuracy (what the API serves; Phase 4 WP5). +"""Read models of the PV and load forecasts and their accuracy (what the API serves; Phase 4 +WP5/WP6). Values are W; ``None`` is an unknown slot (never zero). ``status`` is a :class:`~suncore.forecast.pv.adapter.FetchStatus` value, rendered by the UI as @@ -8,12 +9,17 @@ from __future__ import annotations +from datetime import date from typing import Annotated, Literal from pydantic import AwareDatetime, BaseModel, ConfigDict, Field +from suncore.core.battery import BatteryStrategy, RiskLevel +from suncore.core.decisions import Reason from suncore.core.forecast import AccuracyRecord, HorizonBucket +from suncore.forecast.load import DayType, DayTypeSource, LoadQuality, TemperatureReason from suncore.forecast.pv.adapter import FetchStatus +from suncore.forecast.risk import DistributionSource, RiskClass, RiskWindow MAX_VIEW_HOURS = 96 """Largest ``hours`` the API serves.""" @@ -141,14 +147,150 @@ class ForecastAccuracyView(_View): sources: list[AccuracySourceView] +class LoadDayView(_View): + """The day type (and forecast mean temperature) of one local date of the forecast.""" + + date: date + day_type: DayType + source: DayTypeSource + """``workday_sensor`` (the sensor's state, for today + its ``days_offset``), ``check_date`` + (the Workday action) or ``weekday_rule`` (Mon-Fri workday, Sat/Sun weekend).""" + temperature_c: float | None + """Mean outside temperature of the day the temperature term used (``None``: none).""" + + +class LoadTemperatureView(_View): + """The temperature term of the load profile. + + ``reason``: ``active``, ``not_configured`` (no weather entity), ``too_few_days`` (fewer + than 14 days with a recorded temperature) or ``no_spread`` (the temperatures varied too + little to learn anything).""" + + active: bool + reason: TemperatureReason + days: int + heating_w_per_k: float + """Extra base load (W) per kelvin of heating degrees (below 15 °C).""" + cooling_w_per_k: float + """Extra base load (W) per kelvin of cooling degrees (above 22 °C).""" + + +class LoadForecastView(_View): + """The base load forecast (W, P10/P50/P90 on the 15-minute grid from the current slot). + + The base load is the house consumption without the configured (controllable) devices; their + draw is planned separately. ``quality``: ``ok`` (profiles from history), ``cold_start`` + (fewer than 3 days: a flat line at the recent mean with a wide band) or ``no_history`` + (nothing measured yet: no forecast).""" + + generated_at: AwareDatetime + hours: int + source_id: str + quality: LoadQuality + days_of_history: int + """Complete local days the profiles were learnt from.""" + fitted_at: AwareDatetime | None + start: AwareDatetime | None + step_s: Annotated[int, Field(gt=0)] + p10_w: list[float | None] + p50_w: list[float | None] + p90_w: list[float | None] + days: list[LoadDayView] + temperature: LoadTemperatureView + controlled_devices: list[str] + """Devices subtracted from the house consumption (forecast by the planner, not here).""" + devices_unmeasured: dict[str, int] + """Per device the quarter hours of the fit window without a measurement (counted as 0 W: + the base load still contains its draw there, e.g. in the history rebuilt from Home + Assistant's statistics).""" + clamped_slots: int + """Quarter hours where the devices drew more than the house measured (clipped to 0 W).""" + unmeasured_in_profile: list[DayType] + """Day types whose profile still contains quarter hours with an unmeasured device (fewer + than 7 fully measured days of that type; elsewhere those quarter hours are left out).""" + + +class EnergyView(_View): + """P10/P50/P90 energy (Wh) of a window (the slot quantiles summed).""" + + p10_wh: float + p50_wh: float + p90_wh: float + + +class RiskWindowView(_View): + """One window of the risk model and its net energy (PV - base load) at the risk quantile.""" + + window: RiskWindow + start: AwareDatetime + end: AwareDatetime + weather: RiskClass + pv: EnergyView + load: EnergyView + net_p50_wh: float + net_quantile_wh: float + """Net energy that is reached with the risk level's probability.""" + source: DistributionSource + samples: int + """Learnt samples behind ``net_quantile_wh`` (0 for ``source_quantiles``).""" + pooled: bool + """Learnt over all weather classes of the window (its own class had too few samples).""" + + +class RiskTargetView(_View): + """A computed state of charge with its structured reason.""" + + soc_pct: float + reason: Reason + source: DistributionSource + + +class RiskView(_View): + """The current morning buffer and evening target and what they rest on (Phase 4 WP7). + + ``status``: ``ok``; ``no_battery_model`` (the ``battery`` section has no model yet: the + windows are shown, the targets are ``null``); ``no_forecast`` (no load forecast yet). + ``distribution_source``: ``learned`` once ``min_history_days`` days of measured net errors + exist, else ``source_quantiles`` (the PV sources' P10/P90 and the load residuals). + """ + + generated_at: AwareDatetime + status: Literal["ok", "no_battery_model", "no_forecast"] + risk: RiskLevel + quantile: float + strategy: BatteryStrategy + bridge_next_day: bool + pv_forecast: bool + """A PV forecast was available (without one PV counts as 0 Wh: the cautious direction).""" + distribution_source: DistributionSource + history_days: int + """Days with learnt net-error samples.""" + min_history_days: int + samples: dict[str, int] + """Learnt samples per ``window/weather`` cell.""" + evening_at: AwareDatetime | None + """Start of the ``night`` window: today's sunset (the time the evening target refers to); + after sunset the current quarter hour (the evening has begun).""" + windows: list[RiskWindowView] + morning_buffer: RiskTargetView | None + evening_target: RiskTargetView | None + + __all__ = [ "MAX_VIEW_HOURS", "AccuracySourceView", "BlendWeightsView", "BlendedView", "CalibrationView", + "EnergyView", "ForecastAccuracyView", + "LoadDayView", + "LoadForecastView", + "LoadTemperatureView", "PvForecastView", "PvSeriesView", "PvSourceView", + "RiskTargetView", + "RiskView", + "RiskWindowView", ] diff --git a/suncore/sim/demo_config.py b/suncore/sim/demo_config.py index 2a5d130..44d1efd 100644 --- a/suncore/sim/demo_config.py +++ b/suncore/sim/demo_config.py @@ -12,7 +12,7 @@ from typing import Final -from suncore.core.battery import BatterySettings +from suncore.core.battery import BatteryModel, BatterySettings from suncore.core.config import ( BatteryTarget, DevicesConfig, @@ -21,6 +21,7 @@ ForecastSourceConfig, GoalsConfig, ImportTariff, + LoadForecastConfig, OperationConfig, RegionConfig, TariffConfig, @@ -42,6 +43,8 @@ POOL_SWITCH_ID, PRICE_ID, ROD_LIMIT_ID, + WEATHER_ID, + WORKDAY_ID, ) from suncore.storage import ConfigStore from suncore.templates import DeviceTemplate, TemplateCatalog @@ -196,6 +199,9 @@ async def seed_demo_config(config: ConfigStore, catalog: TemplateCatalog) -> Non template_id="forecast_solar.pv", config_entry_id="sim_entry_forecast_solar" ), ], + load=LoadForecastConfig( + workday=EntityRef(entity_id=WORKDAY_ID), weather=EntityRef(entity_id=WEATHER_ID) + ), ), ) await config.put("devices", demo_devices(catalog)) @@ -207,6 +213,12 @@ async def seed_demo_config(config: ConfigStore, catalog: TemplateCatalog) -> Non ), ) await config.put("operation", OperationConfig(mode=OperatingMode.SHADOW)) - # Observe only: the demo battery is read, never written; the model follows with WP8. - await config.put("battery", BatterySettings()) + # Observe only (no adapter): the demo battery is read, never written. The model (the + # simulator's 10 kWh, 5 kW battery) lets the risk model show its buffer and evening target. + await config.put( + "battery", + BatterySettings( + model=BatteryModel(capacity_wh=10_000.0, max_charge_w=5_000.0, max_discharge_w=5_000.0) + ), + ) await config.put("wizard", WizardState(completed=True, completed_steps=list(WIZARD_STEPS))) diff --git a/suncore/sim/demo_devices.py b/suncore/sim/demo_devices.py index 9ec914f..c7add45 100644 --- a/suncore/sim/demo_devices.py +++ b/suncore/sim/demo_devices.py @@ -43,7 +43,7 @@ import math from dataclasses import dataclass, field -from datetime import datetime, timedelta +from datetime import date, datetime, timedelta from typing import Any, Final from suncore.sim.devices import ( @@ -86,6 +86,11 @@ OPEN_METEO_TODAY_ID: Final = "sensor.sim_open_meteo_energy_production_today" OPEN_METEO_TOMORROW_ID: Final = "sensor.sim_open_meteo_energy_production_tomorrow" TEMPLATE_FORECAST_ID: Final = "sensor.sim_pv_forecast_template" +WORKDAY_ID: Final = "binary_sensor.sim_workday_sensor" +"""Workday-integration-like sensor (``on`` Mon-Fri, deterministic; no holidays).""" +WEATHER_ID: Final = "weather.sim_home" +"""Weather entity: ``temperature`` (°C) from the scenario's outdoor model; hourly forecast via +the read-only action ``weather.get_forecasts``.""" GRID_IMPORT_KWH_ID: Final = "sensor.sim_grid_energy_import" GRID_EXPORT_KWH_ID: Final = "sensor.sim_grid_energy_export" PV_KWH_ID: Final = "sensor.sim_pv_energy" @@ -111,6 +116,8 @@ OPEN_METEO_TODAY_ID, OPEN_METEO_TOMORROW_ID, TEMPLATE_FORECAST_ID, + WORKDAY_ID, + WEATHER_ID, GRID_IMPORT_KWH_ID, GRID_EXPORT_KWH_ID, PV_KWH_ID, @@ -354,6 +361,8 @@ def entities(self) -> list[DemoEntity]: }, ), *forecasts, + self._workday(state.time), + self._weather(state.time), self._kwh(GRID_IMPORT_KWH_ID, "Sim grid energy import", c.grid_import_kwh), self._kwh(GRID_EXPORT_KWH_ID, "Sim grid energy export", c.grid_export_kwh), self._kwh(PV_KWH_ID, "Sim PV energy", c.pv_kwh), @@ -365,6 +374,46 @@ def entities(self) -> list[DemoEntity]: entities.append(self._kwh(POOL_ENERGY_ID, "Sim pool pump energy", c.pool_kwh)) return entities + @staticmethod + def is_workday(day: date) -> bool: + """The demo's workday calendar: Monday to Friday, no holidays.""" + return day.weekday() < 5 + + def _workday(self, now: datetime) -> DemoEntity: + return DemoEntity( + WORKDAY_ID, + "on" if self.is_workday(now.astimezone(self.scenario.start.tzinfo).date()) else "off", + { + "friendly_name": "Sim workday sensor", + "workdays": ["mon", "tue", "wed", "thu", "fri"], + "excludes": ["sat", "sun", "holiday"], + "days_offset": 0, + }, + ) + + def _weather(self, now: datetime) -> DemoEntity: + return DemoEntity( + WEATHER_ID, + "sunny" if self._cloud_at(now) >= 0.8 else "cloudy", + { + "friendly_name": "Sim home", + "temperature": round(self.outdoor_c(now), 1), + "temperature_unit": "°C", + }, + ) + + def hourly_weather(self, now: datetime, hours: int = 48) -> list[dict[str, Any]]: + """``weather.get_forecasts`` items (type ``hourly``) from the next full hour on.""" + first = now.replace(minute=0, second=0, microsecond=0) + timedelta(hours=1) + return [ + { + "datetime": (first + timedelta(hours=h)).isoformat(), + "temperature": round(self.outdoor_c(first + timedelta(hours=h)), 1), + "condition": "sunny", + } + for h in range(hours) + ] + @staticmethod def _kwh(entity_id: str, name: str, value: float) -> DemoEntity: return DemoEntity( diff --git a/suncore/sim/fake_ha.py b/suncore/sim/fake_ha.py index df575d2..ed38f7c 100644 --- a/suncore/sim/fake_ha.py +++ b/suncore/sim/fake_ha.py @@ -50,7 +50,7 @@ import logging from collections.abc import Awaitable, Callable, Collection, Mapping, Sequence from dataclasses import dataclass -from datetime import datetime, timedelta +from datetime import date, datetime, timedelta from functools import partial from typing import Any, Final @@ -82,6 +82,8 @@ FORECAST_TODAY_ID, FORECAST_TOMORROW_ID, SIM_DEVICE_ENTITY_IDS, + WEATHER_ID, + WORKDAY_ID, DemoDevices, sim_price, ) @@ -468,7 +470,7 @@ async def call_action_response( service: str, data: Mapping[str, Any] | None = None, *, - target: Mapping[str, Any] | None = None, # noqa: ARG002 + target: Mapping[str, Any] | None = None, min_interval_s: float | None = None, max_wait_s: float = 0.0, rate_key: str | None = None, @@ -494,6 +496,13 @@ async def call_action_response( if day is not None: slots_data.extend(day.attributes.get("detailedForecast", [])) return {"data": _solcast_query_window(slots_data, data)} + entity_id = str((target or {}).get("entity_id", "")) + demo = self.demo + if (domain, service) == ("weather", "get_forecasts") and entity_id == WEATHER_ID: + return {entity_id: {"forecast": demo.hourly_weather(now)}} + if (domain, service) == ("workday", "check_date") and entity_id == WORKDAY_ID: + asked = date.fromisoformat(str((data or {}).get("check_date"))) + return {entity_id: {"workday": demo.is_workday(asked)}} raise CommandError("not_found", f"Service {domain}.{service} not found.") def entity_metas(self) -> list[EntityMeta]: diff --git a/suncore/sim/registry.py b/suncore/sim/registry.py index 506d25e..3daf4d7 100644 --- a/suncore/sim/registry.py +++ b/suncore/sim/registry.py @@ -144,6 +144,8 @@ def _sim(entity_id: str, name: str) -> RegEntity: ), # A user's Home Assistant template sensor that exposes the generic forecast attribute. RegEntity(d.TEMPLATE_FORECAST_ID, "template", "PV forecast", DEV_HOUSE), + RegEntity(d.WORKDAY_ID, "workday", "Workday sensor", DEV_HOUSE), + RegEntity(d.WEATHER_ID, "met", "Home", DEV_HOUSE), ) AREAS: Final[tuple[tuple[str, str], ...]] = ((AREA_LIVING, "Living room"), (AREA_GARDEN, "Garden")) diff --git a/suncore/storage/timeseries.py b/suncore/storage/timeseries.py index 5d79606..7950c1b 100644 --- a/suncore/storage/timeseries.py +++ b/suncore/storage/timeseries.py @@ -42,7 +42,7 @@ import structlog -from suncore.core.measurements import is_percent_key +from suncore.core.measurements import WEATHER_TEMPERATURE_C, is_percent_key from suncore.core.series import ( MAX_POINTS, PriceIncludes, @@ -223,6 +223,8 @@ def _opt_num(value: object) -> float | None: def _default_unit(key: str) -> SeriesUnit: + if key == WEATHER_TEMPERATURE_C: + return SeriesUnit.CELSIUS return SeriesUnit.PERCENT if is_percent_key(key) else SeriesUnit.W diff --git a/tests/api/conftest.py b/tests/api/conftest.py index 29aecc5..f6315f9 100644 --- a/tests/api/conftest.py +++ b/tests/api/conftest.py @@ -26,6 +26,7 @@ UnknownTemplateError, ) from suncore.core.balance import Flow, FlowSnapshot +from suncore.core.battery import BatterySettings from suncore.core.config import SECTION_MODELS, OperationConfig from suncore.core.control import DeviceState, PowerEstimate from suncore.core.controlstate import ( @@ -52,7 +53,10 @@ from suncore.core.site import EntityRef, GridConfig, PvArray, SiteConfig from suncore.core.wizard import CheckItem, PreviewItem, PreviewTooLargeError from suncore.forecast.accuracy import accuracy_view -from suncore.forecast.views import ForecastAccuracyView, PvForecastView +from suncore.forecast.load_service import LoadForecastService +from suncore.forecast.risk import ErrorModel +from suncore.forecast.risk_learning import risk_view +from suncore.forecast.views import ForecastAccuracyView, LoadForecastView, PvForecastView, RiskView from suncore.storage import ( HistoryEntry, HistoryEntryNotFoundError, @@ -213,6 +217,19 @@ def __init__(self) -> None: self.pv_view = PvForecastView(generated_at=NOW, hours=48, sources=[], blended=None) self.accuracy = accuracy_view([], {}) self.pv_hours: list[int] = [] + self.load_view = LoadForecastService(None, None, clock=lambda: NOW).view(48) + self.load_hours: list[int] = [] + self.risk = risk_view( + NOW, + pv=None, + load=None, + settings=BatterySettings(), + user_target_pct=None, + model=ErrorModel(), + min_history_days=30, + tz=UTC, + plant=None, + ) async def pv_forecast(self, hours: int) -> PvForecastView: self.pv_hours.append(hours) @@ -221,6 +238,13 @@ async def pv_forecast(self, hours: int) -> PvForecastView: async def forecast_accuracy(self) -> ForecastAccuracyView: return self.accuracy + async def load_forecast(self, hours: int) -> LoadForecastView: + self.load_hours.append(hours) + return self.load_view + + async def forecast_risk(self) -> RiskView: + return self.risk + async def health(self) -> HealthReport: self.health_calls += 1 return HealthReport(self.ha_state, 1.5, self.db_ok, self.version) diff --git a/tests/api/test_forecast_api.py b/tests/api/test_forecast_api.py index 6d318d6..3c3e5f7 100644 --- a/tests/api/test_forecast_api.py +++ b/tests/api/test_forecast_api.py @@ -165,3 +165,29 @@ async def test_the_openapi_document_describes_both_routes(services: FakeServices accuracy = schema["paths"]["/api/v1/forecast/accuracy"]["get"]["description"] assert "actual -" in accuracy assert "blended" in accuracy + + +async def test_load_forecast_endpoint() -> None: + services = FakeServices() + async with client_for(create_app(services)) as client: + response = await client.get("/api/v1/forecast/load?hours=12") + assert response.status_code == 200 + body = response.json() + assert body["source_id"] == "load_profile" + assert body["quality"] == "no_history" + assert body["temperature"]["reason"] == "not_configured" + assert (await client.get("/api/v1/forecast/load?hours=0")).status_code == 422 + assert services.load_hours == [12] + + +async def test_risk_endpoint() -> None: + services = FakeServices() + async with client_for(create_app(services)) as client: + response = await client.get("/api/v1/forecast/risk") + assert response.status_code == 200 + body = response.json() + assert body["status"] == "no_forecast" + assert body["risk"] == "cautious" + assert body["quantile"] == 0.1 + assert body["distribution_source"] == "source_quantiles" + assert body["min_history_days"] == 30 diff --git a/tests/app/test_demo.py b/tests/app/test_demo.py index 6339205..1c9b5b0 100644 --- a/tests/app/test_demo.py +++ b/tests/app/test_demo.py @@ -121,9 +121,12 @@ async def test_demo_live_feed_balance_closes(demo_runtime: Runtime) -> None: "sensor.sim_solcast_forecast_tomorrow", "sensor.sim_open_meteo_energy_production_today", "sensor.sim_open_meteo_energy_production_tomorrow", + "binary_sensor.sim_workday_sensor", + "weather.sim_home", } ) -"""The configured PV forecast sources' day sensors are subscribed too (Phase 4 WP5).""" +"""The configured PV forecast sources' day sensors (Phase 4 WP5) and the load forecast's workday +sensor and weather entity (WP6) are subscribed too.""" DEMO_SUBSCRIBED = frozenset(SIM_ENTITY_IDS) | DEMO_DEVICE_ENTITIES | DEMO_FORECAST_ENTITIES diff --git a/tests/app/test_history.py b/tests/app/test_history.py index 937743f..002ad16 100644 --- a/tests/app/test_history.py +++ b/tests/app/test_history.py @@ -158,3 +158,48 @@ async def statistics_during_period(self, ids, *_args, **_kw): # type: ignore[no SITE_HOUSE_W, TimeRange(start=T0, end=T0 + timedelta(hours=1)) ) assert house.total() == pytest.approx(900.0) + + +async def test_rebuild_includes_the_devices_power_sensors(tmp_path) -> None: # type: ignore[no-untyped-def] + from pathlib import Path # noqa: PLC0415 + + from suncore.app.history import device_statistic_specs # noqa: PLC0415 + from suncore.core.measurements import device_power_key # noqa: PLC0415 + from suncore.ha.connection import HAConnection # noqa: PLC0415 + from suncore.sim.demo_config import demo_devices # noqa: PLC0415 + from suncore.templates import load_templates # noqa: PLC0415 + + devices = demo_devices(load_templates(Path(__file__).parents[2] / "templates")).devices + specs = device_statistic_specs(devices) + assert {s.key: s.statistic_id for s in specs} == { + "device:living_room_ac": "sensor.sim_living_room_ac_energy_current", + "device:pool_pump": "sensor.sim_pool_pump_power", + } # the heating rod has no power sensor (nominal model) + asked: list[str] = [] + + class Stats: + async def statistics_during_period(self, ids, *_args, **_kw): # type: ignore[no-untyped-def] + asked.extend(ids) + hour = int(T0.timestamp() * 1000) + rows = { + "sensor.grid": [{"start": hour, "end": hour + 3_600_000, "mean": 400.0}], + "sensor.sim_pool_pump_power": [ + {"start": hour, "end": hour + 3_600_000, "mean": 750.0} + ], + } + return {i: rows[i] for i in ids if i in rows} + + site = SiteConfig(grid=GridConfig(power=ref("grid"))) + async with await open_timeseries(tmp_path / "t.db") as db: + store = TimeseriesStore(db) + connection: HAConnection = Stats() # type: ignore[assignment] + result = await rebuild_history( + connection, store, site, T0 + timedelta(hours=1), days=1, devices=devices + ) + assert "sensor.sim_pool_pump_power" in asked + assert result.stored[device_power_key("pool_pump")] == 4 + assert "sensor.sim_living_room_ac_energy_current" in result.missing + pool = await store.read_series( + device_power_key("pool_pump"), TimeRange(start=T0, end=T0 + timedelta(hours=1)) + ) + assert pool.values == (750.0,) * 4 diff --git a/tests/app/test_load_runtime.py b/tests/app/test_load_runtime.py new file mode 100644 index 0000000..2d05161 --- /dev/null +++ b/tests/app/test_load_runtime.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The load forecast in the runtime: wiring, demo workday/weather fakes, rebuild (Phase 4 WP6).""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator, Callable +from datetime import UTC, date, datetime, timedelta +from pathlib import Path + +import pytest + +from suncore.app.adapters import HaLoadReader +from suncore.app.main import build_runtime +from suncore.app.runtime import Runtime +from suncore.core.config import DevicesConfig, ForecastConfig +from suncore.forecast.load import DayTypeSource, LoadQuality +from suncore.sim.demo_devices import WEATHER_ID, WORKDAY_ID +from tests.app.conftest import RecordingConnection, demo_settings + +SCENARIO_DAY = datetime(2026, 6, 21, tzinfo=UTC) + + +async def until(predicate: Callable[[], bool], timeout_s: float = 10.0) -> None: + async with asyncio.timeout(timeout_s): + while not predicate(): # noqa: ASYNC110 - polling a plain predicate + await asyncio.sleep(0.01) + + +@pytest.fixture +async def runtime(tmp_path: Path) -> AsyncIterator[Runtime]: + conn = RecordingConnection(speed=6000.0) + + def clock() -> datetime: + return conn.sim_time or SCENARIO_DAY + + rt = await build_runtime(demo_settings(tmp_path), env={}, connection=conn, clock=clock) + await rt.start() + try: + yield rt + finally: + await rt.stop() + + +async def test_the_demo_load_forecast_uses_its_workday_and_weather(runtime: Runtime) -> None: + service = runtime.load_service + assert service is not None + assert service.inputs.workday_entity == WORKDAY_ID + assert service.inputs.weather_entity == WEATHER_ID + assert set(service.inputs.device_ids) == {d.id for d in runtime.devices.devices} # type: ignore[union-attr] + await until(lambda: service.model is not None) + await until(lambda: runtime.cache.get(WORKDAY_ID) is not None) + await service.refresh() + view = await runtime.load_forecast(24) + assert view.quality in (LoadQuality.OK, LoadQuality.COLD_START) + assert view.p50_w + assert view.days[0].source is DayTypeSource.WORKDAY_SENSOR + assert {d.source for d in view.days[1:]} <= {DayTypeSource.CHECK_DATE} + assert view.days[1].temperature_c is not None # from weather.get_forecasts + accuracy = await runtime.forecast_accuracy() + assert "load_profile" in [s.source_id for s in accuracy.sources] + + +async def test_the_reader_reads_the_fake_home_assistant(runtime: Runtime) -> None: + await until(lambda: runtime.cache.get(WEATHER_ID) is not None) + reader = HaLoadReader(runtime.connection, runtime.cache) + assert reader.outside_temperature(WEATHER_ID) is not None + state = reader.workday_state(WORKDAY_ID) + assert state is not None + assert state.workday is (SCENARIO_DAY.weekday() < 5) + assert state.offset == 0 + monday = date(2026, 6, 22) + assert await reader.check_workday(WORKDAY_ID, monday, min_interval_s=1.0) is True + assert ( + await reader.check_workday(WORKDAY_ID, monday + timedelta(days=5), min_interval_s=1.0) + is False + ) + points = await reader.hourly_temperatures(WEATHER_ID, min_interval_s=1.0) + assert len(points) == 48 + assert reader.outside_temperature("weather.unknown") is None + assert reader.workday_state("binary_sensor.unknown") is None + + +async def test_a_devices_or_forecast_change_rebuilds_the_load_forecast(runtime: Runtime) -> None: + old = runtime.load_service + assert old is not None + await runtime.put_section("devices", DevicesConfig()) + assert runtime.load_service is not old + assert runtime.load_service is not None + assert runtime.load_service.inputs.device_ids == () + await runtime.put_section("forecast", ForecastConfig()) + assert runtime.load_service.inputs.workday_entity is None + + +async def test_the_demo_risk_view_has_a_buffer_and_a_target(runtime: Runtime) -> None: + service = runtime.load_service + assert service is not None + await until(lambda: service.model is not None) + await until(lambda: runtime.cache.get(WORKDAY_ID) is not None) + await service.refresh() + view = await runtime.forecast_risk() + assert view.status == "ok" # the demo battery has a model + assert view.evening_target is not None + assert view.morning_buffer is not None + assert view.distribution_source == "source_quantiles" + assert view.history_days == 0 + await runtime.update_forecast_accuracy() # the nightly job learns (nothing yet) safely + assert runtime.risk_model.samples == () diff --git a/tests/app/test_pv_runtime.py b/tests/app/test_pv_runtime.py index 6cc3408..2d0b153 100644 --- a/tests/app/test_pv_runtime.py +++ b/tests/app/test_pv_runtime.py @@ -151,7 +151,7 @@ async def test_changing_the_forecast_section_rebuilds_the_service(runtime: Runti assert view.sources == [] assert view.blended is None accuracy = await runtime.forecast_accuracy() - assert accuracy.sources == [] + assert [s.source_id for s in accuracy.sources] == ["load_profile"] # the load forecast async def test_unusable_templates_are_skipped_not_fatal(runtime: Runtime) -> None: @@ -193,6 +193,7 @@ async def test_the_nightly_job_stores_accuracy_that_the_api_reads(runtime: Runti "open_meteo_solar_forecast.pv", "forecast_solar.pv", "blended", + "load_profile", # the load forecast is scored by the same nightly job (WP6) } record = next(r for r in by_id["solcast.pv"].records if r.horizon is HorizonBucket.H6_24) assert record.bias_w == 300.0 diff --git a/tests/app/test_timeseries_runtime.py b/tests/app/test_timeseries_runtime.py index 7cfd5f7..74967fa 100644 --- a/tests/app/test_timeseries_runtime.py +++ b/tests/app/test_timeseries_runtime.py @@ -14,7 +14,7 @@ import structlog.testing from suncore.app.control import ControlStack -from suncore.app.history import site_statistic_specs +from suncore.app.history import device_statistic_specs, site_statistic_specs from suncore.app.main import build_runtime from suncore.app.recorder import MeasurementRecorder from suncore.app.runtime import Runtime @@ -510,7 +510,9 @@ async def test_rebuild_runs_once_after_connect_when_the_15_minute_table_is_empty assert count == (72 * 4,) assert all(c["period"] == "hour" for c in conn.recorder.calls) asked = {i for c in conn.recorder.calls for i in c["ids"]} - assert asked == {s.statistic_id for s in site_statistic_specs(runtime.site)} # type: ignore[arg-type] + devices = runtime.devices.devices # type: ignore[union-attr] + expected = [*site_statistic_specs(runtime.site), *device_statistic_specs(devices)] # type: ignore[arg-type] + assert asked == {s.statistic_id for s in expected} # the devices' power sensors too (M2) calls = len(conn.recorder.calls) runtime._start_history_rebuild() # a reconnect or a site edit does not rebuild again await asyncio.sleep(0.05) diff --git a/tests/core/test_config_phase4.py b/tests/core/test_config_phase4.py index ba72101..3fa38b6 100644 --- a/tests/core/test_config_phase4.py +++ b/tests/core/test_config_phase4.py @@ -207,3 +207,33 @@ def test_device_planner_value_is_optional() -> None: ) with pytest.raises(ValidationError): DeviceConfig.model_validate(doc | {"planner_value_per_kwh": -0.1}) + + +# --- forecast.load (WP6, additive in v2) --------------------------------------------------- + + +def test_forecast_load_inputs_are_optional_and_additive() -> None: + stored_v2 = {"schema_version": 2, "sources": [], "learning": {"min_history_days": 30}} + config = ForecastConfig.model_validate(stored_v2) + assert config.load.workday is None + assert config.load.weather is None + assert config.entity_refs() == [] + + +def test_forecast_load_entities_are_checked_and_listed() -> None: + config = ForecastConfig.model_validate( + { + "load": { + "workday": {"entity_id": "binary_sensor.workday_sensor"}, + "weather": {"entity_id": "weather.forecast_home"}, + } + } + ) + assert [r.entity_id for r in config.entity_refs()] == [ + "binary_sensor.workday_sensor", + "weather.forecast_home", + ] + with pytest.raises(ValidationError, match="forecast_workday_domain"): + ForecastConfig.model_validate({"load": {"workday": {"entity_id": "sensor.x"}}}) + with pytest.raises(ValidationError, match="forecast_weather_domain"): + ForecastConfig.model_validate({"load": {"weather": {"entity_id": "sensor.x"}}}) diff --git a/tests/forecast/test_load_model.py b/tests/forecast/test_load_model.py new file mode 100644 index 0000000..968f84d --- /dev/null +++ b/tests/forecast/test_load_model.py @@ -0,0 +1,504 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The load profile model: base load, day types, smoothing, robustness, temperature, quantiles.""" + +from __future__ import annotations + +import math +import random +from collections.abc import Callable +from datetime import UTC, date, datetime, timedelta +from zoneinfo import ZoneInfo + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from suncore.core.series import SeriesUnit, TimeSeries, planning_grid +from suncore.forecast.load import ( + COLD_START_HIGH, + COLD_START_LOW, + DayType, + LoadQuality, + SlotBlock, + base_load, + daily_mean_temperatures, + day_type_from_workday, + day_values, + degree_days, + fit_load_model, + fit_temperature, + forecast_load, + parse_check_date, + parse_hourly_temperatures, + parse_workday_sensor, + slot_block, + temperature_c, + weekday_rule, +) +from suncore.forecast.stats import clip_outliers, median, weighted_quantile + +BERLIN = ZoneInfo("Europe/Berlin") +Q = timedelta(minutes=15) +START = datetime(2026, 9, 1, tzinfo=BERLIN).astimezone(UTC) # a Tuesday + + +def series(values: list[float | None], start: datetime = START) -> TimeSeries: + return TimeSeries(start=start, step=Q, values=tuple(values), unit=SeriesUnit.W) + + +def history(days: int, shape: Callable[[datetime], float], start: datetime = START) -> TimeSeries: + return series([shape(start + Q * i) for i in range(96 * days)], start) + + +def local(moment: datetime) -> datetime: + return moment.astimezone(BERLIN) + + +def evening_peak(moment: datetime) -> float: + t = local(moment) + level = 300.0 + (500.0 if 17 <= t.hour < 21 else 0.0) + if t.weekday() >= 5 and 9 <= t.hour < 13: + level += 400.0 + return level + + +def at(day: date, hour: int, minute: int = 0) -> datetime: + return datetime(day.year, day.month, day.day, hour, minute, tzinfo=BERLIN).astimezone(UTC) + + +# --- base load --------------------------------------------------------------------------- + + +def test_base_load_subtracts_each_device_once() -> None: + house = series([1000.0, 1000.0, None, 500.0]) + ac = series([400.0, None, 100.0, 800.0]) + pool = series([100.0, 100.0, 100.0, 0.0]) + result = base_load(house, {"ac": ac, "pool": pool}) + assert result.series.values == (500.0, 900.0, None, 0.0) + assert result.unmeasured_slots == {"ac": 1} # the None house slot is not counted + assert result.clamped_slots == 1 + + +def test_base_load_without_devices_is_the_house() -> None: + house = series([1.0, 2.0]) + assert base_load(house, {}).series.values == (1.0, 2.0) + + +def test_base_load_needs_one_grid() -> None: + with pytest.raises(ValueError, match="grid"): + base_load(series([1.0]), {"x": series([1.0], START + Q)}) + + +@given( + st.lists(st.one_of(st.none(), st.floats(0, 10_000)), min_size=1, max_size=40), + st.lists(st.one_of(st.none(), st.floats(-100, 5_000)), min_size=40, max_size=40), +) +def test_base_load_is_never_negative_nor_above_the_house( + house: list[float | None], device: list[float | None] +) -> None: + result = base_load(series(house), {"d": series(device[: len(house)])}) + for total, rest in zip(house, result.series.values, strict=True): + if total is None: + assert rest is None + else: + assert rest is not None + assert 0 <= rest <= total + 1e-9 + + +# --- day types ----------------------------------------------------------------------------- + + +def test_weekday_rule_and_workday_mapping() -> None: + monday, saturday = date(2026, 10, 19), date(2026, 10, 24) + assert weekday_rule(monday) is DayType.WORKDAY + assert weekday_rule(saturday) is DayType.WEEKEND + assert day_type_from_workday(monday, True) is DayType.WORKDAY + assert day_type_from_workday(monday, False) is DayType.HOLIDAY + assert day_type_from_workday(saturday, False) is DayType.WEEKEND + assert day_type_from_workday(saturday, True) is DayType.WORKDAY # a working Saturday + + +def test_workday_state_and_check_date_shapes() -> None: + entity_id = "binary_sensor.workday_sensor" + on = parse_workday_sensor(entity_id, "on", {"days_offset": 0, "workdays": ["mon"]}) + assert on is not None + assert (on.workday, on.offset) == (True, 0) + off = parse_workday_sensor(entity_id, "off", {}) + assert off is not None + assert not off.workday + assert parse_workday_sensor(entity_id, "unavailable", {}) is None + shifted = parse_workday_sensor(entity_id, "on", {"days_offset": 1}) + assert shifted is not None + assert shifted.offset == 1 # describes tomorrow; compensated by the service + assert shifted.fingerprint != on.fingerprint + assert parse_workday_sensor(entity_id, "on", {"days_offset": "1"}) is None + entity = "binary_sensor.workday_sensor" + assert parse_check_date({entity: {"workday": False}}, entity) is False + assert parse_check_date({entity: {"workday": "yes"}}, entity) is None + assert parse_check_date({}, entity) is None + + +def test_slot_blocks() -> None: + assert slot_block(0) is SlotBlock.NIGHT + assert slot_block(24) is SlotBlock.MORNING + assert slot_block(40) is SlotBlock.MIDDAY + assert slot_block(64) is SlotBlock.EVENING + assert slot_block(95) is SlotBlock.LATE + + +# --- day grid (DST) ------------------------------------------------------------------------- + + +def test_day_values_on_dst_days() -> None: + fall = datetime(2026, 10, 25, tzinfo=BERLIN).astimezone(UTC) # 25 h + spring = datetime(2026, 3, 29, tzinfo=BERLIN).astimezone(UTC) # 23 h + for start, slots_known in ((fall, 96), (spring, 92)): + data = series([100.0] * 100, start) + (day,) = day_values(data, BERLIN, start + timedelta(hours=26))[:1] + assert sum(v is not None for v in day.slots) == slots_known + assert day.coverage == 1.0 + + +# --- the profile ------------------------------------------------------------------------------ + + +def test_profile_learns_day_types() -> None: + data = history(42, evening_peak) + now = START + timedelta(days=42) + model = fit_load_model(data, now=now, tz=BERLIN, day_types={}) + assert model.quality is LoadQuality.OK + assert model.days == 42 + workday, weekend = model.profiles[DayType.WORKDAY], model.profiles[DayType.WEEKEND] + assert workday[19 * 4] == pytest.approx(800, abs=5) # 19:00 + assert workday[3 * 4] == pytest.approx(300, abs=5) + assert weekend[11 * 4] > 600 # weekend late-morning load + assert workday[11 * 4] < 320 + # no holidays seen: a holiday looks like a weekend day + assert model.profiles[DayType.HOLIDAY] == pytest.approx(weekend) + + +def test_holiday_profile_from_the_workday_sensor() -> None: + def shape(moment: datetime) -> float: + t = local(moment) + return 900.0 if t.date() in holidays and 12 <= t.hour < 14 else 300.0 + + holidays = {date(2026, 9, d) for d in (7, 14, 21, 28)} # Mondays off + data = history(35, shape) + types = dict.fromkeys(holidays, DayType.HOLIDAY) + model = fit_load_model(data, now=START + timedelta(days=35), tz=BERLIN, day_types=types) + assert model.day_counts[DayType.HOLIDAY] == 4 + assert model.profiles[DayType.HOLIDAY][13 * 4] > 600 # learnt, shrunk 4/(4+3) + assert model.profiles[DayType.WORKDAY][13 * 4] == pytest.approx(300, abs=1) + + +def test_one_outlier_day_barely_moves_the_profile() -> None: + def shape(moment: datetime) -> float: + t = local(moment) + if t.date() == date(2026, 10, 6) and 10 <= t.hour < 12: + return 9000.0 # a sauna afternoon + return 400.0 + + model = fit_load_model( + history(42, shape), now=START + timedelta(days=42), tz=BERLIN, day_types={} + ) + assert model.profiles[DayType.WORKDAY][11 * 4] < 450 + + +def test_exponential_smoothing_follows_a_level_change() -> None: + change = START + timedelta(days=35) + data = history(42, lambda m: 300.0 if m < change else 600.0) + model = fit_load_model(data, now=START + timedelta(days=42), tz=BERLIN, day_types={}) + level = model.profiles[DayType.WORKDAY][12 * 4] + # 7 recent days at 600 against 28 older days at 300: with a 21-day half-life the recent + # week weighs about 40 %, a plain mean would give 20 % + assert 380 < level < 500 + + +def test_cold_start_is_a_flat_wide_band() -> None: + data = history(2, lambda _: 500.0) + now = START + timedelta(days=2, hours=5) + model = fit_load_model(data, now=now, tz=BERLIN, day_types={}) + assert model.quality is LoadQuality.COLD_START + forecast = forecast_load(model, planning_grid(now, timedelta(hours=48)), BERLIN, {}) + assert forecast is not None + assert set(forecast.p50.values) == {500.0} + assert set(forecast.p10.values) == {COLD_START_LOW * 500} + assert set(forecast.p90.values) == {COLD_START_HIGH * 500} + + +def test_no_history_gives_no_forecast() -> None: + data = series([None] * 96) + model = fit_load_model(data, now=START + timedelta(days=3), tz=BERLIN, day_types={}) + assert model.quality is LoadQuality.NO_HISTORY + grid = planning_grid(START + timedelta(days=3), timedelta(hours=1)) + assert forecast_load(model, grid, BERLIN, {}) is None + + +def test_residual_quantiles_cover_the_noise() -> None: + rng = random.Random(7) + data = history(42, lambda _: 500.0 + rng.gauss(0, 100)) + now = START + timedelta(days=42) + model = fit_load_model(data, now=now, tz=BERLIN, day_types={}) + q10, q90 = model.residuals[SlotBlock.MIDDAY] + assert -170 < q10 < -90 + assert 90 < q90 < 170 + forecast = forecast_load(model, planning_grid(now, timedelta(hours=24)), BERLIN, {}) + assert forecast is not None + assert all( + lo is not None and mid is not None and hi is not None and lo <= mid <= hi + for lo, mid, hi in zip( + forecast.p10.values, forecast.p50.values, forecast.p90.values, strict=True + ) + ) + + +def test_forecast_uses_the_day_type_of_each_date() -> None: + data = history(42, evening_peak) + now = START + timedelta(days=42) # Tue 13 Oct 2026 + model = fit_load_model(data, now=now, tz=BERLIN, day_types={}) + grid = planning_grid(now, timedelta(hours=48)) + tomorrow = (now.astimezone(BERLIN) + timedelta(days=1)).date() + workday = forecast_load(model, grid, BERLIN, {}) + holiday = forecast_load(model, grid, BERLIN, {tomorrow: DayType.HOLIDAY}) + assert workday is not None + assert holiday is not None + index = (at(tomorrow, 11) - grid.start) // Q + assert holiday.p50.values[index] > workday.p50.values[index] # type: ignore[operator] + + +@settings(max_examples=30, deadline=None) +@given( + st.lists(st.one_of(st.none(), st.floats(0, 5000)), min_size=96 * 4, max_size=96 * 6), + st.integers(0, 10_000), +) +def test_forecast_quantiles_are_ordered_and_non_negative( + values: list[float | None], seed: int +) -> None: + data = series(values) + now = START + timedelta(days=len(values) // 96, minutes=15 * (seed % 96)) + model = fit_load_model(data, now=now, tz=BERLIN, day_types={}) + forecast = forecast_load(model, planning_grid(now, timedelta(hours=48)), BERLIN, {}) + if forecast is None: + return + for lo, mid, hi in zip( + forecast.p10.values, forecast.p50.values, forecast.p90.values, strict=True + ): + assert lo is not None + assert mid is not None + assert hi is not None + assert 0 <= lo <= mid <= hi + + +# --- temperature --------------------------------------------------------------------------------- + + +def temperatures(days: int, daily: Callable[[int], float]) -> TimeSeries: + values: list[float | None] = [daily(i // 96) for i in range(96 * days)] + return TimeSeries(start=START, step=Q, values=tuple(values), unit=SeriesUnit.CELSIUS) + + +def test_temperature_term_learns_heating() -> None: + def temp_of(day: int) -> float: + return 4.0 + (day * 7) % 13 # 4..16 °C, spread over the days + + def shape(moment: datetime) -> float: + day = (moment - START) // timedelta(days=1) + return 400.0 + 30.0 * degree_days(temp_of(day))[0] + + now = START + timedelta(days=42) + model = fit_load_model( + history(42, shape), + now=now, + tz=BERLIN, + day_types={}, + temperature=temperatures(42, temp_of), + temperature_configured=True, + ) + term = model.temperature + assert term.active + assert term.heating_w_per_k == pytest.approx(30, rel=0.1) + grid = planning_grid(now, timedelta(hours=24)) + day = now.astimezone(BERLIN).date() + cold = forecast_load(model, grid, BERLIN, {}, {day: 0.0}) + mild = forecast_load(model, grid, BERLIN, {}, {day: 14.0}) + assert cold is not None + assert mild is not None + assert cold.p50.values[0] > mild.p50.values[0] + 300 # type: ignore[operator] + + +def test_temperature_term_needs_days_and_spread() -> None: + assert fit_temperature([(0.0, 10.0, 1.0)] * 5).reason == "too_few_days" + flat = fit_temperature([(float(i), 18.0, 1.0) for i in range(20)]) + assert flat.reason == "no_spread" + assert not flat.active + assert flat.offset_w(-10.0) == 0.0 + + +def test_temperature_term_is_bounded() -> None: + samples = [(5000.0 * (i % 2), 5.0 + 10 * (i % 2), 1.0) for i in range(20)] + term = fit_temperature(samples) + assert 0 <= term.heating_w_per_k <= 300 + # negative dependence (load falls when it gets colder) is not a heating term + assert term.heating_w_per_k == 0.0 + + +def test_temperature_not_configured_is_reported() -> None: + model = fit_load_model( + history(10, lambda _: 1.0), now=START + timedelta(days=10), tz=BERLIN, day_types={} + ) + assert model.temperature.reason == "not_configured" + + +# --- Home Assistant shapes ----------------------------------------------------------------------- + + +def test_parse_hourly_temperatures() -> None: + entity = "weather.home" + response = { + entity: { + "forecast": [ + {"datetime": "2026-10-21T10:00:00+00:00", "temperature": 50.0}, + {"datetime": "2026-10-21T11:00:00", "temperature": 51.0}, # no zone + {"datetime": "2026-10-21T12:00:00+00:00", "temperature": None}, + {"datetime": "nonsense", "temperature": 1.0}, + {"condition": "sunny"}, + "junk", + ] + } + } + points = parse_hourly_temperatures(response, entity, "°F") + assert points == [(datetime(2026, 10, 21, 10, tzinfo=UTC), pytest.approx(10.0))] + assert parse_hourly_temperatures({entity: {"forecast": "x"}}, entity, None) == [] + assert parse_hourly_temperatures(response, entity, "X") == [] # unknown unit: never guessed + kelvin = parse_hourly_temperatures(response, entity, "K") + assert kelvin[0][1] == pytest.approx(50.0 - 273.15) + + +def test_temperature_units() -> None: + assert temperature_c("21.5", "°C") == 21.5 + assert temperature_c(32, "°F") == 0.0 + assert temperature_c(True, None) is None + assert temperature_c("nan", None) is None + assert temperature_c("x", None) is None + + +def test_daily_mean_temperatures_weigh_hours_alike() -> None: + day = date(2026, 10, 21) + points = [(at(day, h, m), 10.0) for h in range(6) for m in (0, 15, 30, 45)] + points += [(at(day, h), 20.0) for h in range(6, 12)] + points += [(at(day + timedelta(days=1), h), 5.0) for h in range(3)] + means = daily_mean_temperatures(points, BERLIN) + assert means == {day: 15.0} + + +# --- statistics helpers -------------------------------------------------------------------------- + + +def test_weighted_quantile_and_clipping() -> None: + values = [float(i) for i in range(1, 11)] + assert weighted_quantile(values, [1.0] * 10, 0.5) == pytest.approx(5.5) + assert weighted_quantile(values, [1.0] * 10, 0.0) == 1.0 + assert weighted_quantile(values, [1.0] * 10, 1.0) == 10.0 + assert weighted_quantile([1.0, 2.0], [0.0, 1.0], 0.1) == 2.0 + with pytest.raises(ValueError, match="empty"): + weighted_quantile([], [], 0.5) + with pytest.raises(ValueError, match="positive"): + weighted_quantile([1.0], [0.0], 0.5) + with pytest.raises(ValueError, match="q must"): + weighted_quantile([1.0], [1.0], 2.0) + assert median([3.0, 1.0, 2.0]) == 2.0 + clipped = clip_outliers([10.0, 11.0, 9.0, 10.0, 1000.0], 3.0, 1.0) + assert max(clipped) < 20 + assert clip_outliers([], 3.0, 1.0) == [] + with pytest.raises(ValueError, match="empty"): + median([]) + + +@given(st.lists(st.floats(-1e6, 1e6), min_size=1, max_size=50), st.floats(0, 1)) +def test_weighted_quantile_lies_in_the_sample_range(values: list[float], q: float) -> None: + result = weighted_quantile(values, [1.0] * len(values), q) + assert min(values) - 1e-6 <= result <= max(values) + 1e-6 + assert not math.isnan(result) + + +def test_temperature_term_learns_heating_and_cooling_together() -> None: + def temp_of(day: int) -> float: + return 4.0 + (day * 11) % 27 # 4..30 °C: heating and cooling days + + def shape(moment: datetime) -> float: + hdd, cdd = degree_days(temp_of((moment - START) // timedelta(days=1))) + return 400.0 + 25.0 * hdd + 40.0 * cdd + + model = fit_load_model( + history(56, shape), + now=START + timedelta(days=56), + tz=BERLIN, + day_types={}, + temperature=temperatures(56, temp_of), + temperature_configured=True, + ) + term = model.temperature + assert term.active + assert term.heating_w_per_k == pytest.approx(25, rel=0.15) + assert term.cooling_w_per_k == pytest.approx(40, rel=0.15) + + +def test_a_negative_joint_coefficient_falls_back_to_one_regressor() -> None: + temps = [4.0 + (i * 11) % 27 for i in range(20)] + samples = [ + (30.0 * degree_days(t)[0] - 20.0 * degree_days(t)[1], t, 1.0) # cooling lowers the load + for t in temps + ] + term = fit_temperature(samples) + assert term.active + assert term.cooling_w_per_k == 0.0 + assert term.heating_w_per_k > 0 + + +def test_the_forecast_follows_the_wall_clock_over_a_dst_day() -> None: + data = history(42, evening_peak, datetime(2026, 9, 12, tzinfo=BERLIN).astimezone(UTC)) + now = datetime(2026, 10, 24, 12, 0, tzinfo=BERLIN).astimezone(UTC) # day before fall-back + model = fit_load_model(data, now=now, tz=BERLIN, day_types={}) + grid = planning_grid(now, timedelta(hours=48)) + forecast = forecast_load(model, grid, BERLIN, {}) + assert forecast is not None + assert len(forecast) == 48 * 4 # absolute time: the 25-hour day adds no slot + for begin, _, value in forecast.p50.slots(): + local_hour = begin.astimezone(BERLIN).hour + assert value is not None + if 17 <= local_hour < 21: + assert value > 600 # the evening peak stays at local 17-21 h after the switch + elif 1 <= local_hour < 6: + assert value < 400 + + +def test_unmeasured_device_slots_are_left_out_with_enough_clean_days() -> None: + def house(moment: datetime) -> float: + return 500.0 if 12 <= local(moment).hour < 16 else 300.0 # the AC runs at noon + + def ac(moment: datetime, measured_from: int) -> float | None: + if moment < START + timedelta(days=measured_from): + return None # not measured (history rebuilt without the device) + return 200.0 if 12 <= local(moment).hour < 16 else 0.0 + + now = START + timedelta(days=28) + for measured_from, expect_clean in ((14, True), (25, False)): + starts = [START + Q * i for i in range(96 * 28)] + result = base_load( + series([house(t) for t in starts]), + {"ac": series([ac(t, measured_from) for t in starts])}, + ) + assert any(result.unmeasured) + model = fit_load_model( + result.series, now=now, tz=BERLIN, day_types={}, unmeasured=result.unmeasured + ) + noon = model.profiles[DayType.WORKDAY][13 * 4] + if expect_clean: # 10 measured workdays: the unmeasured noons are left out + # (the shrinkage still pulls a little towards the weekend days, which have only + # 4 measured days and keep their unmeasured noons) + assert noon < 320 + assert DayType.WORKDAY not in model.unmeasured_kept + assert DayType.WEEKEND in model.unmeasured_kept + else: # 3 measured days only: kept and flagged + assert noon > 400 + assert DayType.WORKDAY in model.unmeasured_kept diff --git a/tests/forecast/test_load_service.py b/tests/forecast/test_load_service.py new file mode 100644 index 0000000..efadfb5 --- /dev/null +++ b/tests/forecast/test_load_service.py @@ -0,0 +1,333 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The load forecast service: fit from storage, day types, temperature, store, accuracy.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from datetime import UTC, date, datetime, timedelta +from pathlib import Path +from zoneinfo import ZoneInfo + +import pytest + +from suncore.core.forecast import HorizonBucket +from suncore.core.measurements import SITE_HOUSE_W, WEATHER_TEMPERATURE_C, device_power_key +from suncore.core.series import SeriesUnit, TimeRange, TimeSeries +from suncore.forecast.accuracy import load_accuracy +from suncore.forecast.load import ( + LOAD_SOURCE_ID, + DayType, + DayTypeSource, + LoadQuality, + WorkdayState, + parse_workday_sensor, +) +from suncore.forecast.load_service import ( + MAX_CHECKS_PER_RUN, + LoadForecastService, + LoadInputs, +) +from suncore.storage import Database, LearnedParamStore +from suncore.storage.timeseries import TimeseriesStore, open_timeseries + +BERLIN = ZoneInfo("Europe/Berlin") +Q = timedelta(minutes=15) +START = datetime(2026, 9, 7, tzinfo=BERLIN).astimezone(UTC) # a Monday +NOW = START + timedelta(days=14, hours=10, minutes=5) # Monday 21 Sep, 10:05 local + + +class Clock: + def __init__(self, now: datetime) -> None: + self.now = now + + def __call__(self) -> datetime: + return self.now + + +class FakeReader: + def __init__(self) -> None: + self.today: bool | None = None + self.offset = 0 + self.workdays = ["mon", "tue", "wed", "thu", "fri"] + self.answers: dict[date, bool] = {} + self.fail_after: int | None = None + self.checks: list[date] = [] + self.temperature: float | None = None + self.points: list[tuple[datetime, float]] = [] + self.weather_calls = 0 + + def workday_state(self, entity_id: str) -> WorkdayState | None: + assert entity_id == "binary_sensor.workday" + if self.today is None: + return None + attributes = {"days_offset": self.offset, "workdays": self.workdays} + return parse_workday_sensor(entity_id, "on" if self.today else "off", attributes) + + async def check_workday(self, entity_id: str, day: date, *, min_interval_s: float) -> bool: + assert entity_id == "binary_sensor.workday" + assert min_interval_s > 0 + if self.fail_after is not None and len(self.checks) >= self.fail_after: + msg = "rate limited" + raise RuntimeError(msg) + self.checks.append(day) + asked = day + timedelta(days=self.offset) # Home Assistant adds the sensor's offset + return self.answers.get(asked, asked.weekday() < 5) + + def outside_temperature(self, entity_id: str) -> float | None: + assert entity_id == "weather.home" + return self.temperature + + async def hourly_temperatures( + self, entity_id: str, *, min_interval_s: float + ) -> list[tuple[datetime, float]]: + assert entity_id == "weather.home" + assert min_interval_s > 0 + self.weather_calls += 1 + return self.points + + +@pytest.fixture +async def store(tmp_path: Path) -> AsyncIterator[TimeseriesStore]: + db = await open_timeseries(tmp_path / "timeseries.db") + yield TimeseriesStore(db) + await db.close() + + +@pytest.fixture +async def learned(tmp_path: Path) -> AsyncIterator[LearnedParamStore]: + db = await Database.open(tmp_path / "suncore.db") + yield LearnedParamStore(db, clock=lambda: NOW) + await db.close() + + +def ts(values: list[float | None], start: datetime = START) -> TimeSeries: + return TimeSeries(start=start, step=Q, values=tuple(values), unit=SeriesUnit.W) + + +async def fill_history(store: TimeseriesStore, days: int = 14) -> None: + """House 500 W, of which the AC draws 200 W from 12:00 to 16:00 local.""" + house: list[float | None] = [] + ac: list[float | None] = [] + for i in range(96 * days): + hour = (START + Q * i).astimezone(BERLIN).hour + draw = 200.0 if 12 <= hour < 16 else 0.0 + house.append(300.0 + draw) + ac.append(draw) + await store.store_quarter_hours(SITE_HOUSE_W, ts(house)) + await store.store_quarter_hours(device_power_key("ac"), ts(ac)) + + +def service( + store: TimeseriesStore | None, + learned: LearnedParamStore | None, + clock: Clock, + reader: FakeReader | None = None, + **inputs: object, +) -> LoadForecastService: + return LoadForecastService( + store, + learned, + clock=clock, + tz=lambda: BERLIN, + inputs=LoadInputs(**inputs), # type: ignore[arg-type] + reader=reader, + ) + + +async def test_refresh_forecasts_the_base_load_without_devices( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store) + clock = Clock(NOW) + svc = service(store, learned, clock, device_ids=("ac",)) + forecast = await svc.refresh() + assert forecast is not None + assert len(forecast) == 48 * 4 + 1 # the current slot and 48 h + assert all(v == pytest.approx(300, abs=1) for v in forecast.p50.values) + assert set(svc.device_histories) == {"ac"} + issues = await store.forecast_issues(LOAD_SOURCE_ID, "load") + assert issues == [NOW] + clock.now = NOW + timedelta(minutes=30) + await svc.refresh() + assert len(await store.forecast_issues(LOAD_SOURCE_ID, "load")) == 1 # hourly + clock.now = NOW + timedelta(hours=1) + await svc.refresh() + assert len(await store.forecast_issues(LOAD_SOURCE_ID, "load")) == 2 + view = svc.view(24) + assert view.quality is LoadQuality.OK + assert view.days_of_history == 14 + assert len(view.p50_w) == 96 + assert view.controlled_devices == ["ac"] + assert view.devices_unmeasured == {} + assert view.temperature.reason == "not_configured" + + +async def test_unmeasured_device_is_counted_not_guessed( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store) + svc = service(store, learned, Clock(NOW), device_ids=("ac", "pool")) + await svc.refresh() + view = svc.view() + assert view.devices_unmeasured["pool"] > 0 + assert "ac" not in view.devices_unmeasured + + +async def test_without_history_there_is_no_forecast( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + svc = service(store, learned, Clock(NOW)) + assert await svc.refresh() is None + view = svc.view() + assert view.quality is LoadQuality.NO_HISTORY + assert view.p50_w == [] + assert await store.forecast_issues(LOAD_SOURCE_ID, "load") == [] + + +async def test_no_store_means_no_forecast() -> None: + svc = service(None, None, Clock(NOW)) + assert await svc.refresh() is None + assert await svc.refit() is None + assert await svc.nightly() == [] + + +async def test_cold_start(store: TimeseriesStore, learned: LearnedParamStore) -> None: + await store.store_quarter_hours( + SITE_HOUSE_W, ts([400.0] * 96 * 2, NOW.replace(minute=0) - timedelta(days=2)) + ) + svc = service(store, learned, Clock(NOW)) + forecast = await svc.refresh() + assert forecast is not None + assert svc.view().quality is LoadQuality.COLD_START + assert forecast.p90.values[0] == pytest.approx(800) + + +async def test_day_types_from_the_workday_sensor_and_check_date( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store) + reader = FakeReader() + reader.today = False # Monday 21 Sep: a holiday + tomorrow = date(2026, 9, 22) + reader.answers[tomorrow] = False + svc = service(store, learned, Clock(NOW), reader, workday_entity="binary_sensor.workday") + await svc.refresh() + today = date(2026, 9, 21) + assert svc.day_info(today, today).day_type is DayType.HOLIDAY + assert svc.day_info(today, today).source is DayTypeSource.WORKDAY_SENSOR + assert svc.day_info(tomorrow, today).day_type is DayType.HOLIDAY + assert svc.day_info(tomorrow, today).source is DayTypeSource.CHECK_DATE + assert reader.checks[0] == tomorrow # forecast days first + assert len(reader.checks) == MAX_CHECKS_PER_RUN + view = svc.view(48) + assert [d.day_type for d in view.days][:2] == [DayType.HOLIDAY, DayType.HOLIDAY] + # the answers are cached: a new service asks only for dates it has not seen + again = FakeReader() + svc2 = service(store, learned, Clock(NOW), again, workday_entity="binary_sensor.workday") + await svc2.refresh() + assert tomorrow not in again.checks + assert svc2.day_info(tomorrow, today).day_type is DayType.HOLIDAY + + +async def test_a_failing_check_date_falls_back_to_the_weekday_rule( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store) + reader = FakeReader() + reader.fail_after = 0 + svc = service(store, learned, Clock(NOW), reader, workday_entity="binary_sensor.workday") + await svc.refresh() + tomorrow, today = date(2026, 9, 22), date(2026, 9, 21) + info = svc.day_info(tomorrow, today) + assert info.source is DayTypeSource.WEEKDAY_RULE + assert info.day_type is DayType.WORKDAY + assert svc.day_info(today, today).source is DayTypeSource.WEEKDAY_RULE # state unknown + + +async def test_a_shifted_workday_sensor_is_compensated( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store) + reader = FakeReader() + reader.offset = 1 # the sensor describes tomorrow + reader.today = False # so tomorrow (Tue 22 Sep) is a holiday + friday = date(2026, 9, 25) + reader.answers[friday] = False # a holiday on Friday + svc = service(store, learned, Clock(NOW), reader, workday_entity="binary_sensor.workday") + await svc.refresh() + today, tomorrow = date(2026, 9, 21), date(2026, 9, 22) + info = svc.day_info(tomorrow, today) + assert (info.day_type, info.source) == (DayType.HOLIDAY, DayTypeSource.WORKDAY_SENSOR) + # today is asked as check_date(today - 1): Home Assistant answers for today + assert date(2026, 9, 20) in reader.checks + assert svc.day_info(today, today).day_type is DayType.WORKDAY + assert svc.day_info(today, today).source is DayTypeSource.CHECK_DATE + assert date(2026, 9, 24) in reader.checks or len(reader.checks) == MAX_CHECKS_PER_RUN + svc._cache.clear() + await svc._resolve([friday], today) + assert reader.checks[-1] == date(2026, 9, 24) # Friday - 1 + assert svc.day_info(friday, today).day_type is DayType.HOLIDAY + + +async def test_a_changed_workday_configuration_drops_the_cache( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store) + reader = FakeReader() + reader.today = True + tomorrow, today = date(2026, 9, 22), date(2026, 9, 21) + reader.answers[tomorrow] = False + svc = service(store, learned, Clock(NOW), reader, workday_entity="binary_sensor.workday") + await svc.refresh() + assert svc.day_info(tomorrow, today).day_type is DayType.HOLIDAY + reader.workdays = ["mon", "tue", "wed", "thu", "fri", "sat"] # reconfigured + reader.answers.clear() + assert svc.day_info(tomorrow, today).source is DayTypeSource.WEEKDAY_RULE # stale answer + again = FakeReader() + again.today, again.workdays = True, reader.workdays + svc2 = service(store, learned, Clock(NOW), again, workday_entity="binary_sensor.workday") + await svc2.refresh() + assert tomorrow in again.checks # asked again after the reset + assert svc2.day_info(tomorrow, today).day_type is DayType.WORKDAY + + +async def test_temperature_is_recorded_and_the_forecast_days_get_a_mean( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store) + reader = FakeReader() + reader.temperature = 12.5 + reader.points = [(NOW + timedelta(hours=h), 8.0) for h in range(48)] + clock = Clock(NOW) + svc = service(store, learned, clock, reader, weather_entity="weather.home") + await svc.refresh() + recorded = await store.read_raw(WEATHER_TEMPERATURE_C, TimeRange(start=NOW - Q, end=NOW + Q)) + assert recorded == [(NOW, 12.5)] + temps = svc.daily_temperatures(NOW) + assert temps[date(2026, 9, 22)] == pytest.approx(8.0) + assert date(2026, 9, 21) not in temps # 14 hours of today known: no offset for a partial day + reader.points = [(NOW - timedelta(hours=h), 12.0) for h in range(1, 11)] + reader.points + svc._weather_points = reader.points + assert date(2026, 9, 21) not in svc.daily_temperatures(NOW) # past forecast hours ignored + view = svc.view() + assert view.temperature.reason == "too_few_days" + assert view.days[1].temperature_c == pytest.approx(8.0) + clock.now = NOW + timedelta(minutes=15) + await svc.refresh() + assert reader.weather_calls == 1 # hourly + + +async def test_nightly_scores_the_load_forecast( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store, days=16) # measured until 23 Sep 00:00 local + clock = Clock(NOW) + svc = service(store, learned, clock, device_ids=("ac",)) + await svc.refresh() # issued Monday 10:05, forecasts 300 W + clock.now = START + timedelta(days=16, hours=3) + records = await svc.nightly() + by = {r.horizon: r for r in records} + assert by[HorizonBucket.H0_6].n > 0 + assert by[HorizonBucket.H0_6].mae_w == pytest.approx(0, abs=1) + stored = await load_accuracy(learned, [LOAD_SOURCE_ID]) + assert LOAD_SOURCE_ID in stored diff --git a/tests/forecast/test_risk.py b/tests/forecast/test_risk.py new file mode 100644 index 0000000..0922257 --- /dev/null +++ b/tests/forecast/test_risk.py @@ -0,0 +1,497 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Risk model: windows, net energy quantiles, morning buffer and evening target (WP7).""" + +from __future__ import annotations + +import itertools +import math +from datetime import UTC, date, datetime, timedelta +from zoneinfo import ZoneInfo + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from suncore.core.battery import ( + BatteryModel, + BatterySettings, + BatteryStrategy, + RiskLevel, + effective_reserve_pct, +) +from suncore.core.series import QuantileSeries, SeriesUnit, TimeSeries, planning_grid +from suncore.core.site import BatteryConfig, EntityRef +from suncore.forecast.risk import ( + EnergyQuantiles, + ErrorModel, + ErrorSample, + NetWindow, + RiskWindow, + coverage, + day_windows, + daylight_spans, + evening_target_pct, + measured_energy, + merge_samples, + morning_buffer_pct, + net_energy_quantile, + samples_view, + source_net_quantile, + window_energy, +) + +BERLIN = ZoneInfo("Europe/Berlin") +NOW = datetime(2026, 6, 22, 8, 0, tzinfo=BERLIN).astimezone(UTC) # Monday 08:00 local +BATTERY = BatteryModel(capacity_wh=10_000, max_charge_w=5_000, max_discharge_w=5_000) +LEVELS = (RiskLevel.CAUTIOUS, RiskLevel.BALANCED, RiskLevel.BOLD) + + +def bell(moment: datetime, peak_w: float) -> float: + """PV of a horizontal plant: 06:00-21:00 local, a sine bell.""" + t = moment.astimezone(BERLIN) + hour = t.hour + t.minute / 60 + 0.125 + if not 6 <= hour < 21: + return 0.0 + return peak_w * math.sin(math.pi * (hour - 6) / 15) + + +def pv_forecast( + peaks: tuple[float, ...] = (6000.0, 6000.0, 6000.0), + low: float = 0.85, + high: float = 1.1, + now: datetime = NOW, +) -> QuantileSeries: + grid = planning_grid(now, timedelta(hours=48)) + first = now.astimezone(BERLIN).date() + mids = [bell(t, peaks[(t.astimezone(BERLIN).date() - first).days]) for t in grid.starts()] + + def series(values: list[float]) -> TimeSeries: + return TimeSeries(start=grid.start, step=grid.step, values=tuple(values), unit=SeriesUnit.W) + + return QuantileSeries( + p10=series([v * low for v in mids]), + p50=series(mids), + p90=series([v * high for v in mids]), + ) + + +def load_forecast(level: float = 400.0, now: datetime = NOW, high: float = 1.3) -> QuantileSeries: + grid = planning_grid(now, timedelta(hours=48)) + + def series(value: float) -> TimeSeries: + return TimeSeries( + start=grid.start, step=grid.step, values=(value,) * grid.count, unit=SeriesUnit.W + ) + + return QuantileSeries(p10=series(level * 0.8), p50=series(level), p90=series(level * high)) + + +def targets( + pv: QuantileSeries, + risk: RiskLevel = RiskLevel.CAUTIOUS, + *, + model: ErrorModel | None = None, + user: float | None = None, + manual: float | None = None, + strategy: BatteryStrategy = BatteryStrategy.SELF_CONSUMPTION, + battery: BatteryModel = BATTERY, + load: QuantileSeries | None = None, + now: datetime = NOW, + site: BatteryConfig | None = None, +) -> tuple[float, float, str, str]: + windows = day_windows(now, pv, load or load_forecast(now=now), BERLIN) + config = BatterySettings(risk=risk, morning_buffer_pct=manual, strategy=strategy) + evening = evening_target_pct( + battery, config, windows, user_target_pct=user, site_battery=site, model=model + ) + buffer = morning_buffer_pct( + battery, config, windows, evening.soc_pct, site_battery=site, model=model + ) + return ( + buffer.soc_pct, + evening.soc_pct, + str(buffer.reason.args["cause"]), + str(evening.reason.args["cause"]), + ) + + +# --- windows --------------------------------------------------------------------------------- + + +def test_windows_are_anchored_on_the_next_sunset() -> None: + windows = day_windows(NOW, pv_forecast(), load_forecast(), BERLIN) + rest, night = windows[RiskWindow.REST_OF_TODAY], windows[RiskWindow.NIGHT] + nxt, bridge = windows[RiskWindow.NEXT_DAY], windows[RiskWindow.BRIDGE] + assert rest.start == NOW + assert windows[RiskWindow.REST_OF_NIGHT].start == windows[RiskWindow.REST_OF_NIGHT].end + assert rest.end == night.start == bridge.start + both = windows[RiskWindow.DAY_AND_NIGHT] + assert (both.start, both.end) == (rest.start, night.end) + assert night.end == nxt.start + assert nxt.end == bridge.end + assert rest.end.astimezone(BERLIN).hour in (20, 21) + assert nxt.start.astimezone(BERLIN).date() == date(2026, 6, 23) + assert night.weather == "night" + assert night.pv.p50 == 0 + assert night.load.p50 == pytest.approx(400 * (night.end - night.start) / timedelta(hours=1)) + assert bridge.load.p50 == pytest.approx(night.load.p50 + nxt.load.p50) + + +def test_after_sunset_the_night_starts_now() -> None: + late = datetime(2026, 6, 22, 23, 0, tzinfo=BERLIN).astimezone(UTC) + pv = pv_forecast(now=late) + windows = day_windows(late, pv, load_forecast(now=late), BERLIN) + assert windows[RiskWindow.REST_OF_TODAY].start == windows[RiskWindow.REST_OF_TODAY].end + assert windows[RiskWindow.NIGHT].start == late + both, nxt = windows[RiskWindow.DAY_AND_NIGHT], windows[RiskWindow.NEXT_DAY] + assert both.start == nxt.start # the next daytime and the night after it + assert both.end > nxt.end + + +def test_before_sunrise_the_rest_of_the_night_is_its_own_window() -> None: + early = datetime(2026, 6, 22, 3, 0, tzinfo=BERLIN).astimezone(UTC) + windows = day_windows(early, pv_forecast(now=early), load_forecast(now=early), BERLIN) + rest_night, rest = windows[RiskWindow.REST_OF_NIGHT], windows[RiskWindow.REST_OF_TODAY] + assert rest_night.start == early + assert rest_night.end == rest.start + assert rest.start.astimezone(BERLIN).hour == 6 # today's sunrise, not 03:00 + assert rest_night.weather == "night" + assert rest_night.pv.p50 == 0 + assert windows[RiskWindow.DAY_AND_NIGHT].start == early + + +def test_without_pv_the_fallback_times_apply() -> None: + windows = day_windows(NOW, None, load_forecast(), BERLIN) + assert windows[RiskWindow.REST_OF_TODAY].end.astimezone(BERLIN).hour == 19 + assert windows[RiskWindow.NEXT_DAY].start.astimezone(BERLIN).hour == 7 + assert windows[RiskWindow.NEXT_DAY].weather == "overcast" + assert daylight_spans(None, BERLIN) == {} + + +def test_window_energy_counts_partial_slots() -> None: + load = load_forecast() + start = NOW + timedelta(minutes=5) + energy = window_energy(load, start, start + timedelta(hours=1)) + assert energy.p50 == pytest.approx(400) + assert window_energy(None, NOW, NOW + timedelta(hours=1)) == EnergyQuantiles.zero() + + +# --- distribution ---------------------------------------------------------------------------- + + +def window(pv: tuple[float, float, float], load: tuple[float, float, float]) -> NetWindow: + return NetWindow( + kind=RiskWindow.NEXT_DAY, + start=NOW, + end=NOW + timedelta(hours=10), + pv=EnergyQuantiles(*pv), + load=EnergyQuantiles(*load), + weather="mixed", + ) + + +def test_source_quantiles_formula() -> None: + w = window((6000, 10000, 12000), (4000, 5000, 7000)) + assert source_net_quantile(w, 0.5) == 5000 + z90 = 1.2815515655446004 + sigma = math.hypot(4000 / z90, 2000 / z90) # PV shortfall and load excess + assert source_net_quantile(w, 0.1) == pytest.approx(5000 - z90 * sigma) + assert source_net_quantile(w, 0.9) > 5000 + with pytest.raises(ValueError, match="inside"): + source_net_quantile(w, 1.0) + result = net_energy_quantile(w, RiskLevel.CAUTIOUS) + assert result.source == "source_quantiles" + assert result.samples == 0 + + +def samples( + n_days: int, rel: float, weather: str = "mixed", start: date = date(2026, 5, 1) +) -> list[ErrorSample]: + return [ + ErrorSample( + day=start + timedelta(days=i), + window=RiskWindow.NEXT_DAY, + weather=weather, # type: ignore[arg-type] + error_wh=rel * 10000 + (i % 5 - 2) * 100, + scale_wh=10000, + ) + for i in range(n_days) + ] + + +def test_learned_distribution_after_min_history_days() -> None: + w = window((8000, 10000, 11000), (4000, 5000, 6000)) + young = ErrorModel(tuple(samples(29, -0.3))) + assert net_energy_quantile(w, RiskLevel.CAUTIOUS, young).source == "source_quantiles" + old = ErrorModel(tuple(samples(30, -0.3))) + learned = net_energy_quantile(w, RiskLevel.CAUTIOUS, old) + assert learned.source == "learned" + assert not learned.pooled + assert learned.value_wh == pytest.approx(5000 - 0.32 * 15000, abs=200) + + +def test_a_thin_class_falls_back_to_the_window_pool() -> None: + w = window((8000, 10000, 11000), (4000, 5000, 6000)) + model = ErrorModel( + tuple(samples(25, -0.1, "clear") + samples(5, -0.5, "mixed", date(2026, 7, 1))) + ) + result = net_energy_quantile(w, RiskLevel.BOLD, model) + assert result.source == "learned" + assert result.pooled + assert result.samples == 30 + + +def test_samples_round_trip_and_merge() -> None: + a = samples(3, 0.1) + assert [ErrorSample.from_json(s.to_json()) for s in a] == a + with pytest.raises(ValueError, match="weather"): + ErrorSample.from_json({**a[0].to_json(), "weather": "fog"}) + newer = ErrorSample(a[0].day, a[0].window, "clear", 1.0, 2.0) + merged = merge_samples(a, [newer], keep_days=1, today=a[0].day + timedelta(days=2)) + assert [s.day for s in merged] == [a[1].day, a[2].day] + merged = merge_samples(a, [newer], keep_days=10, today=a[0].day) + assert merged[0] == newer + assert samples_view(a) == {"next_day/mixed": 3} + + +def test_measured_energy_and_coverage() -> None: + series = TimeSeries( + start=NOW, step=timedelta(minutes=15), values=(400.0, None, 400.0, 400.0), unit=SeriesUnit.W + ) + end = NOW + timedelta(hours=1) + assert coverage(series, NOW, end) == 0.75 + assert measured_energy(series, NOW, end) == pytest.approx(400) + assert coverage(series, end, end + timedelta(hours=1)) == 0.0 + assert coverage(series, NOW, NOW + timedelta(hours=2)) == 3 / 8 # beyond the series: unknown + + +# --- properties ---------------------------------------------------------------------------------- + +peaks = st.floats(0, 9000) +spread = st.floats(0.3, 0.99) +"""A band (P10 below P50): a source without any band uses the default band (a step).""" + + +@settings(max_examples=40, deadline=None) +@given(peaks, peaks, spread, st.floats(100, 1500), st.one_of(st.none(), st.floats(0, 100))) +def test_monotone_in_the_risk_level( + today: float, tomorrow: float, low: float, load: float, user: float | None +) -> None: + pv = pv_forecast((today, tomorrow, tomorrow), low=low) + results = [targets(pv, level, user=user, load=load_forecast(load)) for level in LEVELS] + for cautious, bold in itertools.pairwise(results): + assert cautious[0] >= bold[0] - 1e-9 # buffer + assert cautious[1] >= bold[1] - 1e-9 # evening target + + +@settings(max_examples=40, deadline=None) +@given(peaks, peaks, spread, st.floats(0.0, 0.5), st.sampled_from(LEVELS)) +def test_monotone_in_the_uncertainty( + today: float, tomorrow: float, low: float, wider: float, level: RiskLevel +) -> None: + narrow = targets(pv_forecast((today, tomorrow, tomorrow), low=low), level) + wide = targets(pv_forecast((today, tomorrow, tomorrow), low=max(0.0, low - wider)), level) + assert wide[0] >= narrow[0] - 1e-9 + assert wide[1] >= narrow[1] - 1e-9 + + +@settings(max_examples=60, deadline=None) +@given( + peaks, + peaks, + st.sampled_from(LEVELS), + st.one_of(st.none(), st.floats(0, 100)), + st.one_of(st.none(), st.floats(0, 100)), + st.floats(60, 100), + st.sampled_from(list(BatteryStrategy)), +) +def test_limits_always_win( # noqa: PLR0917 - hypothesis passes the drawn values positionally + today: float, + tomorrow: float, + level: RiskLevel, + user: float | None, + manual: float | None, + max_soc: float, + strategy: BatteryStrategy, +) -> None: + battery = BATTERY.model_copy(update={"max_soc_pct": max_soc}) + buffer, evening, _, _ = targets( + pv_forecast((today, tomorrow, tomorrow)), + level, + user=user, + manual=manual, + strategy=strategy, + battery=battery, + ) + for value in (buffer, evening): + assert battery.min_soc_pct - 1e-9 <= value <= max_soc + 1e-9 + assert buffer >= min(max_soc, manual or 0) - 1e-9 + assert evening >= min(max_soc, user or 0) - 1e-9 + + +def test_prefer_full_means_max_soc() -> None: + battery = BATTERY.model_copy(update={"max_soc_pct": 90}) + buffer, evening, cause, evening_cause = targets( + pv_forecast(), strategy=BatteryStrategy.PREFER_FULL, battery=battery + ) + assert (buffer, evening, cause, evening_cause) == (90, 90, "prefer_full", "prefer_full") + + +def test_reasons_carry_the_value_and_the_source() -> None: + windows = day_windows(NOW, pv_forecast(), load_forecast(), BERLIN) + config = BatterySettings() + evening = evening_target_pct(BATTERY, config, windows) + assert evening.reason.key == "planner.battery.evening_target" + assert evening.reason.args["soc_pct"] == round(evening.soc_pct, 1) + assert evening.reason.args["source"] == "source_quantiles" + assert evening.reason.args["risk"] == "cautious" + buffer = morning_buffer_pct(BATTERY, config, windows, evening.soc_pct) + assert buffer.reason.key == "planner.battery.morning_buffer" + + +# --- scenarios ---------------------------------------------------------------------------------- + + +def test_scenario_sunny_day_gives_a_low_buffer() -> None: + buffer, _, cause, _ = targets(pv_forecast((7000, 7000, 7000), low=0.9)) + assert buffer == BATTERY.min_soc_pct + assert cause == "forecast_good" + + +def test_scenario_mixed_clouds_give_a_higher_buffer() -> None: + sunny = targets(pv_forecast((7000, 7000, 7000), low=0.9))[0] + mixed, _, cause, _ = targets(pv_forecast((2500, 2500, 2500), low=0.4)) + assert mixed > sunny + assert cause in ("forecast_uncertain", "forecast_poor") + + +def test_scenario_two_poor_days_raise_the_evening_target_to_max() -> None: + good = targets(pv_forecast((7000, 7000, 7000)))[1] + _, poor, _, cause = targets(pv_forecast((800, 600, 600), low=0.5), load=load_forecast(600)) + assert poor == BATTERY.max_soc_pct + assert cause == "max_soc" + assert good < poor + without_bridge = evening_target_pct( + BATTERY, + BatterySettings(bridge_next_day=False), + day_windows(NOW, pv_forecast((800, 600, 600), low=0.5), load_forecast(600), BERLIN), + ) + assert without_bridge.soc_pct < poor # only the night + + +def test_scenario_a_too_optimistic_forecast_raises_the_target_after_30_days() -> None: + # a poor next day the source calls narrow (P10 = 0.9 x P50): the bridge decides the target + pv = pv_forecast((800, 800, 800), low=0.9) + load = load_forecast(400) + windows = day_windows(NOW, pv, load, BERLIN) + # the PV forecast was 50 % too high for weeks: every daytime net came in lower + bridge = windows[RiskWindow.BRIDGE] + history = [ + ErrorSample( + day=date(2026, 5, 1) + timedelta(days=i), + window=kind, + weather=windows[kind].weather, + error_wh=-0.5 * windows[kind].pv.p50 + (i % 3 - 1) * 50, + scale_wh=windows[kind].scale_wh, + ) + for i in range(40) + for kind in (RiskWindow.NIGHT, RiskWindow.BRIDGE, RiskWindow.REST_OF_TODAY) + ] + before = targets(pv, load=load, model=ErrorModel(tuple(history[: 29 * 3]))) + after = targets(pv, load=load, model=ErrorModel(tuple(history))) + assert bridge.pv.p50 > 0 + assert before[3] == "next_day_deficit" + assert after[1] > before[1] + + +# --- review findings: reserve floor, default band, learned path, load band ------------------- + + +def site_battery(min_soc: float | None) -> BatteryConfig: + return BatteryConfig( + power=EntityRef(entity_id="sensor.bat_power"), + soc=EntityRef(entity_id="sensor.bat_soc"), + min_soc_pct=min_soc, + ) + + +def test_effective_reserve_takes_the_higher_floor() -> None: + assert effective_reserve_pct(None, BATTERY) == 15 # reserve 10 + drift 5 + assert effective_reserve_pct(site_battery(None), BATTERY) == 15 + assert effective_reserve_pct(site_battery(30), BATTERY) == 30 + assert effective_reserve_pct(site_battery(5), BATTERY) == 15 + + +@settings(max_examples=40, deadline=None) +@given(peaks, peaks, st.sampled_from(LEVELS), st.floats(0, 59)) +def test_never_below_the_site_min_soc( + today: float, tomorrow: float, level: RiskLevel, site_min: float +) -> None: + battery = BATTERY.model_copy(update={"max_soc_pct": 60}) + buffer, evening, _, _ = targets( + pv_forecast((today, tomorrow, tomorrow)), + level, + battery=battery, + site=site_battery(site_min), + ) + assert buffer >= site_min - 1e-9 + assert evening >= site_min - 1e-9 + + +def test_a_source_without_a_band_gets_the_default_band() -> None: + flat = pv_forecast((1500, 1500, 1500), low=1.0, high=1.0) + windows = day_windows(NOW, flat, load_forecast(), BERLIN) + day = windows[RiskWindow.REST_OF_TODAY] + cautious = net_energy_quantile(day, RiskLevel.CAUTIOUS) + bold = net_energy_quantile(day, RiskLevel.BOLD) + assert cautious.source == "default_band" + assert cautious.value_wh < bold.value_wh + assert targets(flat, RiskLevel.CAUTIOUS)[0] > targets(flat, RiskLevel.BOLD)[0] + reason = evening_target_pct(BATTERY, BatterySettings(), windows).reason + assert reason.args["source"] == "default_band" + # the night has no PV: its load band is the source's own + assert net_energy_quantile(windows[RiskWindow.NIGHT], 0.1).source == "source_quantiles" + + +def learned_model(seed: int) -> ErrorModel: + rng = __import__("random").Random(seed) + samples = [ + ErrorSample( + day=date(2026, 4, 1) + timedelta(days=i), + window=kind, + weather=weather, # type: ignore[arg-type] + error_wh=rng.gauss(-0.1, 0.2) * 10_000, + scale_wh=10_000, + ) + for i in range(40) + for kind in RiskWindow + for weather in ("clear", "mixed", "overcast", "night") + ] + return ErrorModel(tuple(samples)) + + +@settings(max_examples=25, deadline=None) +@given(peaks, peaks, st.integers(0, 1000)) +def test_monotone_in_the_risk_level_on_the_learned_path( + today: float, tomorrow: float, seed: int +) -> None: + model = learned_model(seed) + pv = pv_forecast((today, tomorrow, tomorrow)) + results = [targets(pv, level, model=model) for level in LEVELS] + for cautious, bold in itertools.pairwise(results): + assert cautious[0] >= bold[0] - 1e-9 + assert cautious[1] >= bold[1] - 1e-9 + windows = day_windows(NOW, pv, load_forecast(), BERLIN) + assert net_energy_quantile(windows[RiskWindow.NIGHT], 0.1, model).source == "learned" + + +@settings(max_examples=30, deadline=None) +@given(peaks, peaks, st.floats(1.0, 1.5), st.floats(0.0, 1.0), st.sampled_from(LEVELS)) +def test_monotone_in_the_load_band( + today: float, tomorrow: float, high: float, wider: float, level: RiskLevel +) -> None: + pv = pv_forecast((today, tomorrow, tomorrow)) + narrow = targets(pv, level, load=load_forecast(high=high)) + wide = targets(pv, level, load=load_forecast(high=high + wider)) + assert wide[0] >= narrow[0] - 1e-9 + assert wide[1] >= narrow[1] - 1e-9 diff --git a/tests/forecast/test_risk_learning.py b/tests/forecast/test_risk_learning.py new file mode 100644 index 0000000..475ebcf --- /dev/null +++ b/tests/forecast/test_risk_learning.py @@ -0,0 +1,302 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Nightly learning of the net energy errors and the risk read model (WP7).""" + +from __future__ import annotations + +import math +from collections.abc import AsyncIterator +from datetime import UTC, date, datetime, timedelta +from pathlib import Path +from zoneinfo import ZoneInfo + +import pytest + +from suncore.core.battery import BatteryModel, BatterySettings +from suncore.core.measurements import SITE_PV_W +from suncore.core.series import QuantileSeries, SeriesUnit, TimeRange, TimeSeries +from suncore.forecast.load import LOAD_SOURCE_ID +from suncore.forecast.risk import ErrorModel, RiskWindow +from suncore.forecast.risk_learning import ( + NS_NET_ERROR, + SAMPLES_KEY, + learn_net_errors, + load_error_model, + risk_view, +) +from suncore.storage import Database, LearnedParamStore +from suncore.storage.learned import NS_NET_ERROR as STORAGE_NS +from suncore.storage.timeseries import TimeseriesStore, open_timeseries + +BERLIN = ZoneInfo("Europe/Berlin") +Q = timedelta(minutes=15) +FIRST = date(2026, 6, 1) +START = datetime(2026, 6, 1, tzinfo=BERLIN).astimezone(UTC) +DAYS = 6 +NOW = datetime(2026, 6, 7, 3, 30, tzinfo=BERLIN).astimezone(UTC) +BATTERY = BatteryModel(capacity_wh=10_000, max_charge_w=5_000, max_discharge_w=5_000) + + +def pv_at(moment: datetime, peak: float) -> float: + t = moment.astimezone(BERLIN) + hour = t.hour + t.minute / 60 + 0.125 + return peak * math.sin(math.pi * (hour - 6) / 15) if 6 <= hour < 21 else 0.0 + + +def series(values: list[float | None], start: datetime = START) -> TimeSeries: + return TimeSeries(start=start, step=Q, values=tuple(values), unit=SeriesUnit.W) + + +@pytest.fixture +async def store(tmp_path: Path) -> AsyncIterator[TimeseriesStore]: + db = await open_timeseries(tmp_path / "timeseries.db") + yield TimeseriesStore(db) + await db.close() + + +@pytest.fixture +async def learned(tmp_path: Path) -> AsyncIterator[LearnedParamStore]: + db = await Database.open(tmp_path / "suncore.db") + yield LearnedParamStore(db, clock=lambda: NOW) + await db.close() + + +async def fill(store: TimeseriesStore, *, optimistic: float = 1.0) -> None: + """Measured PV (peak 4 kW) and a 400 W base load; forecasts issued daily at 00:00 and + 17:00 local saying ``optimistic`` times the PV and the same load.""" + count = 96 * DAYS + starts = [START + Q * i for i in range(count)] + await store.store_quarter_hours(SITE_PV_W, series([pv_at(t, 4000) for t in starts])) + for day in range(DAYS): + for hour in (0, 17): + issued = datetime.combine( + FIRST + timedelta(days=day), datetime.min.time(), tzinfo=BERLIN + ).astimezone(UTC) + timedelta(hours=hour) + grid = [issued + Q * i for i in range(48 * 4)] + pv = series([pv_at(t, 4000 * optimistic) for t in grid], issued) + await store.store_forecast( + "blended", "pv", issued, QuantileSeries(p10=pv, p50=pv, p90=pv) + ) + load = series([400.0] * len(grid), issued) + await store.store_forecast( + LOAD_SOURCE_ID, "load", issued, QuantileSeries.deterministic(load) + ) + + +async def flat_base(window: TimeRange) -> TimeSeries: + count = (window.end - window.start) // Q + return series([400.0] * count, window.start) + + +async def test_learning_measures_the_net_error_per_window( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill(store, optimistic=1.3) + model = await learn_net_errors( + store, learned, NOW, tz=BERLIN, plant=None, base_reader=flat_base, backfill_days=10 + ) + days = {s.day for s in model.samples} + assert min(days) == FIRST # forecasts exist from 1 June on + assert max(days) == date(2026, 6, 5) # today - 2: the bridge of 5 June ended yesterday + by_kind = {s.window: s for s in model.samples if s.day == date(2026, 6, 2)} + assert set(by_kind) == set(RiskWindow) - {RiskWindow.REST_OF_NIGHT} # never sampled + # the PV forecast was 30 % too high: daytime windows come in lower, the night is exact + assert by_kind[RiskWindow.NEXT_DAY].error_wh < 0 + assert by_kind[RiskWindow.REST_OF_TODAY].error_wh < 0 + assert by_kind[RiskWindow.NIGHT].error_wh == pytest.approx(0, abs=1) + # stored and idempotent + stored = await load_error_model(learned) + assert stored.learned_until == date(2026, 6, 5) + key = [(s.day, s.window, s.weather) for s in model.samples] + assert [(s.day, s.window, s.weather) for s in stored.model.samples] == key + for a, b in zip(stored.model.samples, model.samples, strict=True): + assert a.error_wh == pytest.approx(b.error_wh, abs=0.001) # stored rounded to mWh + again = await learn_net_errors( + store, learned, NOW, tz=BERLIN, plant=None, base_reader=flat_base, backfill_days=10 + ) + assert [(s.day, s.window, s.weather) for s in again.samples] == key + assert STORAGE_NS == NS_NET_ERROR + + +async def test_no_load_forecast_means_no_sample( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await store.store_quarter_hours(SITE_PV_W, series([1.0] * 96 * DAYS)) + model = await learn_net_errors( + store, learned, NOW, tz=BERLIN, plant=None, base_reader=flat_base, backfill_days=10 + ) + assert model.samples == () + + +async def test_unreadable_samples_start_again(learned: LearnedParamStore) -> None: + await learned.put(NS_NET_ERROR, SAMPLES_KEY, {"samples": "x"}) + assert (await load_error_model(learned)).model == ErrorModel() + await learned.put(NS_NET_ERROR, SAMPLES_KEY, {"samples": []}, schema_version=99) + assert (await load_error_model(learned)).learned_until is None + + +# --- the read model ---------------------------------------------------------------------------- + + +def forecast(level: float, now: datetime) -> QuantileSeries: + s = series([level] * 192, now) + return QuantileSeries(p10=s, p50=s, p90=s) + + +def test_view_without_a_load_forecast() -> None: + view = risk_view( + NOW, + pv=None, + load=None, + settings=BatterySettings(), + user_target_pct=None, + model=ErrorModel(), + min_history_days=30, + tz=BERLIN, + plant=None, + ) + assert view.status == "no_forecast" + assert view.windows == [] + + +def test_view_without_a_battery_model_shows_the_windows() -> None: + view = risk_view( + NOW, + pv=None, + load=forecast(400, NOW), + settings=BatterySettings(), + user_target_pct=None, + model=ErrorModel(), + min_history_days=30, + tz=BERLIN, + plant=None, + ) + assert view.status == "no_battery_model" + assert [w.window for w in view.windows] == list(RiskWindow) + assert view.morning_buffer is None + assert not view.pv_forecast + assert view.distribution_source == "source_quantiles" + + +def test_view_with_a_battery_model() -> None: + view = risk_view( + NOW, + pv=None, + load=forecast(400, NOW), + settings=BatterySettings(model=BATTERY), + user_target_pct=50.0, + model=ErrorModel(), + min_history_days=30, + tz=BERLIN, + plant=None, + ) + assert view.status == "ok" + assert view.evening_target is not None + assert view.morning_buffer is not None + assert view.evening_target.soc_pct >= 50 + assert view.evening_target.reason.key == "planner.battery.evening_target" + night = next(w for w in view.windows if w.window is RiskWindow.NIGHT) + assert view.evening_at == night.start + + +# --- review findings: anchors, coverage, no PV forecast, clipping ------------------------------ + + +class RecordingData: + """A fake ``RiskData``: forecasts of a sine-bell day between ``rise`` and ``set_`` local + hours, measured PV equal to the forecast, recording the anchors asked for.""" + + def __init__(self, rise: float, set_: float, *, pv: bool = True, load_hours: int = 72) -> None: + self.rise, self.set_ = rise, set_ + self.pv, self.load_hours = pv, load_hours + self.asked: list[datetime] = [] + + def power(self, moment: datetime) -> float: + t = moment.astimezone(BERLIN) + hour = t.hour + t.minute / 60 + 0.125 + if not self.rise <= hour < self.set_: + return 0.0 + return 3000 * math.sin(math.pi * (hour - self.rise) / (self.set_ - self.rise)) + + async def read_series( + self, + key: str, + window: TimeRange, + resolution: str = "15m", + unit: SeriesUnit | None = None, + *, + include_recent: bool = True, + ) -> TimeSeries: + assert key == SITE_PV_W + assert (resolution, unit, include_recent) == ("15m", SeriesUnit.W, True) + count = (window.end - window.start) // Q + return series([self.power(window.start + Q * i) for i in range(count)], window.start) + + async def latest_forecast_before( + self, + source_id: str, + kind: str, + before: datetime, + window: TimeRange | None = None, + ) -> QuantileSeries | None: + assert window is not None + assert source_id in ("blended", "load_profile") + if kind == "pv": + self.asked.append(before) + if not self.pv: + return None + count = (window.end - window.start) // Q + values = [self.power(window.start + Q * i) for i in range(count)] + return QuantileSeries.deterministic(series(values, window.start)) + count = min((window.end - window.start) // Q, self.load_hours * 4) + return QuantileSeries.deterministic(series([400.0] * count, window.start)) + + +@pytest.mark.parametrize(("rise", "set_"), [(8.5, 16.5), (5.0, 21.5)], ids=["winter", "summer"]) +async def test_anchors_follow_the_sun(rise: float, set_: float) -> None: + from suncore.forecast.risk_learning import learn_day # noqa: PLC0415 + + data = RecordingData(rise, set_) + day = date(2026, 6, 2) + samples = await learn_day(data, day, tz=BERLIN, plant=None, base_reader=flat_base) + local = [t.astimezone(BERLIN) for t in data.asked] + hours = sorted(t.hour + t.minute / 60 for t in local if t.date() == day and t.hour > 0) + assert hours[0] == pytest.approx(rise + 1, abs=0.25) # sunrise + 1 h (quarter-hour grid) + assert hours[-1] == pytest.approx(set_ - 1, abs=0.25) # sunset - 1 h + assert {s.window for s in samples} == set(RiskWindow) - {RiskWindow.REST_OF_NIGHT} + assert all(abs(s.error_wh) < 1 for s in samples) # forecast == measurement + + +async def test_without_a_pv_forecast_only_the_night_is_learnt() -> None: + from suncore.forecast.risk_learning import learn_day # noqa: PLC0415 + + data = RecordingData(6.0, 21.0, pv=False) + samples = await learn_day(data, date(2026, 6, 2), tz=BERLIN, plant=None, base_reader=flat_base) + assert [s.window for s in samples] == [RiskWindow.NIGHT] + + +async def test_windows_the_forecast_does_not_cover_are_skipped() -> None: + from suncore.forecast.risk_learning import learn_day # noqa: PLC0415 + + data = RecordingData(6.0, 21.0, load_hours=21) # the load forecast ends 20:00 on day 1 + samples = await learn_day(data, date(2026, 6, 2), tz=BERLIN, plant=None, base_reader=flat_base) + assert {s.window for s in samples} == {RiskWindow.REST_OF_TODAY} + + +async def test_clipped_daytime_windows_are_dropped() -> None: + from suncore.forecast.risk_learning import learn_day # noqa: PLC0415 + + noon = datetime(2026, 6, 3, 13, 0, tzinfo=BERLIN).astimezone(UTC) # on the next day + data = RecordingData(6.0, 21.0) + samples = await learn_day( + data, + date(2026, 6, 2), + tz=BERLIN, + plant=None, + base_reader=flat_base, + is_clipped=lambda slot: slot == noon, + ) + assert {s.window for s in samples} == { + RiskWindow.REST_OF_TODAY, + RiskWindow.DAY_AND_NIGHT, + RiskWindow.NIGHT, + } diff --git a/tests/ha/test_websocket_registry.py b/tests/ha/test_websocket_registry.py index b812b26..c7797a9 100644 --- a/tests/ha/test_websocket_registry.py +++ b/tests/ha/test_websocket_registry.py @@ -214,7 +214,12 @@ def test_allowlist_is_exactly_the_verified_read_only_set() -> None: """Adding an action needs a PR that verifies it against the integration source.""" assert ( frozenset( - {"tibber.get_prices", "solcast_solar.query_forecast_data", "weather.get_forecasts"} + { + "tibber.get_prices", + "solcast_solar.query_forecast_data", + "weather.get_forecasts", + "workday.check_date", + } ) == READ_ONLY_ACTIONS ) diff --git a/tests/safety/test_boundary.py b/tests/safety/test_boundary.py index 5685e7e..5af2c3c 100644 --- a/tests/safety/test_boundary.py +++ b/tests/safety/test_boundary.py @@ -380,7 +380,12 @@ def test_read_only_action_allowlist_is_fixed() -> None: assert ( frozenset( - {"tibber.get_prices", "solcast_solar.query_forecast_data", "weather.get_forecasts"} + { + "tibber.get_prices", + "solcast_solar.query_forecast_data", + "weather.get_forecasts", + "workday.check_date", + } ) == READ_ONLY_ACTIONS )