From e26033f9f101a556e3ba5b2ac9f8c474faf47df0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 16:49:00 +0000 Subject: [PATCH 1/2] wip(forecast): ML load model (WP12) snapshot Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- docs/STATUS.md | 47 +- docs/plans/phase-4.md | 33 ++ docs/user/de/planning-and-battery.md | 29 +- docs/user/en/planning-and-battery.md | 28 +- frontend/e2e/plan.spec.ts | 3 +- frontend/src/lib/api/openapi.json | 106 +++- frontend/src/lib/api/schema.d.ts | 47 +- frontend/src/lib/api/types.ts | 1 + .../src/lib/components/ForecastPanel.svelte | 68 ++- .../src/lib/components/ForecastPanel.test.ts | 145 +++++ frontend/src/lib/i18n/locales/de.json | 25 +- frontend/src/lib/i18n/locales/en.json | 25 +- frontend/src/lib/views/steps/PvStep.svelte | 33 +- frontend/src/lib/wizard/forecastForm.ts | 22 +- frontend/src/lib/wizard/phase4Forms.test.ts | 20 +- scripts/bench_load_ml.py | 122 ++++ suncore/api/app.py | 6 +- suncore/app/adapters.py | 5 +- suncore/app/runtime.py | 15 +- suncore/forecast/load_ml.py | 519 ++++++++++++++++++ suncore/forecast/load_service.py | 264 ++++++++- suncore/forecast/views.py | 29 +- suncore/planner/inputs.py | 9 +- tests/api/test_forecast_api.py | 59 ++ tests/app/test_load_runtime.py | 34 ++ tests/app/test_pv_runtime.py | 4 +- tests/forecast/test_load_ml.py | 396 +++++++++++++ tests/forecast/test_load_ml_bench.py | 31 ++ tests/forecast/test_load_ml_service.py | 290 ++++++++++ tests/planner/test_inputs.py | 9 + 30 files changed, 2363 insertions(+), 61 deletions(-) create mode 100644 frontend/src/lib/components/ForecastPanel.test.ts create mode 100644 scripts/bench_load_ml.py create mode 100644 suncore/forecast/load_ml.py create mode 100644 tests/forecast/test_load_ml.py create mode 100644 tests/forecast/test_load_ml_bench.py create mode 100644 tests/forecast/test_load_ml_service.py diff --git a/docs/STATUS.md b/docs/STATUS.md index 37e5046..82b5ccd 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -5,13 +5,39 @@ ADRs relevant to your task. Keep it short; history lives in git. **Last update:** 2026-10-28 · **Phase:** 4 implemented in the stacked PRs 4-A (contracts), 4-B (storage), 4-C (tariffs), 4-D (PV forecast), 4-E (load forecast and risk model), 4-F (planner and -runtime), 4-G (battery control) and 4-H (UI and docs, WP11 + WP13); only the optional 4-I (ML load -model) is open; waiting for the maintainer's acceptance on his system. Phase 3 (3-A `phase-3/a-safety`, +runtime), 4-G (battery control), 4-H (UI and docs, WP11 + WP13) and the optional 4-I (ML load +model, WP12); waiting for the maintainer's acceptance on his system. Phase 3 (3-A `phase-3/a-safety`, 3-B `phase-3/b-controller`, 3-C `phase-3/c-ui`) is implemented, waiting for the maintainer's shadow -test · **Branch:** `phase-4/h-ui-docs` (stacked on `phase-4/g-battery`) +test · **Branch:** `phase-4/i-ml-load` (stacked on `phase-4/h-ui-docs`) ## Done +- **Phase 4 – PR 4-I, WP12 optional ML load model (branch `phase-4/i-ml-load`, stacked on 4-H; + `optimizer-engineer`)**: + - `suncore/forecast/load_ml.py` (NumPy only): KNN on analogue days - quarter-hour samples of + the base load history (`max(91, min_history_days)` days, ≥ 75 % coverage, unmeasured device + slots dropped after 7 clean days), features day type (holiday ≈ weekend), slot (±2), daily + mean temperature, 7-day load level, weak age term; K = 24, P50 weighted median, P10/P90 + weighted quantiles, ≥ 0, monotone, deterministic (stable sort). Forecasts after 7 usable days. + - `forecast.learning.load_ml_mode`: `off` not computed; `shadow` (default) stored hourly as + source `load_ml`, scored nightly like `load_profile`, shown, never used; `auto` used by the + planner (and the risk view) only with ≥ `min_history_days` days and a pooled 14-day MAE ≥ 2 % + below the profiles' on a comparable sample, else the profiles; every choice has a + `forecast.load_model.*` reason, in `auto` also as a plan-level reason. + - API: `ForecastAccuracyView.load_model` and `LoadForecastView.model` (`LoadModelView`, additive); + `load_ml` rows in `/forecast/accuracy`. UI: "Machine-learning load model" in the forecast + panel (mode, model in use, reason, both MAEs, history days); mode select in wizard step 4. + en/de keys added (`check_reasons` green), user guide §2 "The machine-learning load model". + - Benchmark `scripts/bench_load_ml.py` (synthetic house, 48 h = 193 slots, median of 5, session + amd64 container): training + prediction 30 days 30 ms, 60 days 49 ms, 90 days 36 ms (budget + 1 s; slow test `tests/forecast/test_load_ml_bench.py`). Whole service refit (SQLite read of 91 + days, 3 devices, profile + KNN fit) 128 ms vs 82 ms without the KNN model; aarch64 not + measured (`UNVERIFIED`). + - Tests: `tests/forecast/test_load_ml.py` (features, DST, hypothesis: quantiles monotone, + non-negative, on the grid, deterministic, every `auto` rule), `test_load_ml_service.py` + (shadow/off/auto, fallback, window, nightly bookkeeping, stored scores), API, runtime, planner + pass-through, Vitest panel and form. + - **Phase 4 – PR 4-A (plan, WP1, WP2; branch `claude/zealous-mayer-j5oda9`)**: - Plan `docs/plans/phase-4.md` **revision 2 approved** (2026-10-16), incl. the merged concept "Prognose, Puffer und Akkuplanung" (§0) and the split into stacked PRs 4-A…4-I. @@ -631,10 +657,10 @@ test · **Branch:** `phase-4/h-ui-docs` (stacked on `phase-4/g-battery`) runtime (WP8, WP9) → 4-G battery control (WP10) → 4-H frontend + docs (WP11, WP13); 4-I ML load (WP12, optional, after 4-E). Merge only after the Phase 3 acceptance. - **WP10, WP11 and WP13 done** (4-G and 4-H, see Done). All Phase 4 PRs are open and stacked; merge - order **#25 → #26 → #27/#29 → #30 → #31 → #33 → 4-H** (merge commits are fine; **squash is + order **#25 → #26 → #27/#29 → #30 → #31 → #33 → 4-H → 4-I** (merge commits are fine; **squash is recommended**, one commit per PR on `main`), **only after the Phase 3 acceptance** (plan §6). - ADR-0013 accepted by the maintainer on 2026-10-19. Open: 4-I (optional ML load model), and the - follow-ups below. + ADR-0013 accepted by the maintainer on 2026-10-19. 4-I (optional ML load model) is implemented on + `phase-4/i-ml-load`; open are the follow-ups below. - **Safety note:** the documented fallback-timer measurement lifts the assisted cap (via `fallback_measured`); the procedure gets a `control-safety` review (the lead runs it). It needs the app's Watchdog switch off, otherwise the Supervisor restarts the killed app and SunCore itself @@ -649,7 +675,8 @@ test · **Branch:** `phase-4/h-ui-docs` (stacked on `phase-4/g-battery`) - Follow-ups from 4-H and the docs: `fallback_timeout_s` changes should drop the battery mode to shadow on the server (follows); a switch for `battery.fallback_measured` in the panel (today: export, edit, import), a hint for a too-new `timeseries.db` (the API exposes no such state), the - panel for editing `action_params`/`series_key`/`learning`. + panel for editing `action_params`/`series_key`/`learning.min_history_days` (the ML mode has a + select since 4-I). - Maintainer questions from 4-G (PR description): grid charging on the Kostal (generic force-charge entity or a §3.4 ADR); the fallback timer measurement procedure for `battery.fallback_measured`. Answered: no permanent reserve floor on the inverter. @@ -864,7 +891,7 @@ test · **Branch:** `phase-4/h-ui-docs` (stacked on `phase-4/g-battery`) 4. Optional, later: enable external battery management via Modbus on the inverter, add the modbus hub, and measure the fallback timer with the documented procedure; then set `battery.fallback_measured`. -3. Merge the stacked PRs in order **#25 → #26 → #27/#29 → #30 → #31 → #33 → 4-H** (squash +3. Merge the stacked PRs in order **#25 → #26 → #27/#29 → #30 → #31 → #33 → 4-H → 4-I** (squash recommended), after the Phase 3 acceptance; each rebase onto `main` after the previous merge. Release the next release-please PR for the stable app after acceptance (schema 4: release note "export before updating"). @@ -892,7 +919,7 @@ test · **Branch:** `phase-4/h-ui-docs` (stacked on `phase-4/g-battery`) | Publication time of next-day prices (Tibber, Nord Pool, ENTSO-E, EPEX) | `research/integrations.md` | observe on the maintainer's HA | | Nord Pool `config_entry` id source, sensor translation keys, publication time; Octopus/ENTSO-E/EPEX entity matchers (platform and unit only, no translation keys); `start`/`end` sent to `tibber.get_prices` as ISO with offset; negative-price rule proxy for all-in sources | `templates/tariff/*.yaml`, `suncore/tariff/pricing.py` | live check on a system with the integration | | VAT applied to negative spot prices | `suncore/core/tariff.py` | maintainer's bill / supplier terms | -| KNN load model performance on aarch64 | plan WP12 | CI arm runner benchmark | +| KNN load model run time on aarch64 (amd64: training + 48 h prediction 30–50 ms for 30–90 days, `scripts/bench_load_ml.py`); its constants (K 24, slot window ±2, scales 4 K / 15 % level / 120 days age, 2 % margin) are reasoned, not fitted | plan WP12, `suncore/forecast/load_ml.py` | CI arm runner or a Pi; 30+ days of the maintainer's data in `shadow` | | Energy colour variable names (`--energy-*`) | `frontend/src/app.css` | read HA frontend source | | HA UI menu paths in the user docs (Settings → Apps → App store, Repositories, Show in sidebar; German labels) | `addon/*/DOCS.md`, `README.md`, `docs/user/*/quick-start.md` | maintainer checks on HA OS during the beta install | | Supervisor Core proxy passes `supported_features` through | `suncore/ha/websocket.py` | first run on HA OS | @@ -918,7 +945,7 @@ test · **Branch:** `phase-4/h-ui-docs` (stacked on `phase-4/g-battery`) | `weather.get_forecasts` `hourly` support of the maintainer's weather integration (the `temperature_unit` attribute and the conversion of the forecast to it are verified from source, `research/ha-websocket-api.md`) | `suncore/app/adapters.py` (`HaLoadReader`) | maintainer's HA | | `workday.check_date` over the Supervisor proxy and with the maintainer's Workday configuration (the `days_offset` handling is verified from source; a changed country/holiday list is not visible in the attributes, so the day-type cache only resets on a changed entity, `workdays`, `excludes` or offset) | `suncore/app/adapters.py`, `suncore/forecast/load_service.py` | maintainer maps a Workday sensor | | Load and risk constants (half-life 21 days, outlier z 3, shrink 3 days, 7 clean days, degree-day bases 15/22 °C, coverage 75 %/90 %, 20 h temperature days, anchors sunrise + 1 h / sunset − 1 h, sunrise threshold 2 %/20 W) and the default PV bands per weather class (clear 0.85/1.10, mixed 0.55/1.35, overcast 0.35/1.70) are reasoned, not fitted | `suncore/forecast/load.py`, `suncore/forecast/risk.py`, `risk_learning.py` | 30+ days of the maintainer's data | -| Known limitation (n6): the load history and the risk samples are not re-derived when the device list changes - a device added later is only subtracted from the base load from the day its power is recorded (or rebuilt), and older risk samples keep the old base load | `suncore/forecast/load_service.py`, `risk_learning.py` | accepted for Phase 4; revisit with 4-I | +| Known limitation (n6): the load history and the risk samples are not re-derived when the device list changes - a device added later is only subtracted from the base load from the day its power is recorded (or rebuilt), and older risk samples keep the old base load | `suncore/forecast/load_service.py`, `risk_learning.py` | accepted for Phase 4; the KNN model (4-I) shares the base load and the limitation; revisit in Phase 5 | ## Environment notes diff --git a/docs/plans/phase-4.md b/docs/plans/phase-4.md index 7104ef2..e8b076a 100644 --- a/docs/plans/phase-4.md +++ b/docs/plans/phase-4.md @@ -262,6 +262,9 @@ NumPy KNN on analogue days (features: day type, slot, temperature, recent load); by default: scored next to the profiles, shown in the accuracy view, used only in `auto` while better over the last 14 days. Benchmark on aarch64 (training + prediction < 1 s). +**Implemented in PR 4-I** (branch `phase-4/i-ml-load`); deviations see "Deviations during +implementation". + ### WP13 – Docs, translations, STATUS · `docs-writer`, `i18n-sync` (Haiku) User guide `docs/user//planning-and-battery.md` (forecasts, risk level and buffers, @@ -636,6 +639,36 @@ shadow and assisted run is the acceptance test. - The battery card shows what the control state contains; there is no separate "release pending" field in `BatteryControlState`, so that state is only visible through the reasons. +- PR 4-I, WP12 (optional ML load model; `optimizer-engineer`): + - `suncore/forecast/load_ml.py`: KNN on quarter-hour samples of analogue days; features day + type (holiday close to weekend, far from workday), wall-clock slot (candidates ±2 slots), + daily mean temperature (only when the forecast day has one), level = mean base load of the + **7 days** before the day (a whole week, so the level is free of the weekday pattern; the + query uses the 7 days before the fit), plus a weak age term that also breaks ties. K = 24, + weights `1 / (0.25 + distance)`, P50 = weighted **median** (the profiles use a weighted mean; + the median suits the MAE score), P10/P90 = weighted 10/90 % quantiles of the neighbours. The + constants are reasoned, not fitted (`UNVERIFIED` like the profile constants). + - History window of the KNN model: `max(91 days, min_history_days)` (the profiles keep their + 56 days; the service reads the longer window once and slices it for the profiles). The fit + window now starts at a local midnight, so a DST change inside it no longer drops its first + day (also for the profiles). + - "Better" in `auto` = the pooled (n-weighted over the three horizon buckets) 14-day MAE of + `load_ml` at least **2 %** below `load_profile`'s, with `load_ml` scored on at least 90 % as many + quarter hours (else the two MAEs cover different periods; ≥ 96 in any case) and at least + `min_history_days` usable days; otherwise the profiles, each with a + `forecast.load_model.*` reason. The choice changes with the nightly scores, at most once a + day. + - In `auto` the plan carries the choice as a plan-level reason (`build_inputs(load_reasons=…)`, + planner still imports only `core`); in `off`/`shadow` no plan reason (nothing to explain). + The risk view and the planner get the same selected forecast; the risk model's *learnt* + errors keep coming from the stored `load_profile` forecasts. + - The accuracy view shows `load_ml` as its own source row and `load_model` (mode, model in use, + reason, both pooled MAEs, history days) under "Machine-learning load model"; the forecast + wizard step got the mode select (`forecast.learning.load_ml_mode`, the only learning field + in the UI). + - Benchmark `scripts/bench_load_ml.py` (sibling of `bench_planner.py`), slow test < 1 s on CI; + aarch64 still `UNVERIFIED` (no arm runner job exists yet). + ## 7. Out of scope for Phase 4 Full plan chart with device Gantt, zoom/pan, history, savings vs baselines, what-if and diff --git a/docs/user/de/planning-and-battery.md b/docs/user/de/planning-and-battery.md index 82cfd69..f157483 100644 --- a/docs/user/de/planning-and-battery.md +++ b/docs/user/de/planning-and-battery.md @@ -119,8 +119,33 @@ Zwei optionale Eingaben in Schritt 4 (**Eingaben der Lastprognose**) verbessern zeichnet SunCore deren Außentemperatur selbst auf. Der Temperaturterm der Lastprognose wird nach 14 Tagen mit aufgezeichneter Temperatur aktiv. -Ein Lastmodell mit maschinellem Lernen ist **noch nicht verfügbar**; das Feld sagt es. Die Profile -funktionieren vom ersten Tag an. +Die Profile funktionieren vom ersten Tag an. + +### Das maschinelle Lastmodell + +Neben den Profilen rechnet SunCore ein optionales **maschinelles Lastmodell**. Es sucht die +vergangenen Tage, die dem kommenden am ähnlichsten sind (gleiche Tagesart – ein Feiertag zählt wie +ein Wochenendtag –, gleiche Uhrzeit, ähnliche Außentemperatur, wenn eine Wetter-Entität zugeordnet +ist, und ein ähnliches Lastniveau in der letzten Woche), und sagt jede Viertelstunde aus dem +Verbrauch an diesen Tagen voraus, mit einem Band aus deren Streuung. Es braucht keine zusätzliche +Software und rechnet in wenigen zehn Millisekunden. + +Den Modus wählst du in Schritt 4 unter **Maschinelles Lastmodell**: + +- **Aus**: Es wird nicht berechnet. +- **Nur Vergleich** (Standard): Es wird berechnet, gespeichert und jede Nacht genau wie die Profile + bewertet und in **Prognosen und Genauigkeit** unter *Maschinelles Lastmodell* angezeigt, beide + Fehler nebeneinander. Der Plan nutzt es nie. +- **Automatisch**: Der Plan nutzt es nur, solange zweierlei gilt: Es hat mindestens den Verlauf, + den die Lerneinstellung verlangt (standardmäßig 30 Tage), **und** sein mittlerer absoluter + Fehler lag in den letzten 14 Tagen mindestens 2 % unter dem der Profile (bewertet an etwa + ebenso vielen Viertelstunden). Sonst bleibt der Plan bei den Profilen. Jeder Plan trägt dann + einen kurzen Hinweis, welches Modell er genutzt hat und warum („noch zu wenig Verlauf“, „nicht + deutlich besser“, „genauer als die Profile“ mit beiden Fehlern). + +Die Bewertung entsteht im nächtlichen Lauf, ein Wechsel zwischen den Modellen passiert also +höchstens einmal am Tag. Das Risikomodell lernt seine Fehlerverteilung weiter aus der +Profilprognose. ## 3. Tarife diff --git a/docs/user/en/planning-and-battery.md b/docs/user/en/planning-and-battery.md index 2f738da..a7ed6ed 100644 --- a/docs/user/en/planning-and-battery.md +++ b/docs/user/en/planning-and-battery.md @@ -113,8 +113,32 @@ Two optional inputs in step 4 (**Load forecast inputs**) improve it: records its outdoor temperature itself. The temperature term of the load forecast becomes active after 14 days with a recorded temperature. -A machine-learning load model is **not available yet**; the panel says so. The profiles work from -day one. +The profiles work from day one. + +### The machine-learning load model + +Next to the profiles SunCore runs an optional **machine-learning load model**. It looks for the +past days that resemble the coming one most (same kind of day – a holiday counts like a weekend +day –, the same time of day, a similar outdoor temperature if a weather entity is mapped, and a +similar load level over the past week) and forecasts each quarter hour from what the house used +on those days, with a band from their spread. It needs no extra software and runs in a few tens of +milliseconds. + +Choose its mode in step 4 under **Machine-learning load model**: + +- **Off**: it is not computed. +- **Comparison only** (default): it is computed, stored and scored every night exactly like the + profiles, and shown in **Forecasts and accuracy** under *Machine-learning load model*, with both + errors side by side. The plan never uses it. +- **Automatic**: the plan uses it only while two things hold: it has at least the history the + learning setting asks for (30 days by default), **and** over the last 14 days its mean absolute + error was at least 2 % lower than the profiles' (scored on about as many quarter hours). Otherwise + the plan stays on the profiles. Each plan then carries a short note which model it used and why + ("not enough history yet", "not clearly better", "more accurate than the profiles" with both + errors). + +The scores are computed in the nightly job, so a switch between the models happens at most once a +day. The risk model keeps learning its error distribution from the profile forecast. ## 3. Tariffs diff --git a/frontend/e2e/plan.spec.ts b/frontend/e2e/plan.spec.ts index db74f3b..fffb1fe 100644 --- a/frontend/e2e/plan.spec.ts +++ b/frontend/e2e/plan.spec.ts @@ -69,7 +69,8 @@ test.describe('Plan view', () => { await expect(panel.getByTestId('fc-accuracy')).toBeVisible(); await expect(panel.getByTestId('fc-weights')).toContainText('equal'); await expect(panel.getByTestId('fc-calibration')).toContainText('No history yet'); - await expect(panel.getByTestId('fc-ml')).toContainText('Not available yet'); + await expect(panel.getByTestId('fc-ml-mode')).toContainText('comparison only'); + await expect(panel.getByTestId('fc-ml-in-use')).toContainText('the profiles'); const credit = panel.getByTestId('open-meteo-attribution'); await expect(credit).toContainText('CC BY 4.0'); await expect(credit.getByRole('link', { name: 'Open-Meteo.com' })).toHaveAttribute( diff --git a/frontend/src/lib/api/openapi.json b/frontend/src/lib/api/openapi.json index 989e5e0..066f6ed 100644 --- a/frontend/src/lib/api/openapi.json +++ b/frontend/src/lib/api/openapi.json @@ -3425,8 +3425,18 @@ }, "ForecastAccuracyView": { "additionalProperties": false, - "description": "Accuracy per source and horizon bucket.\n\n``bias_w = actual - forecast`` (positive = the source under-forecasts); ``n`` counts\n15-minute daylight intervals (forecast or measurement above 0 W; clipped ones excluded).\nA record with ``n`` below ``min_samples`` is too thin to rely on.", + "description": "Accuracy per source and horizon bucket.\n\n``bias_w = actual - forecast`` (positive = the source under-forecasts); ``n`` counts\n15-minute daylight intervals (forecast or measurement above 0 W; clipped ones excluded).\nA record with ``n`` below ``min_samples`` is too thin to rely on. ``load_model`` compares the\nKNN load model with the profiles (\"ML vs profile\"; ``None`` without a load forecast).", "properties": { + "load_model": { + "anyOf": [ + { + "$ref": "#/components/schemas/LoadModelView" + }, + { + "type": "null" + } + ] + }, "min_samples": { "title": "Min Samples", "type": "integer" @@ -4571,6 +4581,16 @@ "title": "Hours", "type": "integer" }, + "model": { + "anyOf": [ + { + "$ref": "#/components/schemas/LoadModelView" + }, + { + "type": "null" + } + ] + }, "p10_w": { "items": { "anyOf": [ @@ -4670,6 +4690,88 @@ "title": "LoadForecastView", "type": "object" }, + "LoadModelView": { + "additionalProperties": false, + "description": "Which load model the planner gets: the profiles or the optional KNN model (WP12).\n\n``mode`` is ``forecast.learning.load_ml_mode``; ``reason`` explains ``in_use``\n(``forecast.load_model.*``). The errors are the rolling 14-day MAE of ``load_profile`` and\n``load_ml`` pooled over the horizon buckets (``None``: not scored yet); ``ml_days`` counts the\nusable history days of the KNN model, which ``auto`` needs ``min_history_days`` of.", + "properties": { + "in_use": { + "enum": [ + "profile", + "ml" + ], + "title": "In Use", + "type": "string" + }, + "min_history_days": { + "title": "Min History Days", + "type": "integer" + }, + "ml_days": { + "title": "Ml Days", + "type": "integer" + }, + "ml_mae_w": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Ml Mae W" + }, + "ml_n": { + "title": "Ml N", + "type": "integer" + }, + "ml_ready": { + "title": "Ml Ready", + "type": "boolean" + }, + "mode": { + "enum": [ + "off", + "shadow", + "auto" + ], + "title": "Mode", + "type": "string" + }, + "profile_mae_w": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Profile Mae W" + }, + "profile_n": { + "title": "Profile N", + "type": "integer" + }, + "reason": { + "$ref": "#/components/schemas/Reason" + } + }, + "required": [ + "mode", + "in_use", + "reason", + "ml_ready", + "ml_days", + "min_history_days", + "profile_mae_w", + "ml_mae_w", + "profile_n", + "ml_n" + ], + "title": "LoadModelView", + "type": "object" + }, "LoadQuality": { "enum": [ "ok", @@ -9702,7 +9804,7 @@ }, "/api/v1/forecast/accuracy": { "get": { - "description": "MAE and bias per source and horizon bucket (0-6 h, 6-24 h, 24-48 h) over the last 14\ndays, plus the source id ``blended`` (the blended forecast); ``bias_w = actual -\nforecast`` (positive: the source under-forecasts); ``n`` counts 15-minute intervals;\nclipped quarter hours are excluded; computed nightly. The source id ``load_profile`` is\nthe base load forecast (scored on every quarter hour, not only in daylight).", + "description": "MAE and bias per source and horizon bucket (0-6 h, 6-24 h, 24-48 h) over the last 14\ndays, plus the source id ``blended`` (the blended forecast); ``bias_w = actual -\nforecast`` (positive: the source under-forecasts); ``n`` counts 15-minute intervals;\nclipped quarter hours are excluded; computed nightly. The source id ``load_profile`` is\nthe base load forecast (scored on every quarter hour, not only in daylight), ``load_ml``\nthe optional machine-learning (KNN) load model scored the same way unless\n``forecast.learning.load_ml_mode`` is ``off``. ``load_model`` compares the two (\"ML vs\nprofile\": pooled 14-day MAE) and says which one the planner uses and why\n(``forecast.load_model.*`` reason); ``null`` without a load forecast.", "operationId": "forecast_accuracy_api_v1_forecast_accuracy_get", "responses": { "200": { diff --git a/frontend/src/lib/api/schema.d.ts b/frontend/src/lib/api/schema.d.ts index 7cb452b..d104f14 100644 --- a/frontend/src/lib/api/schema.d.ts +++ b/frontend/src/lib/api/schema.d.ts @@ -382,7 +382,11 @@ export interface paths { * days, plus the source id ``blended`` (the blended forecast); ``bias_w = actual - * forecast`` (positive: the source under-forecasts); ``n`` counts 15-minute intervals; * clipped quarter hours are excluded; computed nightly. The source id ``load_profile`` is - * the base load forecast (scored on every quarter hour, not only in daylight). + * the base load forecast (scored on every quarter hour, not only in daylight), ``load_ml`` + * the optional machine-learning (KNN) load model scored the same way unless + * ``forecast.learning.load_ml_mode`` is ``off``. ``load_model`` compares the two ("ML vs + * profile": pooled 14-day MAE) and says which one the planner uses and why + * (``forecast.load_model.*`` reason); ``null`` without a load forecast. */ get: operations["forecast_accuracy_api_v1_forecast_accuracy_get"]; put?: never; @@ -2084,9 +2088,11 @@ export interface components { * * ``bias_w = actual - forecast`` (positive = the source under-forecasts); ``n`` counts * 15-minute daylight intervals (forecast or measurement above 0 W; clipped ones excluded). - * A record with ``n`` below ``min_samples`` is too thin to rely on. + * A record with ``n`` below ``min_samples`` is too thin to rely on. ``load_model`` compares the + * KNN load model with the profiles ("ML vs profile"; ``None`` without a load forecast). */ ForecastAccuracyView: { + load_model?: components["schemas"]["LoadModelView"] | null; /** Min Samples */ min_samples: number; /** Sources */ @@ -2543,6 +2549,7 @@ export interface components { generated_at: string; /** Hours */ hours: number; + model?: components["schemas"]["LoadModelView"] | null; /** P10 W */ p10_w: (number | null)[]; /** P50 W */ @@ -2560,6 +2567,42 @@ export interface components { /** Unmeasured In Profile */ unmeasured_in_profile: components["schemas"]["DayType"][]; }; + /** + * LoadModelView + * @description Which load model the planner gets: the profiles or the optional KNN model (WP12). + * + * ``mode`` is ``forecast.learning.load_ml_mode``; ``reason`` explains ``in_use`` + * (``forecast.load_model.*``). The errors are the rolling 14-day MAE of ``load_profile`` and + * ``load_ml`` pooled over the horizon buckets (``None``: not scored yet); ``ml_days`` counts the + * usable history days of the KNN model, which ``auto`` needs ``min_history_days`` of. + */ + LoadModelView: { + /** + * In Use + * @enum {string} + */ + in_use: "profile" | "ml"; + /** Min History Days */ + min_history_days: number; + /** Ml Days */ + ml_days: number; + /** Ml Mae W */ + ml_mae_w: number | null; + /** Ml N */ + ml_n: number; + /** Ml Ready */ + ml_ready: boolean; + /** + * Mode + * @enum {string} + */ + mode: "off" | "shadow" | "auto"; + /** Profile Mae W */ + profile_mae_w: number | null; + /** Profile N */ + profile_n: number; + reason: components["schemas"]["Reason"]; + }; /** * LoadQuality * @enum {string} diff --git a/frontend/src/lib/api/types.ts b/frontend/src/lib/api/types.ts index 9df8ff0..a919d81 100644 --- a/frontend/src/lib/api/types.ts +++ b/frontend/src/lib/api/types.ts @@ -86,6 +86,7 @@ export type BatteryBand = S['BatteryBand']; export type PriceReport = S['PriceReport']; export type PvForecastView = S['PvForecastView']; export type ForecastAccuracyView = S['ForecastAccuracyView']; +export type LoadModelView = S['LoadModelView']; export type AccuracySourceView = S['AccuracySourceView']; export type AccuracyRecord = S['AccuracyRecord']; export type RiskView = S['RiskView']; diff --git a/frontend/src/lib/components/ForecastPanel.svelte b/frontend/src/lib/components/ForecastPanel.svelte index 566403e..68b3f06 100644 --- a/frontend/src/lib/components/ForecastPanel.svelte +++ b/frontend/src/lib/components/ForecastPanel.svelte @@ -5,6 +5,7 @@ hasMessage, t, tDateTime, + tDynamic, tNumber, tPercent, tPower, @@ -66,6 +67,11 @@ const basisKeys = $derived([ ...new Set((blended?.weights ?? []).map((w) => w.basis).filter((b) => b != null)), ]); + + /** "ML vs profile": the optional machine-learning load model next to the profiles. */ + const loadModel = $derived(accuracy?.load_model ?? null); + const mae = (w: number | null | undefined): string => + w == null ? t('fc.ml.not_scored') : tPower(w);
@@ -215,7 +221,55 @@

{t('fc.ml.title')}

-

{t('fc.ml.none')}

+

{t('fc.ml.intro')}

+ {#if loadModel} +

+ {t('fc.ml.mode', { mode: loadModel.mode })} + {t('fc.ml.in_use', { model: loadModel.in_use })} +

+

{tDynamic(loadModel.reason.key, loadModel.reason.args)}

+ {#if loadModel.mode !== 'off'} +
+ + + + + + + + + + + + + + + + + + +
{t('fc.ml.compare_caption')}
{t('fc.ml.model')}{t('fc.ml.error')}
{t('fc.ml.profile')} + {mae(loadModel.profile_mae_w)} + {#if loadModel.profile_n > 0}{t('fc.cell_n', { n: loadModel.profile_n })}{/if} +
{t('fc.ml.ml')} + {mae(loadModel.ml_mae_w)} + {#if loadModel.ml_n > 0}{t('fc.cell_n', { n: loadModel.ml_n })}{/if} +
+
+

+ {t('fc.ml.days', { days: loadModel.ml_days, min: loadModel.min_history_days })} + {#if !loadModel.ml_ready}· {t('fc.ml.collecting_model')}{/if} +

+ {/if} + {/if}
{#if openMeteo} @@ -289,6 +343,18 @@ .num { font-variant-numeric: tabular-nums; } + table.compact { + min-width: 0; + } + .tags { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin: 0 0 6px; + } + .chip-tag.active { + font-weight: 600; + } .attribution a { overflow-wrap: anywhere; } diff --git a/frontend/src/lib/components/ForecastPanel.test.ts b/frontend/src/lib/components/ForecastPanel.test.ts new file mode 100644 index 0000000..ee633b0 --- /dev/null +++ b/frontend/src/lib/components/ForecastPanel.test.ts @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: Apache-2.0 +import { cleanup, render, screen, within } from '@testing-library/svelte'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { ForecastAccuracyView, LoadModelView } from '../api/types'; +import { applyLanguage } from '../i18n/index.svelte'; +import ForecastPanel from './ForecastPanel.svelte'; + +beforeEach(() => applyLanguage('en')); +afterEach(() => cleanup()); + +const text = (id: string): string => + (screen.getByTestId(id).textContent ?? '').replace(/\s+/g, ' ').trim(); + +function model(overrides: Partial = {}): LoadModelView { + return { + mode: 'shadow', + in_use: 'profile', + reason: { key: 'forecast.load_model.ml_shadow', args: { days: 12, min_days: 30 } }, + ml_ready: true, + ml_days: 12, + min_history_days: 30, + profile_mae_w: 118.4, + ml_mae_w: null, + profile_n: 1344, + ml_n: 0, + ...overrides, + }; +} + +function accuracy(load: LoadModelView | null): ForecastAccuracyView { + const record = (source_id: string, mae_w: number) => ({ + source_id, + horizon: '0_6h' as const, + mae_w, + bias_w: -4, + n: 1344, + }); + return { + min_samples: 96, + sources: [ + { + source_id: 'load_profile', + computed_at: null, + window_days: 14, + records: [record('load_profile', 118)], + }, + { + source_id: 'load_ml', + computed_at: null, + window_days: 14, + records: [record('load_ml', 97)], + }, + ], + load_model: load, + }; +} + +function show(load: LoadModelView | null) { + return render(ForecastPanel, { props: { accuracy: accuracy(load), pv: null, openMeteo: false } }); +} + +describe('ForecastPanel: ML vs profile', () => { + it('lists the ML model as its own accuracy row', () => { + show(model()); + const table = screen.getByTestId('fc-accuracy'); + expect(within(table).getByText('Base load profile')).toBeTruthy(); + expect(within(table).getByText('Machine-learning load model')).toBeTruthy(); + }); + + it('shadow: compares both errors, the plan keeps the profiles', () => { + show(model()); + expect(text('fc-ml-mode')).toBe('Mode: comparison only'); + expect(text('fc-ml-in-use')).toBe('The plan uses the profiles.'); + expect(text('fc-ml-reason')).toMatch(/runs for comparison only \(12 of 30 days of history\)/); + const compare = screen.getByTestId('fc-ml-compare'); + const profile = compare.querySelector('tr[data-model="profile"]')?.textContent ?? ''; + const ml = compare.querySelector('tr[data-model="ml"]')?.textContent ?? ''; + expect(profile).toMatch(/118/); + expect(profile).toMatch(/1,344 values/); + expect(ml).toMatch(/not scored yet/); + expect(text('fc-ml-days')).toBe('12 of 30 days of history'); + }); + + it('auto: says when the plan uses the ML model and why', () => { + show( + model({ + mode: 'auto', + in_use: 'ml', + reason: { + key: 'forecast.load_model.ml_in_use', + args: { ml_mae_w: 97, profile_mae_w: 118 }, + }, + ml_days: 45, + ml_mae_w: 97.2, + ml_n: 1344, + }), + ); + expect(text('fc-ml-mode')).toBe('Mode: automatic'); + expect(text('fc-ml-in-use')).toBe('The plan uses the machine-learning model.'); + expect(text('fc-ml-reason')).toMatch(/from the machine-learning model.*97 W against 118 W/); + expect(text('fc-ml-days')).toBe('45 of 30 days of history'); + }); + + it('auto with too little history: explains the fallback to the profiles', () => { + show( + model({ + mode: 'auto', + ml_ready: false, + ml_days: 3, + reason: { key: 'forecast.load_model.ml_collecting', args: { days: 3, min_days: 30 } }, + }), + ); + expect(text('fc-ml-in-use')).toBe('The plan uses the profiles.'); + expect(text('fc-ml-reason')).toMatch(/has 3 of the 30 days of history it needs/); + expect(text('fc-ml-days')).toMatch(/Not enough history to forecast yet/); + }); + + it('off: no comparison', () => { + show(model({ mode: 'off', reason: { key: 'forecast.load_model.ml_off', args: {} } })); + expect(text('fc-ml-mode')).toBe('Mode: off'); + expect(screen.queryByTestId('fc-ml-compare')).toBeNull(); + expect(text('fc-ml-reason')).toMatch(/is off/); + }); + + it('without a load forecast only the explanation is shown', () => { + show(null); + expect(text('fc-ml')).toMatch(/similar past days/); + expect(screen.queryByTestId('fc-ml-mode')).toBeNull(); + }); + + it('is translated', () => { + applyLanguage('de'); + show( + model({ + mode: 'auto', + reason: { + key: 'forecast.load_model.ml_not_better', + args: { ml_mae_w: 120, profile_mae_w: 118 }, + }, + }), + ); + expect(text('fc-ml-in-use')).toBe('Der Plan nutzt die Profile.'); + expect(text('fc-ml-reason')).toMatch(/nicht deutlich besser/); + }); +}); diff --git a/frontend/src/lib/i18n/locales/de.json b/frontend/src/lib/i18n/locales/de.json index 7a34cca..d3afc11 100644 --- a/frontend/src/lib/i18n/locales/de.json +++ b/frontend/src/lib/i18n/locales/de.json @@ -1106,6 +1106,7 @@ "fc.no_records": "Noch keine Genauigkeitswerte. Sie werden jede Nacht berechnet, sobald genug Viertelstunden verglichen wurden.", "fc.source.blended": "Mischung (vom Plan genutzt)", "fc.source.load_profile": "Grundlastprofil", + "fc.source.load_ml": "Maschinelles Lastmodell", "fc.weights.title": "Mischgewichte", "fc.weights.intro": "Anteil jeder Solarquelle an der Mischung, je Vorlaufzeit.", "fc.weights.basis.equal_no_history": "gleiche Anteile, bis genug Verlauf vorliegt", @@ -1124,7 +1125,24 @@ "fc.fresh.age": "abgerufen {time}", "fc.fresh.stale": "veraltet", "fc.ml.title": "Maschinelles Lastmodell", - "fc.ml.none": "Noch nicht verfügbar. Die Lastprognose nutzt die Profile für Werktage, Wochenenden und Feiertage.", + "fc.ml.intro": "Ein optionales Modell, das die Grundlast aus ähnlichen vergangenen Tagen vorhersagt (Tagesart, Uhrzeit, Außentemperatur, jüngstes Lastniveau). Es wird neben den Profilen bewertet. Der Plan nutzt es nur im automatischen Modus und nur, solange es in den letzten 14 Tagen deutlich genauer war.", + "fc.ml.mode": "Modus: {mode, select, off {aus} shadow {nur Vergleich} other {automatisch}}", + "fc.ml.in_use": "Der Plan nutzt {model, select, ml {das maschinelle Modell} other {die Profile}}.", + "fc.ml.compare_caption": "Mittlerer absoluter Fehler (MAE) der letzten 14 Tage, alle Vorlaufzeiten", + "fc.ml.model": "Modell", + "fc.ml.error": "MAE", + "fc.ml.profile": "Profile", + "fc.ml.ml": "Maschinelles Modell", + "fc.ml.not_scored": "noch nicht bewertet", + "fc.ml.days": "{days, number} von {min, number} Tagen Verlauf", + "fc.ml.collecting_model": "Noch zu wenig Verlauf für eine Prognose.", + "forecast.load_model.ml_off": "Das maschinelle Lastmodell ist aus; die Prognose nutzt die Profile.", + "forecast.load_model.ml_shadow": "Das maschinelle Lastmodell läuft nur zum Vergleich ({days, number} von {min_days, number} Tagen Verlauf); die Prognose nutzt die Profile.", + "forecast.load_model.ml_collecting": "Lastprognose aus den Profilen: Das maschinelle Modell hat {days, number} der nötigen {min_days, number} Tage Verlauf.", + "forecast.load_model.ml_unavailable": "Lastprognose aus den Profilen: Das maschinelle Modell hat gerade keine Prognose.", + "forecast.load_model.ml_not_scored": "Lastprognose aus den Profilen: Das maschinelle Modell wurde an {n, number} Viertelstunden bewertet, für einen fairen Vergleich sind {needed, number} nötig.", + "forecast.load_model.ml_not_better": "Lastprognose aus den Profilen: Über 14 Tage war das maschinelle Modell nicht deutlich besser (Fehler {ml_mae_w, number, ::0} W gegenüber {profile_mae_w, number, ::0} W).", + "forecast.load_model.ml_in_use": "Lastprognose aus dem maschinellen Modell: Über 14 Tage war es genauer als die Profile (Fehler {ml_mae_w, number, ::0} W gegenüber {profile_mae_w, number, ::0} W).", "fc.attribution": "Solardaten von Open-Meteo.com, lizenziert unter CC BY 4.0.", "fc.attribution.link": "Open-Meteo.com", "fc.attribution.license": "CC BY 4.0", @@ -1200,6 +1218,11 @@ "pvf.load_hint": "Optional: Ein Arbeitstag-Sensor und eine Wetter-Entität verbessern die Prognose der Hauslast. Ohne sie nutzt SunCore Wochentage und die eigene Aufzeichnung der Außentemperatur.", "pvf.workday": "Arbeitstag-Sensor", "pvf.weather": "Wetter-Entität", + "pvf.load_ml": "Maschinelles Lastmodell", + "pvf.load_ml.off": "Aus", + "pvf.load_ml.shadow": "Nur Vergleich (Standard)", + "pvf.load_ml.auto": "Automatisch: nutzen, wenn es genauer ist", + "pvf.load_ml_hint": "Das Modell lernt aus ähnlichen vergangenen Tagen. Im automatischen Modus nutzt der Plan es erst nach {min, number} Tagen Verlauf und nur, solange es die Profile in den letzten 14 Tagen schlägt.", "pvf.open_meteo_note": "Open-Meteo-Daten stehen unter CC BY 4.0; die Namensnennung steht auf der Plan-Seite.", "bat.control": "Steuerung", "bat.control_hint": "SunCore kann den Akku nur beobachten oder auch seine Grenzen über Home-Assistant-Entitäten setzen. Schreiben beginnt immer im Schattenmodus und läuft über die Sicherheitsschicht.", diff --git a/frontend/src/lib/i18n/locales/en.json b/frontend/src/lib/i18n/locales/en.json index 21fed68..dd5878f 100644 --- a/frontend/src/lib/i18n/locales/en.json +++ b/frontend/src/lib/i18n/locales/en.json @@ -1106,6 +1106,7 @@ "fc.no_records": "No accuracy figures yet. They are computed every night once enough quarter hours have been compared.", "fc.source.blended": "Blend (used by the plan)", "fc.source.load_profile": "Base load profile", + "fc.source.load_ml": "Machine-learning load model", "fc.weights.title": "Blend weights", "fc.weights.intro": "Share of each solar source in the blend, per lead time.", "fc.weights.basis.equal_no_history": "equal shares until enough history exists", @@ -1124,7 +1125,24 @@ "fc.fresh.age": "fetched {time}", "fc.fresh.stale": "stale", "fc.ml.title": "Machine-learning load model", - "fc.ml.none": "Not available yet. The load forecast uses the profiles of workdays, weekends and holidays.", + "fc.ml.intro": "An optional model that predicts the base load from similar past days (day type, time of day, outdoor temperature, recent load level). It is scored next to the profiles. The plan uses it only in the automatic mode, and only while it was clearly more accurate over the last 14 days.", + "fc.ml.mode": "Mode: {mode, select, off {off} shadow {comparison only} other {automatic}}", + "fc.ml.in_use": "The plan uses {model, select, ml {the machine-learning model} other {the profiles}}.", + "fc.ml.compare_caption": "Mean absolute error (MAE) over the last 14 days, all lead times", + "fc.ml.model": "Model", + "fc.ml.error": "MAE", + "fc.ml.profile": "Profiles", + "fc.ml.ml": "Machine-learning model", + "fc.ml.not_scored": "not scored yet", + "fc.ml.days": "{days, number} of {min, number} days of history", + "fc.ml.collecting_model": "Not enough history to forecast yet.", + "forecast.load_model.ml_off": "The machine-learning load model is off; the forecast uses the profiles.", + "forecast.load_model.ml_shadow": "The machine-learning load model runs for comparison only ({days, number} of {min_days, number} days of history); the forecast uses the profiles.", + "forecast.load_model.ml_collecting": "Load forecast from the profiles: the machine-learning model has {days, number} of the {min_days, number} days of history it needs.", + "forecast.load_model.ml_unavailable": "Load forecast from the profiles: the machine-learning model has no forecast right now.", + "forecast.load_model.ml_not_scored": "Load forecast from the profiles: the machine-learning model has been scored on {n, number} quarter hours, {needed, number} are needed for a fair comparison.", + "forecast.load_model.ml_not_better": "Load forecast from the profiles: over 14 days the machine-learning model was not clearly better (error {ml_mae_w, number, ::0} W against {profile_mae_w, number, ::0} W).", + "forecast.load_model.ml_in_use": "Load forecast from the machine-learning model: over 14 days it was more accurate than the profiles (error {ml_mae_w, number, ::0} W against {profile_mae_w, number, ::0} W).", "fc.attribution": "Solar data from Open-Meteo.com, licensed under CC BY 4.0.", "fc.attribution.link": "Open-Meteo.com", "fc.attribution.license": "CC BY 4.0", @@ -1200,6 +1218,11 @@ "pvf.load_hint": "Optional: a workday sensor and a weather entity improve the house load forecast. Without them SunCore uses weekdays and its own recording of the outdoor temperature.", "pvf.workday": "Workday sensor", "pvf.weather": "Weather entity", + "pvf.load_ml": "Machine-learning load model", + "pvf.load_ml.off": "Off", + "pvf.load_ml.shadow": "Comparison only (default)", + "pvf.load_ml.auto": "Automatic: use it when it is more accurate", + "pvf.load_ml_hint": "The model learns from similar past days. In the automatic mode the plan uses it only after {min, number} days of history and only while it beats the profiles over the last 14 days.", "pvf.open_meteo_note": "Open-Meteo data is licensed under CC BY 4.0; the attribution is shown on the plan page.", "bat.control": "Control", "bat.control_hint": "SunCore can only observe the battery, or also set its limits through Home Assistant entities. Writing always starts in shadow mode and passes the safety layer.", diff --git a/frontend/src/lib/views/steps/PvStep.svelte b/frontend/src/lib/views/steps/PvStep.svelte index 4cb82d3..72aacca 100644 --- a/frontend/src/lib/views/steps/PvStep.svelte +++ b/frontend/src/lib/views/steps/PvStep.svelte @@ -14,10 +14,23 @@ import { SaveState, wizard } from '../../stores/wizard.svelte'; import { issuesAt } from '../../wizard/issues'; import { emptyPvRow } from '../../wizard/siteForm'; - import { ensureRoleFields, setForecastTemplate } from '../../wizard/forecastForm'; + import { + DEFAULT_LEARNING, + ensureRoleFields, + loadMlMode, + setForecastTemplate, + setLoadMlMode, + type LoadMlMode, + } from '../../wizard/forecastForm'; const save = new SaveState(); const UNIT = { kwp: 'kWp', deg: '°' } as const; + const ML_MODES: readonly LoadMlMode[] = ['off', 'shadow', 'auto']; + const ML_LABEL = { + off: 'pvf.load_ml.off', + shadow: 'pvf.load_ml.shadow', + auto: 'pvf.load_ml.auto', + } as const; let forecasts = $state.raw([]); let details = $state.raw>({}); @@ -313,6 +326,24 @@ bind:field={wizard.forecast.weather} optional /> +
+ + +

+ {t('pvf.load_ml_hint', { + min: (wizard.forecast.learning ?? DEFAULT_LEARNING).min_history_days, + })} +

+
diff --git a/frontend/src/lib/wizard/forecastForm.ts b/frontend/src/lib/wizard/forecastForm.ts index 13feb56..d2ed94f 100644 --- a/frontend/src/lib/wizard/forecastForm.ts +++ b/frontend/src/lib/wizard/forecastForm.ts @@ -1,12 +1,18 @@ // SPDX-License-Identifier: Apache-2.0 // Forecast section (v2) <-> form: the PV forecast sources (templates plus the entities of their // roles, weight, Forecast.Solar config entry) and the load forecast inputs (workday sensor, -// weather entity). Independent of the tariff, so the choice can be stored without prices. The -// learning settings have no form fields yet and are carried through unchanged. +// weather entity, the mode of the machine-learning load model). Independent of the tariff, so the +// choice can be stored without prices. `min_history_days` has no form field and is carried +// through unchanged. import type { ForecastConfig } from '../api/types'; import { emptyRef, fromRef, toRef, type RefField } from './forms'; type Source = NonNullable[number]; +type Learning = NonNullable; +export type LoadMlMode = Learning['load_ml_mode']; + +/** Defaults of the `learning` block (backend `LearningSettings`). */ +export const DEFAULT_LEARNING: Learning = { min_history_days: 30, load_ml_mode: 'shadow' }; export interface ForecastForm { templateIds: string[]; @@ -73,7 +79,7 @@ export function forecastFromForm(form: ForecastForm): ForecastConfig { return { schema_version: 2, sources, - learning: form.learning ?? { min_history_days: 30, load_ml_mode: 'shadow' }, + learning: form.learning ?? DEFAULT_LEARNING, load: { workday: toRef(form.workday, false), weather: toRef(form.weather, false) }, }; } @@ -102,3 +108,13 @@ export function ensureRoleFields(form: ForecastForm, id: string, roles: readonly const own = (form.entities[id] ??= {}); for (const role of roles) own[role] ??= emptyRef(); } + +/** Mode of the machine-learning load model as the form holds it. */ +export function loadMlMode(form: ForecastForm): LoadMlMode { + return (form.learning ?? DEFAULT_LEARNING).load_ml_mode; +} + +/** Set the mode of the machine-learning load model, keeping the other learning settings. */ +export function setLoadMlMode(form: ForecastForm, mode: LoadMlMode): void { + form.learning = { ...(form.learning ?? DEFAULT_LEARNING), load_ml_mode: mode }; +} diff --git a/frontend/src/lib/wizard/phase4Forms.test.ts b/frontend/src/lib/wizard/phase4Forms.test.ts index 49888a4..c9da40b 100644 --- a/frontend/src/lib/wizard/phase4Forms.test.ts +++ b/frontend/src/lib/wizard/phase4Forms.test.ts @@ -10,7 +10,12 @@ import { validateBatteryForm, } from './batteryForm'; import { dirtySections } from './dirty'; -import { forecastFormFromConfig, forecastFromForm } from './forecastForm'; +import { + forecastFormFromConfig, + forecastFromForm, + loadMlMode, + setLoadMlMode, +} from './forecastForm'; import { emptyWindow, isWindowValid, toggleDay, toggleMonth } from './scheduleForm'; import { regionFormFromConfig } from './regionForm'; import { emptySiteForm } from './siteForm'; @@ -149,6 +154,19 @@ describe('forecast form (phase 4 fields)', () => { it('keeps the config entry, weights and the load inputs', () => { expect(forecastFromForm(ffc(cfg))).toEqual(cfg); }); + it('sets the mode of the machine-learning load model and keeps the history days', () => { + const form = forecastFormFromConfig({ + ...cfg, + learning: { min_history_days: 45, load_ml_mode: 'shadow' }, + }); + expect(loadMlMode(form)).toBe('shadow'); + setLoadMlMode(form, 'auto'); + expect(forecastFromForm(form).learning).toEqual({ min_history_days: 45, load_ml_mode: 'auto' }); + const fresh = forecastFormFromConfig(null); + expect(loadMlMode(fresh)).toBe('shadow'); + setLoadMlMode(fresh, 'off'); + expect(forecastFromForm(fresh).learning).toEqual({ min_history_days: 30, load_ml_mode: 'off' }); + }); it('an unset manual weight falls back to automatic instead of an invalid document', () => { const form = forecastFormFromConfig(cfg); form.weights['solcast.pv'] = { weight_mode: 'manual', weight: null }; diff --git a/scripts/bench_load_ml.py b/scripts/bench_load_ml.py new file mode 100644 index 0000000..39d86f5 --- /dev/null +++ b/scripts/bench_load_ml.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Benchmark of the optional KNN load model (Phase 4 WP12; plan §4 "ML training/prediction"). + +A synthetic house (a weekday/weekend profile, a heating term on a daily outdoor temperature, +a level shift after half the history and Gaussian noise; seeded, so every run sees the same data) +with 30, 60 and 90 days of 15-minute history. Prints the median of ``--runs`` runs of the +"training" (:func:`~suncore.forecast.load_ml.fit_knn_model`: building the samples and their +features) and of the prediction of 48 h (:func:`~suncore.forecast.load_ml.forecast_knn`, 193 +quarter hours with P10/P50/P90). Budget: training + prediction < 1 s (plan WP12); CI checks it +on x86 (``tests/forecast/test_load_ml_bench.py``), aarch64 is ``UNVERIFIED`` until it is run on +a Pi or the CI arm runner. + +Usage: ``uv run python scripts/bench_load_ml.py [--runs 5] [--days 30 60 90] [--json]``. +""" + +from __future__ import annotations + +import argparse +import json +import math +import platform +import random +import statistics +import sys +import time +from datetime import UTC, datetime, timedelta +from zoneinfo import ZoneInfo + +from suncore.core.series import SeriesUnit, TimeSeries, planning_grid +from suncore.forecast.load_ml import fit_knn_model, forecast_knn + +TZ = ZoneInfo("Europe/Berlin") +STEP = timedelta(minutes=15) +END = datetime(2026, 11, 2, 10, 5, tzinfo=TZ).astimezone(UTC) # a Monday morning +BUDGET_S = 1.0 + + +def synthetic(days: int, seed: int = 7) -> tuple[TimeSeries, TimeSeries, datetime]: + """Base load (W) and outdoor temperature (°C) of ``days`` complete days before ``END``.""" + rng = random.Random(seed) + midnight = datetime.combine(END.astimezone(TZ).date(), datetime.min.time(), tzinfo=TZ) + start = (midnight - timedelta(days=days)).astimezone(UTC) + count = (midnight.astimezone(UTC) - start) // STEP + daily_temp: dict[object, float] = {} + load: list[float | None] = [] + temp: list[float | None] = [] + for i in range(count): + moment = (start + STEP * i).astimezone(TZ) + day = moment.date() + if day not in daily_temp: + daily_temp[day] = 12.0 + 8.0 * math.sin(i / 96 / 20) + rng.gauss(0, 3) + hour = moment.hour + moment.minute / 60 + shape = 250.0 + 350.0 * math.exp(-((hour - 19.5) ** 2) / 4) + if moment.weekday() >= 5: + shape += 200.0 * math.exp(-((hour - 12.0) ** 2) / 6) + heating = 25.0 * max(0.0, 15.0 - daily_temp[day]) + shift = 80.0 if i > count // 2 else 0.0 + load.append(max(0.0, shape + heating + shift + rng.gauss(0, 60))) + temp.append(daily_temp[day] + 4.0 * math.sin((hour - 9) / 24 * 2 * math.pi)) + return ( + TimeSeries(start=start, step=STEP, values=tuple(load), unit=SeriesUnit.W), + TimeSeries(start=start, step=STEP, values=tuple(temp), unit=SeriesUnit.CELSIUS), + start, + ) + + +def bench(runs: int, days_list: list[int]) -> dict[str, object]: + results: dict[str, object] = { + "machine": f"{platform.machine()} {platform.python_implementation()} " + f"{platform.python_version()}", + "runs": runs, + "budget_s": BUDGET_S, + } + for days in days_list: + base, temperature, _ = synthetic(days) + grid = planning_grid(END, timedelta(hours=48)) + fits: list[float] = [] + predictions: list[float] = [] + samples = 0 + for _ in range(runs): + started = time.perf_counter() + model = fit_knn_model( + base, now=END, tz=TZ, day_types={}, temperature=temperature, history_days=days + ) + fits.append(time.perf_counter() - started) + started = time.perf_counter() + forecast = forecast_knn(model, grid, TZ, {}, {END.astimezone(TZ).date(): 10.0}) + predictions.append(time.perf_counter() - started) + assert forecast is not None + samples = model.samples + fit_s = statistics.median(fits) + predict_s = statistics.median(predictions) + results[f"{days}d"] = { + "samples": samples, + "slots": grid.count, + "fit_s": round(fit_s, 4), + "predict_s": round(predict_s, 4), + "total_s": round(fit_s + predict_s, 4), + } + return results + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--days", type=int, nargs="+", default=[30, 60, 90]) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + result = bench(max(1, args.runs), args.days) + if args.json: + json.dump(result, sys.stdout, indent=2) + sys.stdout.write("\n") + else: + for key, value in result.items(): + sys.stdout.write(f"{key:>10}: {value}\n") + totals = [v["total_s"] for v in result.values() if isinstance(v, dict)] + return 0 if all(t < BUDGET_S for t in totals) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/suncore/api/app.py b/suncore/api/app.py index a50f924..8bd307a 100644 --- a/suncore/api/app.py +++ b/suncore/api/app.py @@ -312,7 +312,11 @@ async def forecast_accuracy() -> ForecastAccuracyView: days, plus the source id ``blended`` (the blended forecast); ``bias_w = actual - forecast`` (positive: the source under-forecasts); ``n`` counts 15-minute intervals; clipped quarter hours are excluded; computed nightly. The source id ``load_profile`` is - the base load forecast (scored on every quarter hour, not only in daylight).""" + the base load forecast (scored on every quarter hour, not only in daylight), ``load_ml`` + the optional machine-learning (KNN) load model scored the same way unless + ``forecast.learning.load_ml_mode`` is ``off``. ``load_model`` compares the two ("ML vs + profile": pooled 14-day MAE) and says which one the planner uses and why + (``forecast.load_model.*`` reason); ``null`` without a load forecast.""" return await services.forecast_accuracy() @app.get(f"{API_PREFIX}/forecast/load", response_model=LoadForecastView) diff --git a/suncore/app/adapters.py b/suncore/app/adapters.py index 67bce42..29b2a68 100644 --- a/suncore/app/adapters.py +++ b/suncore/app/adapters.py @@ -34,7 +34,7 @@ import structlog -from suncore.core.config import ForecastConfig, RegionConfig, TariffConfig +from suncore.core.config import ForecastConfig, LearningSettings, RegionConfig, TariffConfig from suncore.core.site import SiteConfig from suncore.forecast.accuracy import LearnedParams from suncore.forecast.load import ( @@ -209,10 +209,13 @@ def build_load_forecast_service( if data is None: return None load = None if config is None else config.load + learning = LearningSettings() if config is None else config.learning inputs = LoadInputs( device_ids=device_ids, workday_entity=None if load is None or load.workday is None else load.workday.entity_id, weather_entity=None if load is None or load.weather is None else load.weather.entity_id, + ml_mode=learning.load_ml_mode, + min_history_days=learning.min_history_days, ) return LoadForecastService( data, diff --git a/suncore/app/runtime.py b/suncore/app/runtime.py index 4307f35..1e4cd70 100644 --- a/suncore/app/runtime.py +++ b/suncore/app/runtime.py @@ -83,7 +83,6 @@ run_system_check, ) from suncore.forecast.accuracy import accuracy_view, load_accuracy -from suncore.forecast.load import LOAD_SOURCE_ID from suncore.forecast.load_service import LoadForecastService from suncore.forecast.pv.calibration import load_bias from suncore.forecast.pv.clearsky import plant_geometry @@ -386,7 +385,11 @@ async def _collect_planning(self, now: datetime, previous: Plan | None) -> Plann prices = None if self.tariff is None else self.tariff.prices(48.0, now) blended = None if self.pv_service is None else self.pv_service.blended(now) pv = None if blended is None else blended.series - load = None if self.load_service is None else self.load_service.forecast(now) + load: QuantileSeries | None = None + load_reasons: tuple[Reason, ...] = () + if self.load_service is not None: + load, choice = self.load_service.selected(now) + load_reasons = self.load_service.planning_reasons(choice) risk = self._risk(now, battery, pv, load) devices = [] if self.devices is None else list(self.devices.devices) inputs = build_inputs( @@ -406,6 +409,7 @@ async def _collect_planning(self, now: datetime, previous: Plan | None) -> Plann import_limit_w=site.grid.import_limit_w, export_limit_w=export_limit_w(site), previous=previous, + load_reasons=load_reasons, ) complete = ( (prices is None or any(v is not None for v in prices.import_price.values)) @@ -920,9 +924,12 @@ async def pv_forecast(self, hours: int) -> PvForecastView: async def forecast_accuracy(self) -> ForecastAccuracyView: ids = [] if self.pv_service is None else [*self.pv_service.source_ids, BLENDED_SOURCE_ID] if self.load_service is not None: - ids.append(LOAD_SOURCE_ID) + ids.extend(self.load_service.source_ids) stored = {} if self.learned is None else await load_accuracy(self.learned, ids) - return accuracy_view(ids, stored) + view = accuracy_view(ids, stored) + if self.load_service is None: + return view + return view.model_copy(update={"load_model": self.load_service.model_view()}) def next_maintenance_after(self, moment: datetime) -> datetime: """The first ``maintenance_time`` (site time zone, DST safe) strictly after ``moment``.""" diff --git a/suncore/forecast/load_ml.py b/suncore/forecast/load_ml.py new file mode 100644 index 0000000..05161af --- /dev/null +++ b/suncore/forecast/load_ml.py @@ -0,0 +1,519 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Optional machine-learning load model: k nearest neighbours on analogue days (Phase 4 WP12). + +NumPy only (no scikit-learn: its musl/aarch64 wheels are ``UNVERIFIED``, plan §0). Pure +functions; the service (:mod:`suncore.forecast.load_service`) reads the history, scores the +model next to the profiles and decides which one the planner gets (:func:`choose_load_model`). + +**Samples** (the "training"): every quarter hour of every complete local day of the base load +history (the same base load as the profiles: house minus the controlled devices, see +:mod:`suncore.forecast.load`) with at least ``DAY_COVERAGE`` of its quarter hours known, at most +``ML_HISTORY_DAYS`` back (or ``min_history_days`` if that is longer). Each sample carries the +features of its day and slot: + +* **day type** (:class:`~suncore.forecast.load.DayType`, from the same ``workday`` sensor / + ``check_date`` / weekday rule as the profiles); a holiday is close to a weekend day and far + from a workday - the profiles' "a holiday looks like a Sunday" rule as a distance, +* **slot of day** (wall-clock quarter hour, circular), only neighbours within + ``SLOT_WINDOW`` slots are candidates, +* **temperature**: the day's mean outside temperature (recorded ``weather.temperature_c``) - only + used when the forecast day has one, +* **recent load level**: the mean base load of the ``LEVEL_DAYS`` days before the day (a whole + week, so the level does not depend on the weekday pattern), +* **age** in days, a weak term that prefers recent analogues and makes ties deterministic. + +**Prediction** of a forecast quarter hour: the ``K_NEIGHBOURS`` samples with the smallest +distance (Euclidean over the scaled features, :func:`_distances`), sorted stably (equal distances +keep the sample order: fully deterministic), weighted ``1 / (WEIGHT_EPS + distance)``. P50 is the +weighted median of their values, P10/P90 the weighted 10 %/90 % quantiles (the same +interpolation as :func:`suncore.forecast.stats.weighted_quantile`), clipped at 0 W - so +``0 <= p10 <= p50 <= p90`` always holds, on the same 15-minute grid and unit (W) as the profile +forecast. + +**Model choice** (:func:`choose_load_model`, ``forecast.learning.load_ml_mode``): ``off`` - not +computed; ``shadow`` (default) - computed, stored and scored as source ``load_ml``, never used; +``auto`` - used only while it has at least ``min_history_days`` days of history **and** its +rolling 14-day MAE (the same nightly statistic as the profiles', pooled over the horizon buckets) +is at least ``ML_MARGIN`` better than the profiles' on a comparable number of scored quarter hours +(plan §0: "only if measurably better"); otherwise the profiles. Every outcome carries a +:class:`~suncore.core.decisions.Reason`. +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from datetime import UTC, date, datetime, time, timedelta, tzinfo +from typing import Final, Literal + +import numpy as np +import numpy.typing as npt + +from suncore.core.decisions import Reason +from suncore.core.forecast import ACCURACY_MIN_SAMPLES, AccuracyRecord +from suncore.core.series import Grid, QuantileSeries, SeriesUnit, TimeSeries +from suncore.forecast.days import local_midnight +from suncore.forecast.load import ( + DAY_COVERAGE, + MIN_CLEAN_DAYS, + SLOTS_PER_DAY, + TEMPERATURE_DAY_COVERAGE, + DayType, + DayValues, + day_values, + local_slot, + weekday_rule, +) + +LOAD_ML_SOURCE_ID: Final = "load_ml" +"""Source id of the KNN forecast in ``timeseries.db`` and the accuracy statistics.""" + +ML_HISTORY_DAYS: Final = 91 +"""Default look-back of the samples (13 weeks: every weekday 13 times, a season's change).""" +KNN_MIN_DAYS: Final = 7 +"""Usable days before the model forecasts at all (a week: every weekday once).""" +K_NEIGHBOURS: Final = 24 +SLOT_WINDOW: Final = 2 +"""Candidates lie within +-2 quarter hours of the forecast slot (wall clock).""" +SLOT_SCALE: Final = 2.0 +"""Quarter hours per distance unit (one slot off = 0.5).""" +TEMPERATURE_SCALE_K: Final = 4.0 +LEVEL_SCALE: Final = 0.15 +"""Relative level difference per distance unit (``|ln(a / b)|``, 15 %).""" +LEVEL_DAYS: Final = 7 +LEVEL_MIN_SLOTS: Final = 2 * SLOTS_PER_DAY +"""Known quarter hours the level window needs (else the level is unknown).""" +LEVEL_FLOOR_W: Final = 1.0 +AGE_SCALE_DAYS: Final = 120.0 +MISSING_FEATURE: Final = 1.0 +"""Distance of a sample that lacks a feature the forecast day has (temperature, level).""" +WEIGHT_EPS: Final = 0.25 +LOW_QUANTILE: Final = 0.1 +HIGH_QUANTILE: Final = 0.9 +ML_MARGIN: Final = 0.02 +"""``auto`` needs an MAE at least 2 % below the profiles' (measurably better, plan §0).""" +ML_MIN_SCORED_SHARE: Final = 0.9 +"""``auto`` needs the ML forecast scored on at least 90 % as many quarter hours as the profiles +(else it was not stored for most of the 14 days and the two MAEs are not comparable).""" + +_TYPE_INDEX: Final = {DayType.WORKDAY: 0, DayType.WEEKEND: 1, DayType.HOLIDAY: 2} +_TYPE_DISTANCE: Final = np.array( + [ + [0.0, 3.0, 3.0], # workday + [3.0, 0.0, 0.5], # weekend + [3.0, 0.5, 0.0], # holiday: like a weekend day (the profiles' rule), not a workday + ] +) + +FloatArray = npt.NDArray[np.float64] +IntArray = npt.NDArray[np.int64] + +LoadMlMode = Literal["off", "shadow", "auto"] +ModelInUse = Literal["profile", "ml"] + + +@dataclass(frozen=True, slots=True, eq=False) +class KnnLoadModel: + """The samples of the KNN model (column arrays, one entry per known quarter hour).""" + + days: int + """Usable complete local days among the samples.""" + fitted_for: date + """The local date the model was fitted on (ages count from it).""" + recent_level_w: float | None + """Mean base load of the ``LEVEL_DAYS`` days before the fit (the forecast's level).""" + slot: IntArray = field(default_factory=lambda: np.zeros(0, dtype=np.int64)) + day_type: IntArray = field(default_factory=lambda: np.zeros(0, dtype=np.int64)) + temperature: FloatArray = field(default_factory=lambda: np.zeros(0)) + """Daily mean °C of the sample's day (NaN: unknown).""" + level: FloatArray = field(default_factory=lambda: np.zeros(0)) + """Level (W) of the sample's day (NaN: unknown).""" + age_days: FloatArray = field(default_factory=lambda: np.zeros(0)) + value: FloatArray = field(default_factory=lambda: np.zeros(0)) + candidates: tuple[IntArray, ...] = () + """Per slot of the day: the sample indices within ``SLOT_WINDOW`` (ascending).""" + + @property + def ready(self) -> bool: + return self.days >= KNN_MIN_DAYS + + @property + def samples(self) -> int: + return int(self.value.size) + + +# --- fit ----------------------------------------------------------------------------------- + + +def _usable_days( + base: TimeSeries, + *, + tz: tzinfo, + until: datetime, + unmeasured: Sequence[bool], + oldest: date, +) -> list[DayValues]: + """Complete days with ``DAY_COVERAGE`` from ``oldest`` on; quarter hours with an unmeasured + device are left out once ``MIN_CLEAN_DAYS`` days are fully measured (else kept, as the + profiles do before they have clean days).""" + days = [ + d for d in day_values(base, tz, until) if d.coverage >= DAY_COVERAGE and d.day >= oldest + ] + if not any(unmeasured): + return days + masked_values = tuple( + None if i < len(unmeasured) and unmeasured[i] else v for i, v in enumerate(base.values) + ) + masked = { + d.day: d for d in day_values(base.model_copy(update={"values": masked_values}), tz, until) + } + clean = sum(1 for d in days if masked[d.day].slots == d.slots) + if clean < MIN_CLEAN_DAYS: + return days + return [masked[d.day] for d in days if masked[d.day].known()] + + +def daily_temperatures(temperature: TimeSeries, tz: tzinfo, until: datetime) -> dict[date, float]: + """Mean temperature of every complete local day with ``TEMPERATURE_DAY_COVERAGE`` known.""" + out: dict[date, float] = {} + for d in day_values(temperature, tz, until): + known = [v for _, v in d.known()] + if known and d.coverage >= TEMPERATURE_DAY_COVERAGE: + out[d.day] = sum(known) / len(known) + return out + + +def _as_array(series: TimeSeries) -> FloatArray: + return np.array([math.nan if v is None else v for v in series.values], dtype=np.float64) + + +def _window_mean(values: FloatArray, first: int, last: int) -> float | None: + """Mean of the known values in ``values[first:last]`` (clamped) with ``LEVEL_MIN_SLOTS``.""" + first, last = max(0, first), min(values.size, last) + if last <= first: + return None + window = values[first:last] + known = window[~np.isnan(window)] + if known.size < LEVEL_MIN_SLOTS: + return None + return float(known.mean()) + + +def fit_knn_model( + base: TimeSeries, + *, + now: datetime, + tz: tzinfo, + day_types: Mapping[date, DayType], + temperature: TimeSeries | None = None, + unmeasured: Sequence[bool] = (), + history_days: int = ML_HISTORY_DAYS, +) -> KnnLoadModel: + """The samples (module doc) from the base load history up to the local midnight of ``now``. + + ``day_types`` maps history dates to their type (missing: weekday rule); ``temperature`` is + the recorded outside temperature (°C, 15 minutes); ``unmeasured`` flags slots of ``base`` + with an unmeasured device (:attr:`~suncore.forecast.load.BaseLoad.unmeasured`). + """ + today = now.astimezone(tz).date() + midnight = local_midnight(now, tz).astimezone(UTC) + days = _usable_days( + base, + tz=tz, + until=midnight, + unmeasured=unmeasured, + oldest=today - timedelta(days=history_days), + ) + values = _as_array(base) + step = base.step + level_span = (timedelta(days=LEVEL_DAYS)) // step + + def index_of(moment: datetime) -> int: + return (moment - base.start) // step + + now_index = index_of(min(now, base.end)) if len(base) else 0 + recent = _window_mean(values, now_index - level_span, now_index) if len(base) else None + temps = {} if temperature is None else daily_temperatures(temperature, tz, midnight) + + slots: list[int] = [] + kinds: list[int] = [] + temp_col: list[float] = [] + level_col: list[float] = [] + ages: list[float] = [] + vals: list[float] = [] + for d in days: + start = datetime.combine(d.day, time(), tzinfo=tz).astimezone(UTC) + level = _window_mean(values, index_of(start) - level_span, index_of(start)) + kind = _TYPE_INDEX[day_types.get(d.day, weekday_rule(d.day))] + day_temp = temps.get(d.day, math.nan) + age = float((today - d.day).days) + for slot, value in d.known(): + slots.append(slot) + kinds.append(kind) + temp_col.append(day_temp) + level_col.append(math.nan if level is None else level) + ages.append(age) + vals.append(max(0.0, value)) + slot_arr = np.array(slots, dtype=np.int64) + by_slot = [np.flatnonzero(slot_arr == s) for s in range(SLOTS_PER_DAY)] + candidates = tuple( + np.sort( + np.concatenate( + [by_slot[(s + o) % SLOTS_PER_DAY] for o in range(-SLOT_WINDOW, SLOT_WINDOW + 1)] + ) + ).astype(np.int64) + for s in range(SLOTS_PER_DAY) + ) + return KnnLoadModel( + days=len(days), + fitted_for=today, + recent_level_w=recent, + slot=slot_arr, + day_type=np.array(kinds, dtype=np.int64), + temperature=np.array(temp_col, dtype=np.float64), + level=np.array(level_col, dtype=np.float64), + age_days=np.array(ages, dtype=np.float64), + value=np.array(vals, dtype=np.float64), + candidates=candidates, + ) + + +# --- predict ------------------------------------------------------------------------------- + + +def _distances( + model: KnnLoadModel, + idx: IntArray, + *, + slot: int, + kind: int, + temperature_c: float | None, + level_w: float | None, +) -> FloatArray: + """Squared scaled distance of the candidates ``idx`` to one forecast quarter hour.""" + offset = np.abs(model.slot[idx] - slot) + offset = np.minimum(offset, SLOTS_PER_DAY - offset) + d2: FloatArray = (offset / SLOT_SCALE) ** 2 + _TYPE_DISTANCE[kind, model.day_type[idx]] ** 2 + d2 = d2 + (model.age_days[idx] / AGE_SCALE_DAYS) ** 2 + if temperature_c is not None: + temps = model.temperature[idx] + term = np.where( + np.isnan(temps), MISSING_FEATURE, np.abs(temps - temperature_c) / TEMPERATURE_SCALE_K + ) + d2 = d2 + term**2 + if level_w is not None: + levels = model.level[idx] + ratio = np.log(np.maximum(levels, LEVEL_FLOOR_W) / max(level_w, LEVEL_FLOOR_W)) + term = np.where(np.isnan(levels), MISSING_FEATURE, np.abs(ratio) / LEVEL_SCALE) + d2 = d2 + term**2 + return d2 + + +def weighted_quantiles( + values: FloatArray, weights: FloatArray, qs: Sequence[float] +) -> tuple[float, ...]: + """Weighted quantiles with the midpoint interpolation of + :func:`suncore.forecast.stats.weighted_quantile` (no extrapolation; monotone in ``q``).""" + order = np.argsort(values, kind="stable") + v, w = values[order], weights[order] + total = float(w.sum()) + positions = (np.cumsum(w) - w / 2) / total + return tuple(float(np.interp(q, positions, v)) for q in qs) + + +def predict_slot( + model: KnnLoadModel, + *, + slot: int, + day_type: DayType, + temperature_c: float | None, + level_w: float | None, +) -> tuple[float, float, float] | None: + """``(p10, p50, p90)`` W of one wall-clock slot; ``None`` without a ready model.""" + if not model.ready or not model.candidates: + return None + idx = model.candidates[slot] + if idx.size == 0: + return None + d2 = _distances( + model, + idx, + slot=slot, + kind=_TYPE_INDEX[day_type], + temperature_c=temperature_c, + level_w=level_w, + ) + nearest = np.argsort(d2, kind="stable")[:K_NEIGHBOURS] + weights = 1.0 / (WEIGHT_EPS + np.sqrt(d2[nearest])) + p10, p50, p90 = weighted_quantiles( + model.value[idx[nearest]], weights, (LOW_QUANTILE, 0.5, HIGH_QUANTILE) + ) + p10, p50, p90 = max(0.0, p10), max(0.0, p50), max(0.0, p90) + return min(p10, p50), p50, max(p50, p90) + + +def forecast_knn( + model: KnnLoadModel, + grid: Grid, + tz: tzinfo, + day_types: Mapping[date, DayType], + daily_temperature: Mapping[date, float] | None = None, +) -> QuantileSeries | None: + """P10/P50/P90 of the base load (W) on ``grid``; ``None`` while the model is not ready.""" + if not model.ready: + return None + temps = daily_temperature or {} + lows: list[float | None] = [] + mids: list[float | None] = [] + highs: list[float | None] = [] + cache: dict[tuple[int, DayType, float | None], tuple[float, float, float] | None] = {} + for begin in grid.starts(): + day, slot = local_slot(begin, tz) + kind = day_types.get(day, weekday_rule(day)) + temp = temps.get(day) + key = (slot, kind, temp) + if key not in cache: # a repeated wall-clock hour (DST) asks the same slot twice + cache[key] = predict_slot( + model, slot=slot, day_type=kind, temperature_c=temp, level_w=model.recent_level_w + ) + q = cache[key] + if q is None: + return None + lows.append(q[0]) + mids.append(q[1]) + highs.append(q[2]) + + def series(values: list[float | None]) -> TimeSeries: + return TimeSeries(start=grid.start, step=grid.step, values=tuple(values), unit=SeriesUnit.W) + + return QuantileSeries(p10=series(lows), p50=series(mids), p90=series(highs)) + + +# --- model choice -------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class PooledError: + """MAE over the horizon buckets of one source (weighted by their quarter hours).""" + + mae_w: float + n: int + + +def pooled_error(records: Sequence[AccuracyRecord]) -> PooledError | None: + n = sum(r.n for r in records) + if n <= 0: + return None + return PooledError(mae_w=sum(r.mae_w * r.n for r in records) / n, n=n) + + +@dataclass(frozen=True, slots=True) +class LoadModelChoice: + """Which load model the planner gets, and why.""" + + mode: LoadMlMode + in_use: ModelInUse + reason: Reason + ml_days: int + min_history_days: int + profile: PooledError | None + ml: PooledError | None + + +def choose_load_model( + mode: LoadMlMode, + *, + ml_days: int, + ml_available: bool, + min_history_days: int, + profile_records: Sequence[AccuracyRecord], + ml_records: Sequence[AccuracyRecord], +) -> LoadModelChoice: + """The model the planner uses (module doc). Pure and deterministic. + + ``ml_days``: usable history days of the KNN model; ``ml_available``: it produced a forecast + now; the records are the stored rolling 14-day accuracy of ``load_profile`` and ``load_ml``. + """ + profile = pooled_error(profile_records) + ml = pooled_error(ml_records) + if mode == "off": + in_use: ModelInUse = "profile" + reason = Reason(key="forecast.load_model.ml_off") + elif mode == "shadow": + in_use = "profile" + reason = Reason( + key="forecast.load_model.ml_shadow", + args={"days": ml_days, "min_days": min_history_days}, + ) + else: + in_use, reason = _auto( + ml_days=ml_days, + ml_available=ml_available, + min_history_days=min_history_days, + profile=profile, + ml=ml, + ) + return LoadModelChoice( + mode=mode, + in_use=in_use, + reason=reason, + ml_days=ml_days, + min_history_days=min_history_days, + profile=profile, + ml=ml, + ) + + +def _auto( + *, + ml_days: int, + ml_available: bool, + min_history_days: int, + profile: PooledError | None, + ml: PooledError | None, +) -> tuple[ModelInUse, Reason]: + """Mode ``auto``: the KNN model only with enough history, a forecast now, a comparable + score and an MAE ``ML_MARGIN`` below the profiles'.""" + if ml_days < min_history_days: + return "profile", Reason( + key="forecast.load_model.ml_collecting", + args={"days": ml_days, "min_days": min_history_days}, + ) + if not ml_available: + return "profile", Reason(key="forecast.load_model.ml_unavailable") + needed = max( + ACCURACY_MIN_SAMPLES, + 0 if profile is None else math.ceil(ML_MIN_SCORED_SHARE * profile.n), + ) + if profile is None or ml is None or ml.n < needed: + return "profile", Reason( + key="forecast.load_model.ml_not_scored", + args={"n": 0 if ml is None else ml.n, "needed": needed}, + ) + args = {"ml_mae_w": round(ml.mae_w), "profile_mae_w": round(profile.mae_w)} # whole W + if ml.mae_w < (1.0 - ML_MARGIN) * profile.mae_w: + return "ml", Reason(key="forecast.load_model.ml_in_use", args=args) + return "profile", Reason(key="forecast.load_model.ml_not_better", args=args) + + +__all__ = [ + "KNN_MIN_DAYS", + "K_NEIGHBOURS", + "LOAD_ML_SOURCE_ID", + "ML_HISTORY_DAYS", + "ML_MARGIN", + "ML_MIN_SCORED_SHARE", + "KnnLoadModel", + "LoadMlMode", + "LoadModelChoice", + "ModelInUse", + "PooledError", + "choose_load_model", + "daily_temperatures", + "fit_knn_model", + "forecast_knn", + "pooled_error", + "predict_slot", + "weighted_quantiles", +] diff --git a/suncore/forecast/load_service.py b/suncore/forecast/load_service.py index c649837..af8712f 100644 --- a/suncore/forecast/load_service.py +++ b/suncore/forecast/load_service.py @@ -25,6 +25,14 @@ ``timeseries.db`` (kind ``load``, source ``load_profile``) at most hourly. * **Nightly**: accuracy (MAE and bias = actual - forecast per horizon bucket, all quarter hours) of ``load_profile`` against the measured base load, then a refit. +* **ML load model** (WP12, :mod:`suncore.forecast.load_ml`, ``forecast.learning.load_ml_mode``): + unless ``off``, the KNN model is fitted with the profiles (from a window of + ``max(ML_HISTORY_DAYS, min_history_days)`` days; the profiles still see only the last + ``history_days``), its forecast is stored hourly as source ``load_ml`` and scored nightly + exactly like ``load_profile``. :meth:`LoadForecastService.forecast` - what the planner and the + risk model get - is the profile forecast unless the mode is ``auto`` and + :func:`~suncore.forecast.load_ml.choose_load_model` selects the KNN model; + :meth:`LoadForecastService.choice` says which one and why. Storage and Home Assistant are reached only through protocols (the composition root passes the time-series store, the learned-parameter store and an adapter); time and sleeping are @@ -36,11 +44,12 @@ import asyncio from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from dataclasses import dataclass -from datetime import UTC, date, datetime, timedelta, tzinfo +from datetime import UTC, date, datetime, time, timedelta, tzinfo from typing import Any, Final, Literal, Protocol import structlog +from suncore.core.decisions import Reason from suncore.core.forecast import AccuracyRecord from suncore.core.measurements import SITE_HOUSE_W, WEATHER_TEMPERATURE_C, device_power_key from suncore.core.series import ( @@ -52,8 +61,13 @@ floor_to_step, planning_grid, ) -from suncore.forecast.accuracy import AccuracyData, LearnedParams, update_accuracy -from suncore.forecast.days import local_midnight +from suncore.forecast.accuracy import ( + AccuracyData, + LearnedParams, + load_accuracy, + update_accuracy, +) +from suncore.forecast.days import local_midnight, slice_series from suncore.forecast.load import ( FORECAST_TEMPERATURE_HOURS, HISTORY_DAYS, @@ -72,10 +86,21 @@ forecast_load, weekday_rule, ) +from suncore.forecast.load_ml import ( + LOAD_ML_SOURCE_ID, + ML_HISTORY_DAYS, + KnnLoadModel, + LoadMlMode, + LoadModelChoice, + choose_load_model, + fit_knn_model, + forecast_knn, +) from suncore.forecast.views import ( MAX_VIEW_HOURS, LoadDayView, LoadForecastView, + LoadModelView, LoadTemperatureView, ) @@ -141,6 +166,12 @@ class LoadInputs: device_ids: tuple[str, ...] = () workday_entity: str | None = None weather_entity: str | None = None + ml_mode: LoadMlMode = "off" + """``forecast.learning.load_ml_mode`` (the composition root passes the configured value, + default ``shadow``; ``off`` here keeps a bare service free of the KNN model).""" + min_history_days: int = 30 + """``forecast.learning.min_history_days``: history ``auto`` needs before the KNN model is + selectable.""" class LoadForecastService: @@ -188,7 +219,11 @@ def __init__( self._weather_at: datetime | None = None self._observed: list[tuple[datetime, float]] = [] """Today's recorded outside temperatures (from the fit's read plus every refresh).""" - self._last_stored_at: datetime | None = None + self._last_stored_at: dict[str, datetime] = {} + self._ml: KnnLoadModel | None = None + self._scores: dict[str, tuple[AccuracyRecord, ...]] = {} + """Latest rolling 14-day accuracy of ``load_profile`` and ``load_ml``.""" + self._scores_loaded = False self._lock = asyncio.Lock() self._listeners: list[Callable[[], None]] = [] @@ -202,6 +237,26 @@ def inputs(self) -> LoadInputs: def model(self) -> LoadModel | None: return self._model + @property + def ml_model(self) -> KnnLoadModel | None: + """The KNN load model (``None``: mode ``off`` or not fitted yet).""" + return self._ml + + @property + def ml_enabled(self) -> bool: + return self._inputs.ml_mode != "off" + + @property + def source_ids(self) -> tuple[str, ...]: + """The load forecast sources that are stored and scored.""" + return (LOAD_SOURCE_ID, LOAD_ML_SOURCE_ID) if self.ml_enabled else (LOAD_SOURCE_ID,) + + def _window_days(self) -> int: + """Days of history read for a fit (the KNN model may look further back).""" + if not self.ml_enabled: + return self._history_days + return max(self._history_days, ML_HISTORY_DAYS, self._inputs.min_history_days) + @property def device_histories(self) -> Mapping[str, TimeSeries]: """The controlled devices' measured power (15 minutes, W) of the fit window.""" @@ -248,7 +303,7 @@ async def _load_cache(self) -> None: async def _save_cache(self, today: date) -> None: if self._learned is None: return - oldest = today - timedelta(days=self._history_days + 7) + oldest = today - timedelta(days=self._window_days() + 7) self._cache = {d: v for d, v in self._cache.items() if d >= oldest} payload = { "fingerprint": self._fingerprint, @@ -314,6 +369,10 @@ async def base_load_series(self, window: TimeRange) -> TimeSeries: return result.series async def _read_base(self, window: TimeRange) -> tuple[BaseLoad, dict[str, TimeSeries]]: + house, devices = await self._read_inputs(window) + return base_load(house, devices), devices + + async def _read_inputs(self, window: TimeRange) -> tuple[TimeSeries, dict[str, TimeSeries]]: assert self._data is not None house = await self._data.read_series(SITE_HOUSE_W, window, "15m", SeriesUnit.W) devices = { @@ -322,7 +381,7 @@ async def _read_base(self, window: TimeRange) -> tuple[BaseLoad, dict[str, TimeS ) for device_id in self._inputs.device_ids } - return base_load(house, devices), devices + return house, devices async def refit(self, now: datetime | None = None) -> LoadModel | None: """Read the history and fit the model (kept on failure).""" @@ -331,24 +390,33 @@ async def refit(self, now: datetime | None = None) -> LoadModel | None: moment = self._clock() if now is None else now tz = self._tz() await self._load_cache() + await self._load_scores() today = moment.astimezone(tz).date() end = floor_to_step(moment, PLANNING_STEP) - start = local_midnight(moment, tz).astimezone(UTC) - timedelta(days=self._history_days) - window = TimeRange(start=start, end=end) - history_days = [today - timedelta(days=i) for i in range(1, self._history_days + 1)] + window_days = self._window_days() + # local midnights (a DST change inside the window must not cut off its first day) + window = TimeRange(start=_local_start(today, window_days, tz), end=end) + profile_start = _local_start(today, self._history_days, tz) + history_days = [today - timedelta(days=i) for i in range(1, window_days + 1)] await self._resolve([*self._forecast_days(moment, tz), *history_days], today) try: - base, devices = await self._read_base(window) + house, devices = await self._read_inputs(window) + full = base_load(house, devices) + base = full + if window_days > self._history_days: # the profiles see only their own window + devices = {k: slice_series(v, profile_start, v.end) for k, v in devices.items()} + base = base_load(slice_series(house, profile_start, house.end), devices) temperature = None if self._inputs.weather_entity is not None: temperature = await self._data.read_series( WEATHER_TEMPERATURE_C, window, "15m", SeriesUnit.CELSIUS ) + day_types = self.day_types(history_days, today) model = fit_load_model( base.series, now=moment, tz=tz, - day_types=self.day_types(history_days, today), + day_types=day_types, temperature=temperature, temperature_configured=self._inputs.weather_entity is not None, unmeasured=base.unmeasured, @@ -360,6 +428,15 @@ async def refit(self, now: datetime | None = None) -> LoadModel | None: return self._model self._model, self._fitted_at = model, moment self._base, self._devices = base, devices + if self.ml_enabled: + self._fit_ml( + full, + moment=moment, + tz=tz, + day_types=day_types, + temperature=temperature, + window_days=window_days, + ) if temperature is not None: midnight = local_midnight(moment, tz) live = [(t, v) for t, v in self._observed if t >= end] # not aggregated yet @@ -377,6 +454,47 @@ async def refit(self, now: datetime | None = None) -> LoadModel | None: ) return model + def _fit_ml( + self, + full: BaseLoad, + *, + moment: datetime, + tz: tzinfo, + day_types: Mapping[date, DayType], + temperature: TimeSeries | None, + window_days: int, + ) -> None: + """Fit the KNN model (a failure keeps the previous one; the profiles are unaffected).""" + try: + ml = fit_knn_model( + full.series, + now=moment, + tz=tz, + day_types=day_types, + temperature=temperature, + unmeasured=full.unmeasured, + history_days=window_days, + ) + except Exception: + log.exception("forecast.load.ml_fit_failed") + return + self._ml = ml + log.info("forecast.load.ml_fitted", days=ml.days, samples=ml.samples) + + async def _load_scores(self) -> None: + """The stored accuracy of both models (once; the nightly job keeps it current).""" + if self._scores_loaded or self._learned is None or not self.ml_enabled: + self._scores_loaded = True + return + self._scores_loaded = True + try: + stored = await load_accuracy(self._learned, self.source_ids) + except Exception: + log.exception("forecast.load.scores_unreadable") + return + for source_id, item in stored.items(): + self._scores.setdefault(source_id, item.records) + # --- temperature ------------------------------------------------------------------ async def _record_temperature(self, now: datetime) -> None: @@ -436,8 +554,8 @@ def _forecast_days(self, now: datetime, tz: tzinfo) -> list[date]: last = (now + self._horizon).astimezone(tz).date() return [first + timedelta(days=i) for i in range((last - first).days + 1)] - def forecast(self, now: datetime | None = None) -> QuantileSeries | None: - """The base load forecast from the current slot for ``horizon`` (``None``: no model).""" + def profile_forecast(self, now: datetime | None = None) -> QuantileSeries | None: + """The profile forecast from the current slot for ``horizon`` (``None``: no model).""" model = self._model if model is None: return None @@ -449,6 +567,69 @@ def forecast(self, now: datetime | None = None) -> QuantileSeries | None: temps = self.daily_temperatures(moment) if model.temperature.active else {} return forecast_load(model, grid, tz, types, temps) + def ml_forecast(self, now: datetime | None = None) -> QuantileSeries | None: + """The KNN forecast on the same grid (``None``: mode ``off`` or too little history).""" + ml = self._ml + if ml is None or not ml.ready or not self.ml_enabled: + return None + moment = self._clock() if now is None else now + tz = self._tz() + grid = planning_grid(moment, self._horizon) + today = moment.astimezone(tz).date() + types = self.day_types(self._forecast_days(moment, tz), today) + temps = self.daily_temperatures(moment) if self._inputs.weather_entity else {} + try: + return forecast_knn(ml, grid, tz, types, temps) + except Exception: + log.exception("forecast.load.ml_forecast_failed") + return None + + def set_scores(self, scores: Mapping[str, Sequence[AccuracyRecord]]) -> None: + """Replace the rolling accuracy the model choice uses (the nightly job's result).""" + for source_id, records in scores.items(): + self._scores[source_id] = tuple(records) + + def choice(self, ml_available: bool | None = None) -> LoadModelChoice: + """Which model :meth:`forecast` returns and why (:func:`choose_load_model`). + + ``ml_available`` defaults to "the KNN model is fitted and ready".""" + ml = self._ml + if ml_available is None: + ml_available = ml is not None and ml.ready + return choose_load_model( + self._inputs.ml_mode, + ml_days=0 if ml is None else ml.days, + ml_available=ml_available, + min_history_days=self._inputs.min_history_days, + profile_records=self._scores.get(LOAD_SOURCE_ID, ()), + ml_records=self._scores.get(LOAD_ML_SOURCE_ID, ()), + ) + + def selected( + self, now: datetime | None = None + ) -> tuple[QuantileSeries | None, LoadModelChoice]: + """The forecast the planner uses and the choice behind it: the KNN forecast only when + the choice selects it and it exists, else the profile forecast.""" + choice = self.choice() + if choice.in_use == "ml": + ml = self.ml_forecast(now) + if ml is not None: + return ml, choice + choice = self.choice(ml_available=False) + return self.profile_forecast(now), choice + + def forecast(self, now: datetime | None = None) -> QuantileSeries | None: + """The base load forecast the planner and the risk model use, from the current slot for + ``horizon`` (``None``: no model): the profiles unless ``auto`` selected the KNN model.""" + return self.selected(now)[0] + + def planning_reasons(self, choice: LoadModelChoice | None = None) -> tuple[Reason, ...]: + """Plan-level reasons of the load model: in ``auto`` which model the plan uses and why + (in ``off``/``shadow`` the plan always uses the profiles, nothing to explain). + ``choice``: the one :meth:`selected` returned with the forecast (default: now).""" + choice = self.choice() if choice is None else choice + return (choice.reason,) if choice.mode == "auto" else () + async def refresh(self) -> QuantileSeries | None: """One refresh (record, refit if due, weather, forecast, store); never raises.""" async with self._lock: @@ -467,9 +648,15 @@ async def refresh(self) -> QuantileSeries | None: else: await self._resolve(self._forecast_days(now, tz), now.astimezone(tz).date()) await self._update_weather(now) - forecast = self.forecast(now) - if forecast is not None: - await self._maybe_store(forecast, now) + profile = self.profile_forecast(now) + ml = self.ml_forecast(now) + if profile is not None: + await self._maybe_store(LOAD_SOURCE_ID, profile, now) + if ml is not None: + await self._maybe_store(LOAD_ML_SOURCE_ID, ml, now) + forecast = profile + if ml is not None and self.choice().in_use == "ml": + forecast = ml except asyncio.CancelledError: raise except Exception: @@ -496,19 +683,20 @@ def _notify(self) -> None: except Exception: log.exception("forecast.load.listener_failed") - async def _maybe_store(self, forecast: QuantileSeries, now: datetime) -> None: + async def _maybe_store(self, source_id: str, forecast: QuantileSeries, now: datetime) -> None: if self._data is None: return - if self._last_stored_at is not None and now - self._last_stored_at < self._store_interval: + last = self._last_stored_at.get(source_id) + if last is not None and now - last < self._store_interval: return try: - await self._data.store_forecast(LOAD_SOURCE_ID, "load", now, forecast) + await self._data.store_forecast(source_id, "load", now, forecast) except asyncio.CancelledError: raise except Exception: - log.exception("forecast.load.store_failed") + log.exception("forecast.load.store_failed", source=source_id) return - self._last_stored_at = now + self._last_stored_at[source_id] = now async def run(self) -> None: """Refresh now, then every ``refresh_interval_s``.""" @@ -519,18 +707,21 @@ async def run(self) -> None: # --- nightly ------------------------------------------------------------------------ async def nightly(self, now: datetime | None = None) -> list[AccuracyRecord]: - """Accuracy of ``load_profile`` (stored in ``forecast_accuracy``) and a refit.""" + """Accuracy of ``load_profile`` (and ``load_ml`` unless the mode is ``off``; stored in + ``forecast_accuracy``, kept for the model choice) and a refit. Returns the profile's + records.""" moment = self._clock() if now is None else now records: list[AccuracyRecord] = [] if self._data is not None and self._learned is not None: results = await update_accuracy( self._data, self._learned, - [LOAD_SOURCE_ID], + list(self.source_ids), moment, kind="load", actual_reader=self.base_load_series, ) + self.set_scores(results) records = results.get(LOAD_SOURCE_ID, []) async with self._lock: await self.refit(moment) @@ -543,7 +734,7 @@ def view(self, hours: int = 48) -> LoadForecastView: hours = min(max(hours, 1), MAX_VIEW_HOURS) tz = self._tz() model = self._model - forecast = self.forecast(now) + forecast, choice = self.selected(now) count = hours * 4 p10: list[float | None] = [] p50: list[float | None] = [] @@ -572,7 +763,7 @@ def view(self, hours: int = 48) -> LoadForecastView: return LoadForecastView( generated_at=now, hours=hours, - source_id=LOAD_SOURCE_ID, + source_id=LOAD_ML_SOURCE_ID if choice.in_use == "ml" else LOAD_SOURCE_ID, quality=LoadQuality.NO_HISTORY if model is None else model.quality, days_of_history=0 if model is None else model.days, fitted_at=self._fitted_at, @@ -603,9 +794,32 @@ def view(self, hours: int = 48) -> LoadForecastView: unmeasured_in_profile=( [] if model is None else sorted(model.unmeasured_kept, key=lambda t: t.value) ), + model=self.model_view(choice), + ) + + def model_view(self, choice: LoadModelChoice | None = None) -> LoadModelView: + """ "ML vs profile": the model in use, why, and both rolling errors.""" + choice = self.choice() if choice is None else choice + ml = self._ml + return LoadModelView( + mode=choice.mode, + in_use=choice.in_use, + reason=choice.reason, + ml_ready=self.ml_enabled and ml is not None and ml.ready, + ml_days=choice.ml_days, + min_history_days=choice.min_history_days, + profile_mae_w=None if choice.profile is None else round(choice.profile.mae_w, 1), + ml_mae_w=None if choice.ml is None else round(choice.ml.mae_w, 1), + profile_n=0 if choice.profile is None else choice.profile.n, + ml_n=0 if choice.ml is None else choice.ml.n, ) +def _local_start(today: date, days: int, tz: tzinfo) -> datetime: + """Local midnight ``days`` before ``today``, in UTC.""" + return datetime.combine(today - timedelta(days=days), time(), tzinfo=tz).astimezone(UTC) + + def _decode_days(payload: Mapping[str, Any]) -> dict[date, DayInfo]: out: dict[date, DayInfo] = {} raw = payload.get("days") diff --git a/suncore/forecast/views.py b/suncore/forecast/views.py index 23aa91b..cbdfa69 100644 --- a/suncore/forecast/views.py +++ b/suncore/forecast/views.py @@ -18,6 +18,7 @@ from suncore.core.decisions import Reason from suncore.core.forecast import AccuracyRecord, HorizonBucket from suncore.forecast.load import DayType, DayTypeSource, LoadQuality, TemperatureReason +from suncore.forecast.load_ml import LoadMlMode, ModelInUse from suncore.forecast.pv.adapter import FetchStatus from suncore.forecast.risk import DistributionSource, RiskClass, RiskWindow @@ -135,16 +136,39 @@ class AccuracySourceView(_View): records: list[AccuracyRecord] +class LoadModelView(_View): + """Which load model the planner gets: the profiles or the optional KNN model (WP12). + + ``mode`` is ``forecast.learning.load_ml_mode``; ``reason`` explains ``in_use`` + (``forecast.load_model.*``). The errors are the rolling 14-day MAE of ``load_profile`` and + ``load_ml`` pooled over the horizon buckets (``None``: not scored yet); ``ml_days`` counts the + usable history days of the KNN model, which ``auto`` needs ``min_history_days`` of.""" + + mode: LoadMlMode + in_use: ModelInUse + reason: Reason + ml_ready: bool + """The KNN model has enough history to forecast (7 days) and produced a forecast.""" + ml_days: int + min_history_days: int + profile_mae_w: float | None + ml_mae_w: float | None + profile_n: int + ml_n: int + + class ForecastAccuracyView(_View): """Accuracy per source and horizon bucket. ``bias_w = actual - forecast`` (positive = the source under-forecasts); ``n`` counts 15-minute daylight intervals (forecast or measurement above 0 W; clipped ones excluded). - A record with ``n`` below ``min_samples`` is too thin to rely on. + A record with ``n`` below ``min_samples`` is too thin to rely on. ``load_model`` compares the + KNN load model with the profiles ("ML vs profile"; ``None`` without a load forecast). """ min_samples: int sources: list[AccuracySourceView] + load_model: LoadModelView | None = None class LoadDayView(_View): @@ -208,6 +232,8 @@ class LoadForecastView(_View): unmeasured_in_profile: list[DayType] """Day types whose profile still contains quarter hours with an unmeasured device (fewer than 7 fully measured days of that type; elsewhere those quarter hours are left out).""" + model: LoadModelView | None = None + """Which model the series above comes from (profiles or the KNN model) and why.""" class EnergyView(_View): @@ -286,6 +312,7 @@ class RiskView(_View): "ForecastAccuracyView", "LoadDayView", "LoadForecastView", + "LoadModelView", "LoadTemperatureView", "PvForecastView", "PvSeriesView", diff --git a/suncore/planner/inputs.py b/suncore/planner/inputs.py index a7b5e23..9224cf0 100644 --- a/suncore/planner/inputs.py +++ b/suncore/planner/inputs.py @@ -705,8 +705,13 @@ def build_inputs( weights: PlannerWeights | None = None, horizon: timedelta = PLANNER_HORIZON, load_fallback_w: float | None = None, + load_reasons: Sequence[Reason] = (), ) -> PlannerInputs: - """The planner's snapshot (module doc); pure.""" + """The planner's snapshot (module doc); pure. + + ``load_reasons``: plan-level reasons about the load forecast from its producer (which load + model it is, WP12); passed through unchanged, after the planner's own input reasons. + """ goals = goals or GoalsConfig() grid = planning_grid(now, horizon, PLANNING_STEP) starts = tuple(grid.starts()) @@ -723,6 +728,8 @@ def build_inputs( flat = max(flat, MIN_BASE_LOAD_W) l10, l50, l90 = [flat * 0.5] * grid.count, [flat] * grid.count, [flat * 2] * grid.count found.append(Reason(key=rk.INPUT_NO_LOAD_FORECAST, args={"load_w": round(flat)})) + elif load_reasons: + found.extend(load_reasons) price_data = _prices(prices, grid, tz) found = price_data.reasons + found planner_battery = _battery(battery, soc_pct, site_battery) diff --git a/tests/api/test_forecast_api.py b/tests/api/test_forecast_api.py index 3c3e5f7..69bb33b 100644 --- a/tests/api/test_forecast_api.py +++ b/tests/api/test_forecast_api.py @@ -4,12 +4,14 @@ from __future__ import annotations from suncore.api import create_app +from suncore.core.decisions import Reason from suncore.core.forecast import AccuracyRecord, HorizonBucket from suncore.forecast.accuracy import StoredAccuracy, accuracy_view from suncore.forecast.views import ( BlendedView, BlendWeightsView, CalibrationView, + LoadModelView, PvForecastView, PvSeriesView, PvSourceView, @@ -155,6 +157,62 @@ async def test_accuracy_shape(services: FakeServices) -> None: assert other["computed_at"] is None +async def test_accuracy_compares_the_ml_model_with_the_profiles(services: FakeServices) -> None: + records = { + source: StoredAccuracy( + source, + NOW, + 14, + ( + AccuracyRecord( + source_id=source, horizon=HorizonBucket.H0_6, mae_w=mae, bias_w=5.0, n=1344 + ), + ), + ) + for source, mae in (("load_profile", 120.0), ("load_ml", 90.0)) + } + view = accuracy_view(["load_profile", "load_ml"], records) + services.accuracy = view.model_copy( + update={ + "load_model": LoadModelView( + mode="auto", + in_use="ml", + reason=Reason( + key="forecast.load_model.ml_in_use", + args={"ml_mae_w": 90, "profile_mae_w": 120}, + ), + ml_ready=True, + ml_days=45, + min_history_days=30, + profile_mae_w=120.0, + ml_mae_w=90.0, + profile_n=1344, + ml_n=1344, + ) + } + ) + async with client_for(create_app(services)) as client: + r = await client.get("/api/v1/forecast/accuracy") + body = r.json() + assert r.status_code == 200 + assert [s["source_id"] for s in body["sources"]] == ["load_profile", "load_ml"] + model = body["load_model"] + assert model["mode"] == "auto" + assert model["in_use"] == "ml" + assert model["reason"] == { + "key": "forecast.load_model.ml_in_use", + "args": {"ml_mae_w": 90, "profile_mae_w": 120}, + } + assert model["ml_days"] == 45 + assert model["ml_mae_w"] == 90.0 + + +async def test_accuracy_without_a_load_forecast_has_no_model(services: FakeServices) -> None: + async with client_for(create_app(services)) as client: + body = (await client.get("/api/v1/forecast/accuracy")).json() + assert body["load_model"] is None + + async def test_the_openapi_document_describes_both_routes(services: FakeServices) -> None: schema = create_app(services).openapi() for path in ("/api/v1/forecast/pv", "/api/v1/forecast/accuracy"): @@ -165,6 +223,7 @@ async def test_the_openapi_document_describes_both_routes(services: FakeServices accuracy = schema["paths"]["/api/v1/forecast/accuracy"]["get"]["description"] assert "actual -" in accuracy assert "blended" in accuracy + assert "load_ml" in accuracy async def test_load_forecast_endpoint() -> None: diff --git a/tests/app/test_load_runtime.py b/tests/app/test_load_runtime.py index 2d05161..3ac6f48 100644 --- a/tests/app/test_load_runtime.py +++ b/tests/app/test_load_runtime.py @@ -106,3 +106,37 @@ async def test_the_demo_risk_view_has_a_buffer_and_a_target(runtime: Runtime) -> assert view.history_days == 0 await runtime.update_forecast_accuracy() # the nightly job learns (nothing yet) safely assert runtime.risk_model.samples == () + + +async def test_the_demo_compares_the_ml_model_with_the_profiles(runtime: Runtime) -> None: + """Default mode ``shadow``: the KNN model is configured, scored and shown, never used.""" + service = runtime.load_service + assert service is not None + assert service.inputs.ml_mode == "shadow" + assert service.inputs.min_history_days == 30 + await until(lambda: service.model is not None) + accuracy = await runtime.forecast_accuracy() + assert [s.source_id for s in accuracy.sources][-2:] == ["load_profile", "load_ml"] + assert accuracy.load_model is not None + assert accuracy.load_model.mode == "shadow" + assert accuracy.load_model.in_use == "profile" + assert accuracy.load_model.reason.key == "forecast.load_model.ml_shadow" + assert service.planning_reasons() == () + + +async def test_auto_mode_explains_the_load_model_in_the_plan(runtime: Runtime) -> None: + current = await runtime.config.get("forecast") + assert isinstance(current, ForecastConfig) + learning = current.learning.model_copy(update={"load_ml_mode": "auto"}) + await runtime.put_section("forecast", current.model_copy(update={"learning": learning})) + service = runtime.load_service + assert service is not None + assert service.inputs.ml_mode == "auto" + await until(lambda: service.model is not None) + await until(lambda: runtime.cache.get(WORKDAY_ID) is not None) + await service.refresh() + assert service.forecast() is not None + snapshot = await runtime._collect_planning(runtime.clock(), None) + assert snapshot.inputs is not None + keys = [r.key for r in snapshot.inputs.reasons] + assert "forecast.load_model.ml_collecting" in keys # the demo has no 30 days of history diff --git a/tests/app/test_pv_runtime.py b/tests/app/test_pv_runtime.py index 2d0b153..df65779 100644 --- a/tests/app/test_pv_runtime.py +++ b/tests/app/test_pv_runtime.py @@ -151,7 +151,8 @@ async def test_changing_the_forecast_section_rebuilds_the_service(runtime: Runti assert view.sources == [] assert view.blended is None accuracy = await runtime.forecast_accuracy() - assert [s.source_id for s in accuracy.sources] == ["load_profile"] # the load forecast + # the load forecast: profiles and the ML load model in shadow (default, WP12) + assert [s.source_id for s in accuracy.sources] == ["load_profile", "load_ml"] async def test_unusable_templates_are_skipped_not_fatal(runtime: Runtime) -> None: @@ -194,6 +195,7 @@ async def test_the_nightly_job_stores_accuracy_that_the_api_reads(runtime: Runti "forecast_solar.pv", "blended", "load_profile", # the load forecast is scored by the same nightly job (WP6) + "load_ml", # and the ML load model in shadow (WP12) } record = next(r for r in by_id["solcast.pv"].records if r.horizon is HorizonBucket.H6_24) assert record.bias_w == 300.0 diff --git a/tests/forecast/test_load_ml.py b/tests/forecast/test_load_ml.py new file mode 100644 index 0000000..4152480 --- /dev/null +++ b/tests/forecast/test_load_ml.py @@ -0,0 +1,396 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The optional KNN load model (WP12): features, quantiles, determinism and the model choice.""" + +from __future__ import annotations + +import random +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from zoneinfo import ZoneInfo + +import numpy as np +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from suncore.core.forecast import ACCURACY_MIN_SAMPLES, AccuracyRecord, HorizonBucket +from suncore.core.series import SeriesUnit, TimeSeries, planning_grid +from suncore.forecast.load import DayType +from suncore.forecast.load_ml import ( + KNN_MIN_DAYS, + ML_MARGIN, + LoadMlMode, + choose_load_model, + fit_knn_model, + forecast_knn, + pooled_error, + predict_slot, + weighted_quantiles, +) +from suncore.forecast.stats import weighted_quantile + +BERLIN = ZoneInfo("Europe/Berlin") +Q = timedelta(minutes=15) +START = datetime(2026, 7, 6, tzinfo=BERLIN).astimezone(UTC) # a Monday + + +def series(values: list[float | None], start: datetime = START) -> TimeSeries: + return TimeSeries(start=start, step=Q, values=tuple(values), unit=SeriesUnit.W) + + +def history(days: int, shape: Callable[[datetime], float], start: datetime = START) -> TimeSeries: + return series([shape(start + Q * i) for i in range(96 * days)], start) + + +def weekly(moment: datetime) -> float: + """300 W, +400 W in the evening, weekends +200 W around noon.""" + t = moment.astimezone(BERLIN) + level = 300.0 + (400.0 if 18 <= t.hour < 21 else 0.0) + if t.weekday() >= 5 and 10 <= t.hour < 14: + level += 200.0 + return level + + +def at(days: int, hour: int = 10) -> datetime: + """``hour`` local on day ``days`` after START.""" + d = START.astimezone(BERLIN).date() + timedelta(days=days) + return datetime(d.year, d.month, d.day, hour, 5, tzinfo=BERLIN).astimezone(UTC) + + +def p50_at(forecast_values: tuple[float | None, ...], start: datetime, moment: datetime) -> float: + value = forecast_values[(moment - start) // Q] + assert value is not None + return value + + +# --- fit and predict ----------------------------------------------------------------------- + + +def test_learns_day_type_and_time_of_day() -> None: + base = history(35, weekly) + now = at(35) # Monday + model = fit_knn_model(base, now=now, tz=BERLIN, day_types={}) + assert model.ready + assert model.days == 35 + assert model.samples == 35 * 96 + assert model.recent_level_w is not None + forecast = forecast_knn(model, planning_grid(now, timedelta(hours=48 * 3)), BERLIN, {}) + assert forecast is not None + values = forecast.p50.values + assert p50_at(values, forecast.start, at(35, 19)) == pytest.approx(700, abs=1) + assert p50_at(values, forecast.start, at(35, 11)) == pytest.approx(300, abs=1) + saturday = at(40, 11) # Saturday noon + assert p50_at(values, forecast.start, saturday) == pytest.approx(500, abs=1) + + +def test_a_holiday_is_predicted_like_a_weekend_day() -> None: + base = history(35, weekly) + now = at(35) + tuesday = (now + timedelta(days=1)).astimezone(BERLIN).date() + model = fit_knn_model(base, now=now, tz=BERLIN, day_types={}) + grid = planning_grid(now, timedelta(hours=48)) + holiday = forecast_knn(model, grid, BERLIN, {tuesday: DayType.HOLIDAY}) + workday = forecast_knn(model, grid, BERLIN, {}) + assert holiday is not None + assert workday is not None + noon = at(36, 11) + assert p50_at(holiday.p50.values, grid.start, noon) == pytest.approx(500, abs=1) + assert p50_at(workday.p50.values, grid.start, noon) == pytest.approx(300, abs=1) + + +def test_temperature_selects_analogue_days() -> None: + """Cold days draw 600 W more; the forecast of a cold day follows the cold analogues.""" + rng = random.Random(3) + temps = { + START.astimezone(BERLIN).date() + timedelta(days=i): rng.choice([0.0, 20.0]) + for i in range(42) + } + + def shape(moment: datetime) -> float: + return 300.0 + (600.0 if temps[moment.astimezone(BERLIN).date()] < 10 else 0.0) + + base = history(42, shape) + temperature = series([temps[(START + Q * i).astimezone(BERLIN).date()] for i in range(96 * 42)]) + temperature = temperature.model_copy(update={"unit": SeriesUnit.CELSIUS}) + now = at(42) + model = fit_knn_model(base, now=now, tz=BERLIN, day_types={}, temperature=temperature) + tomorrow = (now + timedelta(days=1)).astimezone(BERLIN).date() + grid = planning_grid(now, timedelta(hours=40)) + cold = forecast_knn(model, grid, BERLIN, {}, {tomorrow: 0.0}) + warm = forecast_knn(model, grid, BERLIN, {}, {tomorrow: 20.0}) + assert cold is not None + assert warm is not None + noon = at(43, 12) + assert p50_at(cold.p50.values, grid.start, noon) == pytest.approx(900, abs=1) + assert p50_at(warm.p50.values, grid.start, noon) == pytest.approx(300, abs=1) + + +def test_recent_level_selects_analogue_days() -> None: + """The house drew 800 W for three weeks, then 300 W for three weeks, then 800 W again: + with the recent level at 800 W the 800 W weeks are the analogues.""" + + def shape(moment: datetime) -> float: + week = (moment - START) // timedelta(days=7) + return 800.0 if week < 3 or week >= 6 else 300.0 + + base = history(56, shape) + now = at(56) + model = fit_knn_model(base, now=now, tz=BERLIN, day_types={}) + assert model.recent_level_w == pytest.approx(800) + forecast = forecast_knn(model, planning_grid(now, timedelta(hours=24)), BERLIN, {}) + assert forecast is not None + assert all(v == pytest.approx(800, abs=1) for v in forecast.p50.values) + + +def test_too_little_history_gives_no_forecast() -> None: + base = history(KNN_MIN_DAYS - 1, weekly) + now = at(KNN_MIN_DAYS - 1) + model = fit_knn_model(base, now=now, tz=BERLIN, day_types={}) + assert not model.ready + assert forecast_knn(model, planning_grid(now, timedelta(hours=48)), BERLIN, {}) is None + assert ( + predict_slot(model, slot=40, day_type=DayType.WORKDAY, temperature_c=None, level_w=None) + is None + ) + + +def test_empty_history() -> None: + empty = series([]) + model = fit_knn_model(empty, now=START, tz=BERLIN, day_types={}) + assert model.days == 0 + assert model.recent_level_w is None + assert forecast_knn(model, planning_grid(START, timedelta(hours=4)), BERLIN, {}) is None + + +def test_days_with_little_coverage_and_unmeasured_quarter_hours_are_left_out() -> None: + values: list[float | None] = [weekly(START + Q * i) for i in range(96 * 21)] + for i in range(96 * 3, 96 * 3 + 40): # day 3: only 56 of 96 quarter hours known + values[i] = None + unmeasured = [False] * len(values) + for i in range(96 * 10, 96 * 10 + 8): # day 10: a device without a measurement + unmeasured[i] = True + model = fit_knn_model( + series(values), now=at(21), tz=BERLIN, day_types={}, unmeasured=unmeasured + ) + assert model.days == 20 + assert model.samples == 20 * 96 - 8 + + +def test_history_window_is_respected() -> None: + base = history(60, weekly) + model = fit_knn_model(base, now=at(60), tz=BERLIN, day_types={}, history_days=30) + assert model.days == 30 + + +def test_spring_forward_day_has_a_forecast_for_every_slot() -> None: + start = datetime(2026, 2, 23, tzinfo=BERLIN).astimezone(UTC) + base = history(35, weekly, start) + now = datetime(2026, 3, 28, 10, tzinfo=BERLIN).astimezone(UTC) + model = fit_knn_model(base, now=now, tz=BERLIN, day_types={}) + grid = planning_grid(now, timedelta(hours=48)) # contains the 23-hour day + forecast = forecast_knn(model, grid, BERLIN, {}) + assert forecast is not None + assert len(forecast) == grid.count + assert all(v is not None for v in forecast.p50.values) + + +# --- properties -------------------------------------------------------------------------- + + +def noisy(seed: int, scale: float) -> Callable[[datetime], float]: + rng = random.Random(seed) + cache: dict[datetime, float] = {} + + def shape(moment: datetime) -> float: + if moment not in cache: + cache[moment] = max(0.0, weekly(moment) * scale + rng.gauss(0, 150)) + return cache[moment] + + return shape + + +@settings(max_examples=25, deadline=None) +@given( + days=st.integers(KNN_MIN_DAYS, 40), + seed=st.integers(0, 10_000), + scale=st.floats(0.0, 5.0), + hours=st.integers(1, 48), + temp=st.one_of(st.none(), st.floats(-20, 40)), +) +def test_quantiles_are_monotone_non_negative_and_on_the_grid( + days: int, seed: int, scale: float, hours: int, temp: float | None +) -> None: + base = history(days, noisy(seed, scale)) + now = at(days, 7) + model = fit_knn_model(base, now=now, tz=BERLIN, day_types={}) + grid = planning_grid(now, timedelta(hours=hours)) + temps = {} if temp is None else {now.astimezone(BERLIN).date(): temp} + forecast = forecast_knn(model, grid, BERLIN, {}, temps) + assert forecast is not None + assert forecast.start == grid.start + assert len(forecast) == grid.count + assert forecast.unit is SeriesUnit.W + for lo, mid, hi in zip( + forecast.p10.values, forecast.p50.values, forecast.p90.values, strict=True + ): + assert lo is not None + assert mid is not None + assert hi is not None + assert 0 <= lo <= mid <= hi + + +@settings(max_examples=10, deadline=None) +@given(days=st.integers(KNN_MIN_DAYS, 30), seed=st.integers(0, 10_000)) +def test_fit_and_forecast_are_deterministic(days: int, seed: int) -> None: + now = at(days, 13) + grid = planning_grid(now, timedelta(hours=48)) + results = [] + for _ in range(2): + base = history(days, noisy(seed, 1.0)) + model = fit_knn_model(base, now=now, tz=BERLIN, day_types={}) + results.append(forecast_knn(model, grid, BERLIN, {})) + assert results[0] == results[1] + + +def test_ties_are_broken_deterministically() -> None: + """Every day identical: all distances tie apart from the age; the forecast is the value.""" + base = history(28, lambda _m: 400.0) + now = at(28) + model = fit_knn_model(base, now=now, tz=BERLIN, day_types={}) + first = forecast_knn(model, planning_grid(now, timedelta(hours=48)), BERLIN, {}) + second = forecast_knn(model, planning_grid(now, timedelta(hours=48)), BERLIN, {}) + assert first == second + assert first is not None + assert set(first.p50.values) == {400.0} + + +@given( + st.lists(st.tuples(st.floats(-1e5, 1e5), st.floats(0.01, 10)), min_size=1, max_size=40), + st.floats(0, 1), +) +def test_weighted_quantiles_match_the_reference(pairs: list[tuple[float, float]], q: float) -> None: + values = [v for v, _ in pairs] + weights = [w for _, w in pairs] + (ours,) = weighted_quantiles(np.array(values), np.array(weights), (q,)) + assert ours == pytest.approx(weighted_quantile(values, weights, q), rel=1e-9, abs=1e-6) + + +# --- model choice ------------------------------------------------------------------------ + + +def records(source: str, mae: float, n: int = 1344) -> list[AccuracyRecord]: + return [ + AccuracyRecord(source_id=source, horizon=h, mae_w=mae, bias_w=0.0, n=n) + for h in HorizonBucket + ] + + +def choose( + mode: LoadMlMode = "auto", + *, + ml_days: int = 45, + ml_available: bool = True, + min_history_days: int = 30, + profile: list[AccuracyRecord] | None = None, + ml: list[AccuracyRecord] | None = None, +) -> tuple[str, str]: + choice = choose_load_model( + mode, + ml_days=ml_days, + ml_available=ml_available, + min_history_days=min_history_days, + profile_records=records("load_profile", 100.0) if profile is None else profile, + ml_records=records("load_ml", 80.0) if ml is None else ml, + ) + return choice.in_use, choice.reason.key + + +def test_off_and_shadow_never_use_the_model() -> None: + assert choose("off") == ("profile", "forecast.load_model.ml_off") + assert choose("shadow") == ("profile", "forecast.load_model.ml_shadow") + + +def test_auto_uses_the_model_only_when_it_is_measurably_better() -> None: + assert choose() == ("ml", "forecast.load_model.ml_in_use") + better_by_less_than_the_margin = 100.0 * (1 - ML_MARGIN / 2) + assert choose(ml=records("load_ml", better_by_less_than_the_margin)) == ( + "profile", + "forecast.load_model.ml_not_better", + ) + assert choose(ml=records("load_ml", 120.0)) == ("profile", "forecast.load_model.ml_not_better") + + +def test_auto_needs_enough_history() -> None: + assert choose(ml_days=29) == ("profile", "forecast.load_model.ml_collecting") + assert choose(ml_days=30) == ("ml", "forecast.load_model.ml_in_use") + assert choose(ml_days=10, min_history_days=10) == ("ml", "forecast.load_model.ml_in_use") + + +def test_auto_needs_a_forecast_and_a_comparable_score() -> None: + assert choose(ml_available=False) == ("profile", "forecast.load_model.ml_unavailable") + assert choose(ml=[]) == ("profile", "forecast.load_model.ml_not_scored") + assert choose(profile=[]) == ("profile", "forecast.load_model.ml_not_scored") + # scored on 3 of the profiles' 14 days (stored only for the last few days) + assert choose(ml=records("load_ml", 50.0, n=288)) == ( + "profile", + "forecast.load_model.ml_not_scored", + ) + thin = records("load_ml", 50.0, n=ACCURACY_MIN_SAMPLES // 6) + assert choose(profile=records("load_profile", 100.0, n=10), ml=thin) == ( + "profile", + "forecast.load_model.ml_not_scored", + ) + + +@given( + mode=st.sampled_from(["off", "shadow", "auto"]), + ml_days=st.integers(0, 400), + min_days=st.integers(1, 365), + available=st.booleans(), + profile_mae=st.floats(0, 5000), + ml_mae=st.floats(0, 5000), + profile_n=st.integers(0, 5000), + ml_n=st.integers(0, 5000), +) +def test_the_model_is_used_only_under_every_rule( + *, + mode: LoadMlMode, + ml_days: int, + min_days: int, + available: bool, + profile_mae: float, + ml_mae: float, + profile_n: int, + ml_n: int, +) -> None: + choice = choose_load_model( + mode, + ml_days=ml_days, + ml_available=available, + min_history_days=min_days, + profile_records=records("load_profile", profile_mae, profile_n) if profile_n else [], + ml_records=records("load_ml", ml_mae, ml_n) if ml_n else [], + ) + assert choice.reason.key.startswith("forecast.load_model.") + if choice.in_use == "ml": + assert mode == "auto" + assert available + assert ml_days >= min_days + assert ml_n >= profile_n * 0.9 + assert 3 * ml_n >= ACCURACY_MIN_SAMPLES + assert ml_mae < profile_mae + assert choice.ml_days == ml_days + assert choice.min_history_days == min_days + + +def test_pooled_error_weights_the_buckets_by_their_quarter_hours() -> None: + pooled = pooled_error( + [ + AccuracyRecord(source_id="x", horizon=HorizonBucket.H0_6, mae_w=10, bias_w=0, n=1), + AccuracyRecord(source_id="x", horizon=HorizonBucket.H6_24, mae_w=40, bias_w=0, n=3), + ] + ) + assert pooled is not None + assert pooled.mae_w == pytest.approx(32.5) + assert pooled.n == 4 + assert pooled_error([]) is None diff --git a/tests/forecast/test_load_ml_bench.py b/tests/forecast/test_load_ml_bench.py new file mode 100644 index 0000000..812b83d --- /dev/null +++ b/tests/forecast/test_load_ml_bench.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Run-time budget of the KNN load model (plan WP12; ``scripts/bench_load_ml.py``). + +Training (building the samples) plus the prediction of 48 h must stay below 1 s for 30 to 90 +days of 15-minute history. Measured on CI's x86 runner; aarch64 is ``UNVERIFIED``. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "bench_load_ml.py" +BUDGET_S = 1.0 + + +@pytest.mark.slow +def test_knn_training_and_prediction_stay_within_the_budget() -> None: + spec = importlib.util.spec_from_file_location("bench_load_ml", SCRIPT) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + result = module.bench(3, [30, 90]) + for days in (30, 90): + row = result[f"{days}d"] + assert row["samples"] == days * 96 + assert row["slots"] == 193 + assert row["total_s"] < BUDGET_S diff --git a/tests/forecast/test_load_ml_service.py b/tests/forecast/test_load_ml_service.py new file mode 100644 index 0000000..7dc434b --- /dev/null +++ b/tests/forecast/test_load_ml_service.py @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The KNN load model inside the load forecast service (WP12): shadow, off, auto, bookkeeping.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta +from pathlib import Path +from zoneinfo import ZoneInfo + +import pytest + +from suncore.core.forecast import AccuracyRecord, HorizonBucket +from suncore.core.measurements import SITE_HOUSE_W, device_power_key +from suncore.core.series import SeriesUnit, TimeSeries +from suncore.forecast.accuracy import load_accuracy, save_accuracy +from suncore.forecast.load import LOAD_SOURCE_ID +from suncore.forecast.load_ml import LOAD_ML_SOURCE_ID, LoadMlMode +from suncore.forecast.load_service import LoadForecastService, LoadInputs +from suncore.storage import Database, LearnedParamStore +from suncore.storage.timeseries import TimeseriesStore, open_timeseries + +BERLIN = ZoneInfo("Europe/Berlin") +Q = timedelta(minutes=15) +START = datetime(2026, 8, 3, tzinfo=BERLIN).astimezone(UTC) # a Monday + + +def moment(days: int, hour: int = 10, minute: int = 5) -> datetime: + d = START.astimezone(BERLIN).date() + timedelta(days=days) + return datetime(d.year, d.month, d.day, hour, minute, tzinfo=BERLIN).astimezone(UTC) + + +class Clock: + def __init__(self, now: datetime) -> None: + self.now = now + + def __call__(self) -> datetime: + return self.now + + +@pytest.fixture +async def store(tmp_path: Path) -> AsyncIterator[TimeseriesStore]: + db = await open_timeseries(tmp_path / "timeseries.db") + yield TimeseriesStore(db) + await db.close() + + +@pytest.fixture +async def learned(tmp_path: Path) -> AsyncIterator[LearnedParamStore]: + db = await Database.open(tmp_path / "suncore.db") + yield LearnedParamStore(db, clock=lambda: START) + await db.close() + + +def ts(values: list[float | None], start: datetime = START) -> TimeSeries: + return TimeSeries(start=start, step=Q, values=tuple(values), unit=SeriesUnit.W) + + +async def fill_history(store: TimeseriesStore, days: int) -> None: + """House 300 W plus 400 W from 18:00 to 21:00 local; the AC draws 200 W of it at noon.""" + house: list[float | None] = [] + ac: list[float | None] = [] + for i in range(96 * days): + hour = (START + Q * i).astimezone(BERLIN).hour + draw = 200.0 if 12 <= hour < 14 else 0.0 + house.append(300.0 + (400.0 if 18 <= hour < 21 else 0.0) + draw) + ac.append(draw) + await store.store_quarter_hours(SITE_HOUSE_W, ts(house)) + await store.store_quarter_hours(device_power_key("ac"), ts(ac)) + + +def service( + store: TimeseriesStore | None, + learned: LearnedParamStore | None, + clock: Clock, + mode: LoadMlMode, + min_history_days: int = 30, +) -> LoadForecastService: + return LoadForecastService( + store, + learned, + clock=clock, + tz=lambda: BERLIN, + inputs=LoadInputs(device_ids=("ac",), ml_mode=mode, min_history_days=min_history_days), + ) + + +def records(source: str, mae: float) -> list[AccuracyRecord]: + return [ + AccuracyRecord(source_id=source, horizon=h, mae_w=mae, bias_w=0.0, n=1344) + for h in HorizonBucket + ] + + +async def test_shadow_is_stored_and_shown_but_never_used( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store, 35) + now = moment(35) + svc = service(store, learned, Clock(now), "shadow") + svc.set_scores( + { + LOAD_SOURCE_ID: records(LOAD_SOURCE_ID, 100), + LOAD_ML_SOURCE_ID: records(LOAD_ML_SOURCE_ID, 10), + } + ) + forecast = await svc.refresh() + assert forecast is not None + assert forecast == svc.profile_forecast(now) # even though the ML model scored better + ml = svc.ml_forecast(now) + assert ml is not None + assert ml.start == forecast.start + assert len(ml) == len(forecast) + assert await store.forecast_issues(LOAD_SOURCE_ID, "load") == [now] + assert await store.forecast_issues(LOAD_ML_SOURCE_ID, "load") == [now] + assert svc.source_ids == (LOAD_SOURCE_ID, LOAD_ML_SOURCE_ID) + assert svc.planning_reasons() == () + view = svc.view() + assert view.source_id == LOAD_SOURCE_ID + assert view.model is not None + assert view.model.mode == "shadow" + assert view.model.in_use == "profile" + assert view.model.ml_ready + assert view.model.ml_days == 35 + assert view.model.reason.key == "forecast.load_model.ml_shadow" + assert view.model.reason.args == {"days": 35, "min_days": 30} + assert view.model.ml_mae_w == 10 + assert view.model.profile_mae_w == 100 + + +async def test_off_computes_nothing(store: TimeseriesStore, learned: LearnedParamStore) -> None: + await fill_history(store, 35) + now = moment(35) + svc = service(store, learned, Clock(now), "off") + assert await svc.refresh() is not None + assert svc.ml_model is None + assert svc.ml_forecast(now) is None + assert svc.source_ids == (LOAD_SOURCE_ID,) + assert await store.forecast_issues(LOAD_ML_SOURCE_ID, "load") == [] + view = svc.view() + assert view.model is not None + assert view.model.reason.key == "forecast.load_model.ml_off" + assert not view.model.ml_ready + + +async def test_auto_with_too_little_history_keeps_the_profiles( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store, 20) + now = moment(20) + svc = service(store, learned, Clock(now), "auto") + svc.set_scores( + { + LOAD_SOURCE_ID: records(LOAD_SOURCE_ID, 100), + LOAD_ML_SOURCE_ID: records(LOAD_ML_SOURCE_ID, 10), + } + ) + forecast = await svc.refresh() + assert forecast == svc.profile_forecast(now) + (reason,) = svc.planning_reasons() + assert reason.key == "forecast.load_model.ml_collecting" + assert reason.args == {"days": 20, "min_days": 30} + + +async def test_auto_uses_the_model_while_it_is_better_and_falls_back( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store, 35) + now = moment(35) + svc = service(store, learned, Clock(now), "auto") + await svc.refresh() + assert svc.choice().reason.key == "forecast.load_model.ml_not_scored" # nothing scored yet + assert svc.forecast(now) == svc.profile_forecast(now) + + svc.set_scores( + { + LOAD_SOURCE_ID: records(LOAD_SOURCE_ID, 100), + LOAD_ML_SOURCE_ID: records(LOAD_ML_SOURCE_ID, 60), + } + ) + ml = svc.ml_forecast(now) + assert ml is not None + assert svc.forecast(now) == ml + load, choice = svc.selected(now) + assert load == ml + assert svc.planning_reasons(choice)[0].key == "forecast.load_model.ml_in_use" + view = svc.view() + assert view.source_id == LOAD_ML_SOURCE_ID + assert view.p50_w == list(ml.p50.values[: 48 * 4]) + + svc.set_scores({LOAD_ML_SOURCE_ID: records(LOAD_ML_SOURCE_ID, 120)}) # worse now + assert svc.forecast(now) == svc.profile_forecast(now) + assert svc.choice().reason.key == "forecast.load_model.ml_not_better" + + +async def test_auto_falls_back_when_the_model_has_no_forecast( + store: TimeseriesStore, learned: LearnedParamStore, monkeypatch: pytest.MonkeyPatch +) -> None: + await fill_history(store, 35) + now = moment(35) + svc = service(store, learned, Clock(now), "auto") + await svc.refresh() + svc.set_scores( + { + LOAD_SOURCE_ID: records(LOAD_SOURCE_ID, 100), + LOAD_ML_SOURCE_ID: records(LOAD_ML_SOURCE_ID, 60), + } + ) + + def broken(*_args: object, **_kwargs: object) -> None: + msg = "boom" + raise RuntimeError(msg) + + monkeypatch.setattr("suncore.forecast.load_service.forecast_knn", broken) + load, choice = svc.selected(now) + assert load == svc.profile_forecast(now) + assert choice.in_use == "profile" + assert choice.reason.key == "forecast.load_model.ml_unavailable" + + +async def test_the_profiles_keep_their_own_window( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store, 70) + now = moment(70) + with_ml = service(store, learned, Clock(now), "shadow") + without = service(store, learned, Clock(now), "off") + await with_ml.refit(now) + await without.refit(now) + assert with_ml.model is not None + assert without.model is not None + assert with_ml.model.days == without.model.days == 56 + assert with_ml.profile_forecast(now) == without.profile_forecast(now) + assert with_ml.view().devices_unmeasured == without.view().devices_unmeasured + assert with_ml.ml_model is not None + assert with_ml.ml_model.days == 70 + assert with_ml.model_view().ml_days == 70 + + +async def test_long_min_history_widens_the_window( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store, 100) + now = moment(100) + svc = service(store, learned, Clock(now), "auto", min_history_days=95) + await svc.refit(now) + assert svc.ml_model is not None + assert svc.ml_model.days == 95 + + +async def test_nightly_scores_both_models_and_keeps_the_result( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store, 16) + clock = Clock(moment(14)) + svc = service(store, learned, clock, "shadow") + await svc.refresh() # issues of both models + clock.now = moment(16, 3, 0) + records_ = await svc.nightly() + assert {r.source_id for r in records_} == {LOAD_SOURCE_ID} + stored = await load_accuracy(learned, [LOAD_SOURCE_ID, LOAD_ML_SOURCE_ID]) + assert set(stored) == {LOAD_SOURCE_ID, LOAD_ML_SOURCE_ID} + ml = {r.horizon: r for r in stored[LOAD_ML_SOURCE_ID].records} + profile = {r.horizon: r for r in stored[LOAD_SOURCE_ID].records} + assert ml[HorizonBucket.H0_6].n == profile[HorizonBucket.H0_6].n > 0 + assert ml[HorizonBucket.H0_6].mae_w == pytest.approx(0, abs=1) # same metric, same actuals + view = svc.model_view() + assert view.ml_n == sum(r.n for r in ml.values()) + assert view.profile_n == sum(r.n for r in profile.values()) + + +async def test_scores_of_an_earlier_run_are_read_at_the_first_fit( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store, 35) + now = moment(35) + await save_accuracy(learned, LOAD_SOURCE_ID, records(LOAD_SOURCE_ID, 100), now=now) + await save_accuracy(learned, LOAD_ML_SOURCE_ID, records(LOAD_ML_SOURCE_ID, 50), now=now) + svc = service(store, learned, Clock(now), "auto") + await svc.refresh() + assert svc.choice().in_use == "ml" + assert svc.model_view().ml_mae_w == 50 + + +async def test_without_storage_nothing_breaks() -> None: + svc = service(None, None, Clock(moment(1)), "auto") + assert await svc.refresh() is None + assert svc.forecast() is None + assert svc.choice().reason.key == "forecast.load_model.ml_collecting" + assert await svc.nightly() == [] diff --git a/tests/planner/test_inputs.py b/tests/planner/test_inputs.py index 98f3026..5d74dc2 100644 --- a/tests/planner/test_inputs.py +++ b/tests/planner/test_inputs.py @@ -422,6 +422,15 @@ def test_without_a_load_forecast_a_flat_load_and_no_devices(series) -> None: # assert set(recent.load_p50) == {730.0} +def test_the_load_model_reason_is_passed_through(series) -> None: # type: ignore[no-untyped-def] + why = Reason(key="forecast.load_model.ml_in_use", args={"ml_mae_w": 80, "profile_mae_w": 95}) + inputs = _build(series, load_reasons=(why,)) + assert why in inputs.reasons + # without a load forecast the flat-load reason says it all (no model was used) + flat = _build(series, load=None, load_reasons=(why,)) + assert why not in flat.reasons + + @dataclass(frozen=True) class _Target: soc_pct: float From 498e59198197a1295657cfe66dbe5c4075b465e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 21:24:10 +0000 Subject: [PATCH 2/2] fix(forecast): review fixes for the ML load model, work in progress Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VHXsfH6PkT62o9idByAd6e --- docs/STATUS.md | 12 +- docs/plans/phase-4.md | 19 +- docs/user/de/planning-and-battery.md | 22 +- docs/user/en/planning-and-battery.md | 20 +- frontend/src/lib/api/openapi.json | 30 ++- frontend/src/lib/api/schema.d.ts | 16 +- .../src/lib/components/ForecastPanel.svelte | 15 +- .../src/lib/components/ForecastPanel.test.ts | 14 +- frontend/src/lib/i18n/locales/de.json | 5 +- frontend/src/lib/i18n/locales/en.json | 5 +- frontend/src/lib/views/steps/PvStep.svelte | 3 +- suncore/forecast/accuracy.py | 116 +++++++-- suncore/forecast/load_ml.py | 96 +++++--- suncore/forecast/load_service.py | 217 +++++++++++++---- suncore/forecast/views.py | 15 +- tests/api/test_forecast_api.py | 4 +- tests/forecast/test_load_ml.py | 128 +++++++--- tests/forecast/test_load_ml_service.py | 223 +++++++++++++----- 18 files changed, 690 insertions(+), 270 deletions(-) diff --git a/docs/STATUS.md b/docs/STATUS.md index 82b5ccd..5d196ff 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -19,11 +19,13 @@ test · **Branch:** `phase-4/i-ml-load` (stacked on `phase-4/h-ui-docs`) slots dropped after 7 clean days), features day type (holiday ≈ weekend), slot (±2), daily mean temperature, 7-day load level, weak age term; K = 24, P50 weighted median, P10/P90 weighted quantiles, ≥ 0, monotone, deterministic (stable sort). Forecasts after 7 usable days. - - `forecast.learning.load_ml_mode`: `off` not computed; `shadow` (default) stored hourly as - source `load_ml`, scored nightly like `load_profile`, shown, never used; `auto` used by the - planner (and the risk view) only with ≥ `min_history_days` days and a pooled 14-day MAE ≥ 2 % - below the profiles' on a comparable sample, else the profiles; every choice has a - `forecast.load_model.*` reason, in `auto` also as a plan-level reason. + - `forecast.learning.load_ml_mode`: `off` not computed; `shadow` (default) stored hourly with + the profiles' issue time as source `load_ml`, scored nightly like `load_profile`, shown, never + used; `auto` used by the planner (and the risk view) only with ≥ `min_history_days` days and a + fresh (≤ 36 h) paired 14-day comparison on the same issues and quarter hours (≥ 7 days) with + an MAE ≥ 2 % below the profiles' (left again only when no longer below: hysteresis), else the + profiles; fitted once a day in a worker thread, stale after a day without a fit; every choice + has a `forecast.load_model.*` reason, in `auto` also as a plan-level reason. - API: `ForecastAccuracyView.load_model` and `LoadForecastView.model` (`LoadModelView`, additive); `load_ml` rows in `/forecast/accuracy`. UI: "Machine-learning load model" in the forecast panel (mode, model in use, reason, both MAEs, history days); mode select in wizard step 4. diff --git a/docs/plans/phase-4.md b/docs/plans/phase-4.md index e8b076a..a725ebb 100644 --- a/docs/plans/phase-4.md +++ b/docs/plans/phase-4.md @@ -652,12 +652,19 @@ shadow and assisted run is the acceptance test. 56 days; the service reads the longer window once and slices it for the profiles). The fit window now starts at a local midnight, so a DST change inside it no longer drops its first day (also for the profiles). - - "Better" in `auto` = the pooled (n-weighted over the three horizon buckets) 14-day MAE of - `load_ml` at least **2 %** below `load_profile`'s, with `load_ml` scored on at least 90 % as many - quarter hours (else the two MAEs cover different periods; ≥ 96 in any case) and at least - `min_history_days` usable days; otherwise the profiles, each with a - `forecast.load_model.*` reason. The choice changes with the nightly scores, at most once a - day. + - "Better" in `auto` = a **paired** 14-day score (review M2): the KNN forecast is stored with + the same issue time as the profile forecast, and the nightly job scores both on the common + issues and only on quarter hours where both and the measurement are known (pooled over the + horizon buckets, ≥ 7 × 96 quarter hours); it is kept in `learned_param` `ml_state/ + load_comparison` with its time and counts only while ≤ 36 h old (review M1; a failed nightly + comparison drops it, a missing `load_ml` score is removed from the display). The KNN model is + entered at an MAE ≥ **2 %** below the profiles' and left only when it is no longer below + (hysteresis, review m4), with at least `min_history_days` usable days; otherwise the + profiles, each with a `forecast.load_model.*` reason. The per-source rows stay for display. + - The KNN model is fitted once per local day (or after `request_refit`) in a worker thread + (`asyncio.to_thread`); its forecast level is the 7 days before local midnight like every + sample's; a model fitted before yesterday (its fit keeps failing) is stale and not used + (review m1, m3, m4). The KNN forecast is computed only when it is stored or in use (m2). - In `auto` the plan carries the choice as a plan-level reason (`build_inputs(load_reasons=…)`, planner still imports only `core`); in `off`/`shadow` no plan reason (nothing to explain). The risk view and the planner get the same selected forecast; the risk model's *learnt* diff --git a/docs/user/de/planning-and-battery.md b/docs/user/de/planning-and-battery.md index f157483..883b02a 100644 --- a/docs/user/de/planning-and-battery.md +++ b/docs/user/de/planning-and-battery.md @@ -137,15 +137,19 @@ Den Modus wählst du in Schritt 4 unter **Maschinelles Lastmodell**: bewertet und in **Prognosen und Genauigkeit** unter *Maschinelles Lastmodell* angezeigt, beide Fehler nebeneinander. Der Plan nutzt es nie. - **Automatisch**: Der Plan nutzt es nur, solange zweierlei gilt: Es hat mindestens den Verlauf, - den die Lerneinstellung verlangt (standardmäßig 30 Tage), **und** sein mittlerer absoluter - Fehler lag in den letzten 14 Tagen mindestens 2 % unter dem der Profile (bewertet an etwa - ebenso vielen Viertelstunden). Sonst bleibt der Plan bei den Profilen. Jeder Plan trägt dann - einen kurzen Hinweis, welches Modell er genutzt hat und warum („noch zu wenig Verlauf“, „nicht - deutlich besser“, „genauer als die Profile“ mit beiden Fehlern). - -Die Bewertung entsteht im nächtlichen Lauf, ein Wechsel zwischen den Modellen passiert also -höchstens einmal am Tag. Das Risikomodell lernt seine Fehlerverteilung weiter aus der -Profilprognose. + den die Lerneinstellung verlangt (standardmäßig 30 Tage), **und** im Vergleich der letzten Nacht + lag sein mittlerer absoluter Fehler über 14 Tage mindestens 2 % unter dem der Profile. Beide + Modelle werden an genau denselben Prognosen und Viertelstunden verglichen (mindestens eine + Woche davon). Ist es einmal in Gebrauch, bleibt es, bis es gar nicht mehr besser ist, damit es + nicht hin und her springt. Ohne frischen Vergleich (der nächtliche Lauf ist fehlgeschlagen) + oder mit einem Modell, das seit gestern nicht neu angelernt werden konnte, bleibt der Plan bei + den Profilen. Jeder Plan trägt dann einen kurzen Hinweis, welches Modell er genutzt hat und + warum („noch zu wenig Verlauf“, „nicht deutlich besser“, „genauer als die Profile“ mit beiden + Fehlern). + +Das Modell wird einmal am Tag neu angelernt, der Vergleich entsteht im nächtlichen Lauf, ein +Wechsel zwischen den Modellen passiert also höchstens einmal am Tag. Das Risikomodell lernt seine +Fehlerverteilung weiter aus der Profilprognose. ## 3. Tarife diff --git a/docs/user/en/planning-and-battery.md b/docs/user/en/planning-and-battery.md index a7ed6ed..66ab2fb 100644 --- a/docs/user/en/planning-and-battery.md +++ b/docs/user/en/planning-and-battery.md @@ -131,14 +131,18 @@ Choose its mode in step 4 under **Machine-learning load model**: profiles, and shown in **Forecasts and accuracy** under *Machine-learning load model*, with both errors side by side. The plan never uses it. - **Automatic**: the plan uses it only while two things hold: it has at least the history the - learning setting asks for (30 days by default), **and** over the last 14 days its mean absolute - error was at least 2 % lower than the profiles' (scored on about as many quarter hours). Otherwise - the plan stays on the profiles. Each plan then carries a short note which model it used and why - ("not enough history yet", "not clearly better", "more accurate than the profiles" with both - errors). - -The scores are computed in the nightly job, so a switch between the models happens at most once a -day. The risk model keeps learning its error distribution from the profile forecast. + learning setting asks for (30 days by default), **and** in last night's comparison its mean + absolute error over the last 14 days was at least 2 % lower than the profiles'. Both models are + compared on exactly the same forecasts and quarter hours (at least a week of them). Once in use, + the model stays until it is no longer better at all, so it does not flip back and forth. Without + a fresh comparison (the nightly job failed) or with a model that could not be refitted since + yesterday, the plan stays on the profiles. Each plan then carries a short note which model it + used and why ("not enough history yet", "not clearly better", "more accurate than the profiles" + with both errors). + +The model is refitted once a day, the comparison is computed in the nightly job, so a switch +between the models happens at most once a day. The risk model keeps learning its error +distribution from the profile forecast. ## 3. Tariffs diff --git a/frontend/src/lib/api/openapi.json b/frontend/src/lib/api/openapi.json index 066f6ed..a2fe72f 100644 --- a/frontend/src/lib/api/openapi.json +++ b/frontend/src/lib/api/openapi.json @@ -4692,8 +4692,24 @@ }, "LoadModelView": { "additionalProperties": false, - "description": "Which load model the planner gets: the profiles or the optional KNN model (WP12).\n\n``mode`` is ``forecast.learning.load_ml_mode``; ``reason`` explains ``in_use``\n(``forecast.load_model.*``). The errors are the rolling 14-day MAE of ``load_profile`` and\n``load_ml`` pooled over the horizon buckets (``None``: not scored yet); ``ml_days`` counts the\nusable history days of the KNN model, which ``auto`` needs ``min_history_days`` of.", + "description": "Which load model the planner gets: the profiles or the optional KNN model (WP12).\n\n``mode`` is ``forecast.learning.load_ml_mode``; ``reason`` explains ``in_use``\n(``forecast.load_model.*``). The two errors are the paired comparison ``auto`` decides on:\nthe 14-day MAE of ``load_profile`` and ``load_ml`` on the same issues and quarter hours\n(``compared_n``), computed by the nightly job at ``compared_at`` (``None``: none yet, or\nolder than 36 h). ``ml_days`` counts the usable history days of the KNN model, which\n``auto`` needs ``min_history_days`` of.", "properties": { + "compared_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Compared At" + }, + "compared_n": { + "title": "Compared N", + "type": "integer" + }, "in_use": { "enum": [ "profile", @@ -4721,10 +4737,6 @@ ], "title": "Ml Mae W" }, - "ml_n": { - "title": "Ml N", - "type": "integer" - }, "ml_ready": { "title": "Ml Ready", "type": "boolean" @@ -4749,10 +4761,6 @@ ], "title": "Profile Mae W" }, - "profile_n": { - "title": "Profile N", - "type": "integer" - }, "reason": { "$ref": "#/components/schemas/Reason" } @@ -4766,8 +4774,8 @@ "min_history_days", "profile_mae_w", "ml_mae_w", - "profile_n", - "ml_n" + "compared_n", + "compared_at" ], "title": "LoadModelView", "type": "object" diff --git a/frontend/src/lib/api/schema.d.ts b/frontend/src/lib/api/schema.d.ts index d104f14..15cf67b 100644 --- a/frontend/src/lib/api/schema.d.ts +++ b/frontend/src/lib/api/schema.d.ts @@ -2572,11 +2572,17 @@ export interface components { * @description Which load model the planner gets: the profiles or the optional KNN model (WP12). * * ``mode`` is ``forecast.learning.load_ml_mode``; ``reason`` explains ``in_use`` - * (``forecast.load_model.*``). The errors are the rolling 14-day MAE of ``load_profile`` and - * ``load_ml`` pooled over the horizon buckets (``None``: not scored yet); ``ml_days`` counts the - * usable history days of the KNN model, which ``auto`` needs ``min_history_days`` of. + * (``forecast.load_model.*``). The two errors are the paired comparison ``auto`` decides on: + * the 14-day MAE of ``load_profile`` and ``load_ml`` on the same issues and quarter hours + * (``compared_n``), computed by the nightly job at ``compared_at`` (``None``: none yet, or + * older than 36 h). ``ml_days`` counts the usable history days of the KNN model, which + * ``auto`` needs ``min_history_days`` of. */ LoadModelView: { + /** Compared At */ + compared_at: string | null; + /** Compared N */ + compared_n: number; /** * In Use * @enum {string} @@ -2588,8 +2594,6 @@ export interface components { ml_days: number; /** Ml Mae W */ ml_mae_w: number | null; - /** Ml N */ - ml_n: number; /** Ml Ready */ ml_ready: boolean; /** @@ -2599,8 +2603,6 @@ export interface components { mode: "off" | "shadow" | "auto"; /** Profile Mae W */ profile_mae_w: number | null; - /** Profile N */ - profile_n: number; reason: components["schemas"]["Reason"]; }; /** diff --git a/frontend/src/lib/components/ForecastPanel.svelte b/frontend/src/lib/components/ForecastPanel.svelte index 68b3f06..678b4fc 100644 --- a/frontend/src/lib/components/ForecastPanel.svelte +++ b/frontend/src/lib/components/ForecastPanel.svelte @@ -235,7 +235,7 @@ {#if loadModel.mode !== 'off'}
- + @@ -247,26 +247,23 @@
{t('fc.ml.compare_caption')}{t('fc.ml.compare_caption', { n: loadModel.compared_n })}
{t('fc.ml.model')}{t('fc.ml.profile')} {mae(loadModel.profile_mae_w)} - {#if loadModel.profile_n > 0}{t('fc.cell_n', { n: loadModel.profile_n })}{/if}
{t('fc.ml.ml')} {mae(loadModel.ml_mae_w)} - {#if loadModel.ml_n > 0}{t('fc.cell_n', { n: loadModel.ml_n })}{/if}

- {t('fc.ml.days', { days: loadModel.ml_days, min: loadModel.min_history_days })} - {#if !loadModel.ml_ready}· {t('fc.ml.collecting_model')}{/if} + {t('fc.ml.days', { + days: loadModel.ml_days, + min: loadModel.min_history_days, + ready: loadModel.ml_ready ? 'yes' : 'no', + })}

{/if} {/if} diff --git a/frontend/src/lib/components/ForecastPanel.test.ts b/frontend/src/lib/components/ForecastPanel.test.ts index ee633b0..82bcf9b 100644 --- a/frontend/src/lib/components/ForecastPanel.test.ts +++ b/frontend/src/lib/components/ForecastPanel.test.ts @@ -21,8 +21,8 @@ function model(overrides: Partial = {}): LoadModelView { min_history_days: 30, profile_mae_w: 118.4, ml_mae_w: null, - profile_n: 1344, - ml_n: 0, + compared_n: 0, + compared_at: null, ...overrides, }; } @@ -76,7 +76,7 @@ describe('ForecastPanel: ML vs profile', () => { const profile = compare.querySelector('tr[data-model="profile"]')?.textContent ?? ''; const ml = compare.querySelector('tr[data-model="ml"]')?.textContent ?? ''; expect(profile).toMatch(/118/); - expect(profile).toMatch(/1,344 values/); + expect(compare.textContent).toMatch(/none compared yet/); expect(ml).toMatch(/not scored yet/); expect(text('fc-ml-days')).toBe('12 of 30 days of history'); }); @@ -92,13 +92,15 @@ describe('ForecastPanel: ML vs profile', () => { }, ml_days: 45, ml_mae_w: 97.2, - ml_n: 1344, + compared_n: 1344, + compared_at: '2026-10-30T02:00:00Z', }), ); expect(text('fc-ml-mode')).toBe('Mode: automatic'); expect(text('fc-ml-in-use')).toBe('The plan uses the machine-learning model.'); expect(text('fc-ml-reason')).toMatch(/from the machine-learning model.*97 W against 118 W/); expect(text('fc-ml-days')).toBe('45 of 30 days of history'); + expect(screen.getByTestId('fc-ml-compare').textContent).toMatch(/same 1,344 quarter hours/); }); it('auto with too little history: explains the fallback to the profiles', () => { @@ -112,7 +114,9 @@ describe('ForecastPanel: ML vs profile', () => { ); expect(text('fc-ml-in-use')).toBe('The plan uses the profiles.'); expect(text('fc-ml-reason')).toMatch(/has 3 of the 30 days of history it needs/); - expect(text('fc-ml-days')).toMatch(/Not enough history to forecast yet/); + expect(text('fc-ml-days')).toBe( + '3 of 30 days of history · Not enough history to forecast yet.', + ); }); it('off: no comparison', () => { diff --git a/frontend/src/lib/i18n/locales/de.json b/frontend/src/lib/i18n/locales/de.json index d3afc11..28cbba8 100644 --- a/frontend/src/lib/i18n/locales/de.json +++ b/frontend/src/lib/i18n/locales/de.json @@ -1128,14 +1128,13 @@ "fc.ml.intro": "Ein optionales Modell, das die Grundlast aus ähnlichen vergangenen Tagen vorhersagt (Tagesart, Uhrzeit, Außentemperatur, jüngstes Lastniveau). Es wird neben den Profilen bewertet. Der Plan nutzt es nur im automatischen Modus und nur, solange es in den letzten 14 Tagen deutlich genauer war.", "fc.ml.mode": "Modus: {mode, select, off {aus} shadow {nur Vergleich} other {automatisch}}", "fc.ml.in_use": "Der Plan nutzt {model, select, ml {das maschinelle Modell} other {die Profile}}.", - "fc.ml.compare_caption": "Mittlerer absoluter Fehler (MAE) der letzten 14 Tage, alle Vorlaufzeiten", + "fc.ml.compare_caption": "Mittlerer absoluter Fehler (MAE) der letzten 14 Tage, beide Modelle an denselben {n, plural, =0 {Viertelstunden (noch keine verglichen)} one {# Viertelstunde} other {# Viertelstunden}}", "fc.ml.model": "Modell", "fc.ml.error": "MAE", "fc.ml.profile": "Profile", "fc.ml.ml": "Maschinelles Modell", "fc.ml.not_scored": "noch nicht bewertet", - "fc.ml.days": "{days, number} von {min, number} Tagen Verlauf", - "fc.ml.collecting_model": "Noch zu wenig Verlauf für eine Prognose.", + "fc.ml.days": "{days, number} von {min, number} Tagen Verlauf{ready, select, no { · Noch zu wenig Verlauf für eine Prognose.} other {}}", "forecast.load_model.ml_off": "Das maschinelle Lastmodell ist aus; die Prognose nutzt die Profile.", "forecast.load_model.ml_shadow": "Das maschinelle Lastmodell läuft nur zum Vergleich ({days, number} von {min_days, number} Tagen Verlauf); die Prognose nutzt die Profile.", "forecast.load_model.ml_collecting": "Lastprognose aus den Profilen: Das maschinelle Modell hat {days, number} der nötigen {min_days, number} Tage Verlauf.", diff --git a/frontend/src/lib/i18n/locales/en.json b/frontend/src/lib/i18n/locales/en.json index dd5878f..7f62e5e 100644 --- a/frontend/src/lib/i18n/locales/en.json +++ b/frontend/src/lib/i18n/locales/en.json @@ -1128,14 +1128,13 @@ "fc.ml.intro": "An optional model that predicts the base load from similar past days (day type, time of day, outdoor temperature, recent load level). It is scored next to the profiles. The plan uses it only in the automatic mode, and only while it was clearly more accurate over the last 14 days.", "fc.ml.mode": "Mode: {mode, select, off {off} shadow {comparison only} other {automatic}}", "fc.ml.in_use": "The plan uses {model, select, ml {the machine-learning model} other {the profiles}}.", - "fc.ml.compare_caption": "Mean absolute error (MAE) over the last 14 days, all lead times", + "fc.ml.compare_caption": "Mean absolute error (MAE) over the last 14 days, both models on the same {n, plural, =0 {quarter hours (none compared yet)} one {# quarter hour} other {# quarter hours}}", "fc.ml.model": "Model", "fc.ml.error": "MAE", "fc.ml.profile": "Profiles", "fc.ml.ml": "Machine-learning model", "fc.ml.not_scored": "not scored yet", - "fc.ml.days": "{days, number} of {min, number} days of history", - "fc.ml.collecting_model": "Not enough history to forecast yet.", + "fc.ml.days": "{days, number} of {min, number} days of history{ready, select, no { · Not enough history to forecast yet.} other {}}", "forecast.load_model.ml_off": "The machine-learning load model is off; the forecast uses the profiles.", "forecast.load_model.ml_shadow": "The machine-learning load model runs for comparison only ({days, number} of {min_days, number} days of history); the forecast uses the profiles.", "forecast.load_model.ml_collecting": "Load forecast from the profiles: the machine-learning model has {days, number} of the {min_days, number} days of history it needs.", diff --git a/frontend/src/lib/views/steps/PvStep.svelte b/frontend/src/lib/views/steps/PvStep.svelte index 72aacca..e41d875 100644 --- a/frontend/src/lib/views/steps/PvStep.svelte +++ b/frontend/src/lib/views/steps/PvStep.svelte @@ -330,6 +330,7 @@ -

+

{t('pvf.load_ml_hint', { min: (wizard.forecast.learning ?? DEFAULT_LEARNING).min_history_days, })} diff --git a/suncore/forecast/accuracy.py b/suncore/forecast/accuracy.py index 7c296bc..c5bd485 100644 --- a/suncore/forecast/accuracy.py +++ b/suncore/forecast/accuracy.py @@ -202,26 +202,8 @@ async def compute_accuracy( daylight_only = kind == "pv" issues = await data.forecast_issues(source_id, kind, start - BUCKETS[-1][2]) sums = {bucket: _Sums() for bucket, _, _ in BUCKETS} - for index, issued in enumerate(issues): - following = issues[index + 1] if index + 1 < len(issues) else None - ranges: list[tuple[HorizonBucket, datetime, datetime]] = [] - for bucket, lower, upper in BUCKETS: - lo = _ceil(max(start, issued + lower)) - upper_edge = min(end, issued + upper) - if following is not None: - upper_edge = min(upper_edge, following + lower) # then a newer issue is judged - hi = _ceil(upper_edge) - if hi > lo: - ranges.append((bucket, lo, hi)) - if not ranges: - continue - # One read per issue over the union of its buckets; a forecast slot may start before - # the first quarter hour and still cover it. - first = min(lo for _, lo, _ in ranges) - last = max(hi for _, _, hi in ranges) - forecast = await data.read_forecast( - source_id, kind, issued, TimeRange(start=first - timedelta(hours=1), end=last) - ) + for issued, ranges, read in _issue_ranges(issues, start, end): + forecast = await data.read_forecast(source_id, kind, issued, read) if forecast is None: continue for bucket, lo, hi in ranges: @@ -243,6 +225,98 @@ async def compute_accuracy( ] +def _issue_ranges( + issues: Sequence[datetime], start: datetime, end: datetime +) -> list[tuple[datetime, list[tuple[HorizonBucket, datetime, datetime]], TimeRange]]: + """Per issue the ``(bucket, from, to)`` ranges it is judged on (module doc) and the window + to read: an issue counts for a bucket only inside its lead time and only until a newer + issue takes over; one read per issue over the union of its buckets (a forecast slot may + start before the first quarter hour and still cover it).""" + out: list[tuple[datetime, list[tuple[HorizonBucket, datetime, datetime]], TimeRange]] = [] + for index, issued in enumerate(issues): + following = issues[index + 1] if index + 1 < len(issues) else None + ranges: list[tuple[HorizonBucket, datetime, datetime]] = [] + for bucket, lower, upper in BUCKETS: + lo = _ceil(max(start, issued + lower)) + upper_edge = min(end, issued + upper) + if following is not None: + upper_edge = min(upper_edge, following + lower) # then a newer issue is judged + hi = _ceil(upper_edge) + if hi > lo: + ranges.append((bucket, lo, hi)) + if not ranges: + continue + first = min(lo for _, lo, _ in ranges) + last = max(hi for _, _, hi in ranges) + out.append((issued, ranges, TimeRange(start=first - timedelta(hours=1), end=last))) + return out + + +@dataclass(frozen=True, slots=True) +class PairedAccuracy: + """Two sources scored on exactly the same quarter hours (WP12 model choice). + + Only issues both sources stored at the same time count, and in them only the quarter hours + where both forecasts and the measurement are known, pooled over the horizon buckets; so the + two MAEs compare like with like. ``computed_at`` is when the nightly job computed it.""" + + source_a: str + source_b: str + mae_a_w: float + mae_b_w: float + n: int + computed_at: datetime + + +async def compute_paired_accuracy( + data: AccuracyData, + source_a: str, + source_b: str, + now: datetime, + *, + kind: ForecastKind, + actual_reader: ActualReader, + window_days: int = WINDOW_DAYS, +) -> PairedAccuracy: + """MAE of ``source_a`` and ``source_b`` on their common issues and quarter hours (every + quarter hour counts, as for the load forecast).""" + end = floor_to_step(now, PLANNING_STEP) + start = end - timedelta(days=window_days) + actual = await actual_reader(TimeRange(start=start, end=end)) + since = start - BUCKETS[-1][2] + common = sorted( + set(await data.forecast_issues(source_a, kind, since)) + & set(await data.forecast_issues(source_b, kind, since)) + ) + abs_a = abs_b = 0.0 + n = 0 + for issued, ranges, read in _issue_ranges(common, start, end): + fa = await data.read_forecast(source_a, kind, issued, read) + fb = await data.read_forecast(source_b, kind, issued, read) + if fa is None or fb is None: + continue + for _, lo, hi in ranges: + measured = _slice(actual, lo, hi) + if not len(measured): + continue + grid = Grid(start=measured.start, step=measured.step, count=len(measured)) + pa, pb = resample_power(fa.p50, grid), resample_power(fb.p50, grid) + for value, a, b in zip(measured.values, pa.values, pb.values, strict=True): + if value is None or a is None or b is None: + continue + abs_a += abs(value - a) + abs_b += abs(value - b) + n += 1 + return PairedAccuracy( + source_a=source_a, + source_b=source_b, + mae_a_w=abs_a / n if n else 0.0, + mae_b_w=abs_b / n if n else 0.0, + n=n, + computed_at=now, + ) + + def _ceil(moment: datetime) -> datetime: """``moment`` rounded up to the 15-minute grid.""" return floor_to_step(moment + PLANNING_STEP - timedelta(microseconds=1), PLANNING_STEP) @@ -391,9 +465,11 @@ async def update_accuracy( "ClipFilter", "ForecastKind", "LearnedParams", + "PairedAccuracy", "StoredAccuracy", "accuracy_view", "compute_accuracy", + "compute_paired_accuracy", "load_accuracy", "no_clipping", "quarter_errors", diff --git a/suncore/forecast/load_ml.py b/suncore/forecast/load_ml.py index 05161af..e692102 100644 --- a/suncore/forecast/load_ml.py +++ b/suncore/forecast/load_ml.py @@ -32,11 +32,13 @@ **Model choice** (:func:`choose_load_model`, ``forecast.learning.load_ml_mode``): ``off`` - not computed; ``shadow`` (default) - computed, stored and scored as source ``load_ml``, never used; -``auto`` - used only while it has at least ``min_history_days`` days of history **and** its -rolling 14-day MAE (the same nightly statistic as the profiles', pooled over the horizon buckets) -is at least ``ML_MARGIN`` better than the profiles' on a comparable number of scored quarter hours -(plan §0: "only if measurably better"); otherwise the profiles. Every outcome carries a -:class:`~suncore.core.decisions.Reason`. +``auto`` - used only while it has at least ``min_history_days`` days of history, a current +forecast (a model fitted before yesterday is stale) **and** a fresh paired score (both models on +the same issues and quarter hours of the last 14 days, one nightly run at most +``SCORE_MAX_AGE`` old, at least ``ML_MIN_PAIRED_SAMPLES`` quarter hours) in which its MAE is +``ML_MARGIN`` below the profiles' (plan §0: "only if measurably better"); once in use it stays +until its MAE is no longer below the profiles' (hysteresis). Otherwise the profiles. Every +outcome carries a :class:`~suncore.core.decisions.Reason`. """ from __future__ import annotations @@ -51,7 +53,7 @@ import numpy.typing as npt from suncore.core.decisions import Reason -from suncore.core.forecast import ACCURACY_MIN_SAMPLES, AccuracyRecord +from suncore.core.forecast import AccuracyRecord from suncore.core.series import Grid, QuantileSeries, SeriesUnit, TimeSeries from suncore.forecast.days import local_midnight from suncore.forecast.load import ( @@ -93,9 +95,12 @@ HIGH_QUANTILE: Final = 0.9 ML_MARGIN: Final = 0.02 """``auto`` needs an MAE at least 2 % below the profiles' (measurably better, plan §0).""" -ML_MIN_SCORED_SHARE: Final = 0.9 -"""``auto`` needs the ML forecast scored on at least 90 % as many quarter hours as the profiles -(else it was not stored for most of the 14 days and the two MAEs are not comparable).""" +ML_MIN_PAIRED_SAMPLES: Final = 7 * SLOTS_PER_DAY +"""Quarter hours both models must have been scored on (a week) before ``auto`` compares them.""" +SCORE_MAX_AGE: Final = timedelta(hours=36) +"""A paired score older than this (a nightly job failed or did not run) does not count.""" +ML_MAX_AGE_DAYS: Final = 1 +"""A KNN model fitted before yesterday is stale (its fit keeps failing): no forecast.""" _TYPE_INDEX: Final = {DayType.WORKDAY: 0, DayType.WEEKEND: 1, DayType.HOLIDAY: 2} _TYPE_DISTANCE: Final = np.array( @@ -122,7 +127,8 @@ class KnnLoadModel: fitted_for: date """The local date the model was fitted on (ages count from it).""" recent_level_w: float | None - """Mean base load of the ``LEVEL_DAYS`` days before the fit (the forecast's level).""" + """Mean base load of the ``LEVEL_DAYS`` days before the local midnight of the fit (the + forecast's level, measured like every sample's level).""" slot: IntArray = field(default_factory=lambda: np.zeros(0, dtype=np.int64)) day_type: IntArray = field(default_factory=lambda: np.zeros(0, dtype=np.int64)) temperature: FloatArray = field(default_factory=lambda: np.zeros(0)) @@ -232,8 +238,10 @@ def fit_knn_model( def index_of(moment: datetime) -> int: return (moment - base.start) // step - now_index = index_of(min(now, base.end)) if len(base) else 0 - recent = _window_mean(values, now_index - level_span, now_index) if len(base) else None + # the level of the forecast days: the 7 days before today's local midnight, measured like + # the samples' levels (a partial today would mix in a different time-of-day pattern) + end_index = index_of(midnight) if len(base) else 0 + recent = _window_mean(values, end_index - level_span, end_index) if len(base) else None temps = {} if temperature is None else daily_temperatures(temperature, tz, midnight) slots: list[int] = [] @@ -402,12 +410,24 @@ class PooledError: def pooled_error(records: Sequence[AccuracyRecord]) -> PooledError | None: + """Display figure of one source; the model choice uses :class:`PairedScore` instead.""" n = sum(r.n for r in records) if n <= 0: return None return PooledError(mae_w=sum(r.mae_w * r.n for r in records) / n, n=n) +@dataclass(frozen=True, slots=True) +class PairedScore: + """The profiles and the KNN model scored on the same issues and quarter hours by one nightly + run (:func:`suncore.forecast.accuracy.compute_paired_accuracy`).""" + + profile_mae_w: float + ml_mae_w: float + n: int + computed_at: datetime + + @dataclass(frozen=True, slots=True) class LoadModelChoice: """Which load model the planner gets, and why.""" @@ -417,8 +437,8 @@ class LoadModelChoice: reason: Reason ml_days: int min_history_days: int - profile: PooledError | None - ml: PooledError | None + score: PairedScore | None + """The comparison the choice used (``None``: none, or too old to count).""" def choose_load_model( @@ -427,16 +447,19 @@ def choose_load_model( ml_days: int, ml_available: bool, min_history_days: int, - profile_records: Sequence[AccuracyRecord], - ml_records: Sequence[AccuracyRecord], + score: PairedScore | None, + now: datetime, + ml_in_use: bool = False, ) -> LoadModelChoice: """The model the planner uses (module doc). Pure and deterministic. - ``ml_days``: usable history days of the KNN model; ``ml_available``: it produced a forecast - now; the records are the stored rolling 14-day accuracy of ``load_profile`` and ``load_ml``. + ``ml_days``: usable history days of the KNN model; ``ml_available``: it has a current + forecast; ``score``: the latest paired comparison; ``ml_in_use``: the previous choice was the + KNN model (hysteresis: it is entered at ``ML_MARGIN`` better and left only when it is no + longer better at all). A score older than ``SCORE_MAX_AGE`` counts as missing. """ - profile = pooled_error(profile_records) - ml = pooled_error(ml_records) + if score is not None and now - score.computed_at > SCORE_MAX_AGE: + score = None if mode == "off": in_use: ModelInUse = "profile" reason = Reason(key="forecast.load_model.ml_off") @@ -451,8 +474,8 @@ def choose_load_model( ml_days=ml_days, ml_available=ml_available, min_history_days=min_history_days, - profile=profile, - ml=ml, + score=score, + ml_in_use=ml_in_use, ) return LoadModelChoice( mode=mode, @@ -460,8 +483,7 @@ def choose_load_model( reason=reason, ml_days=ml_days, min_history_days=min_history_days, - profile=profile, - ml=ml, + score=score, ) @@ -470,11 +492,11 @@ def _auto( ml_days: int, ml_available: bool, min_history_days: int, - profile: PooledError | None, - ml: PooledError | None, + score: PairedScore | None, + ml_in_use: bool, ) -> tuple[ModelInUse, Reason]: - """Mode ``auto``: the KNN model only with enough history, a forecast now, a comparable - score and an MAE ``ML_MARGIN`` below the profiles'.""" + """Mode ``auto``: the KNN model only with enough history, a current forecast, a fresh paired + score on ``ML_MIN_PAIRED_SAMPLES`` quarter hours and a smaller MAE (hysteresis).""" if ml_days < min_history_days: return "profile", Reason( key="forecast.load_model.ml_collecting", @@ -482,17 +504,14 @@ def _auto( ) if not ml_available: return "profile", Reason(key="forecast.load_model.ml_unavailable") - needed = max( - ACCURACY_MIN_SAMPLES, - 0 if profile is None else math.ceil(ML_MIN_SCORED_SHARE * profile.n), - ) - if profile is None or ml is None or ml.n < needed: + if score is None or score.n < ML_MIN_PAIRED_SAMPLES: return "profile", Reason( key="forecast.load_model.ml_not_scored", - args={"n": 0 if ml is None else ml.n, "needed": needed}, + args={"n": 0 if score is None else score.n, "needed": ML_MIN_PAIRED_SAMPLES}, ) - args = {"ml_mae_w": round(ml.mae_w), "profile_mae_w": round(profile.mae_w)} # whole W - if ml.mae_w < (1.0 - ML_MARGIN) * profile.mae_w: + args = {"ml_mae_w": round(score.ml_mae_w), "profile_mae_w": round(score.profile_mae_w)} + threshold = 1.0 if ml_in_use else 1.0 - ML_MARGIN + if score.ml_mae_w < threshold * score.profile_mae_w: return "ml", Reason(key="forecast.load_model.ml_in_use", args=args) return "profile", Reason(key="forecast.load_model.ml_not_better", args=args) @@ -503,11 +522,14 @@ def _auto( "LOAD_ML_SOURCE_ID", "ML_HISTORY_DAYS", "ML_MARGIN", - "ML_MIN_SCORED_SHARE", + "ML_MAX_AGE_DAYS", + "ML_MIN_PAIRED_SAMPLES", + "SCORE_MAX_AGE", "KnnLoadModel", "LoadMlMode", "LoadModelChoice", "ModelInUse", + "PairedScore", "PooledError", "choose_load_model", "daily_temperatures", diff --git a/suncore/forecast/load_service.py b/suncore/forecast/load_service.py index af8712f..9c29cb8 100644 --- a/suncore/forecast/load_service.py +++ b/suncore/forecast/load_service.py @@ -64,6 +64,7 @@ from suncore.forecast.accuracy import ( AccuracyData, LearnedParams, + compute_paired_accuracy, load_accuracy, update_accuracy, ) @@ -89,9 +90,11 @@ from suncore.forecast.load_ml import ( LOAD_ML_SOURCE_ID, ML_HISTORY_DAYS, + ML_MAX_AGE_DAYS, KnnLoadModel, LoadMlMode, LoadModelChoice, + PairedScore, choose_load_model, fit_knn_model, forecast_knn, @@ -117,6 +120,10 @@ NS_LOAD_PROFILE: Final = "load_profile" """``learned_param`` namespace (equal to ``suncore.storage.learned.NS_LOAD_PROFILE``).""" DAY_TYPES_KEY: Final = "day_types" +NS_ML_STATE: Final = "ml_state" +"""``learned_param`` namespace of the KNN model (= ``suncore.storage.learned.NS_ML_STATE``).""" +COMPARISON_KEY: Final = "load_comparison" +COMPARISON_SCHEMA: Final = 1 DAY_TYPES_SCHEMA: Final = 1 @@ -224,6 +231,11 @@ def __init__( self._scores: dict[str, tuple[AccuracyRecord, ...]] = {} """Latest rolling 14-day accuracy of ``load_profile`` and ``load_ml``.""" self._scores_loaded = False + self._comparison: PairedScore | None = None + """Latest paired score of both models (decides ``auto``).""" + self._ml_in_use = False + """The last selection used the KNN model (hysteresis).""" + self._ml_refit = False self._lock = asyncio.Lock() self._listeners: list[Callable[[], None]] = [] @@ -393,7 +405,12 @@ async def refit(self, now: datetime | None = None) -> LoadModel | None: await self._load_scores() today = moment.astimezone(tz).date() end = floor_to_step(moment, PLANNING_STEP) - window_days = self._window_days() + ml_due = self.ml_enabled and ( + self._ml_refit or self._ml is None or self._ml.fitted_for != today + ) + # the KNN model is fitted once per local day (its samples and level end at midnight); + # the hourly profile refits read only the profiles' own window + window_days = self._window_days() if ml_due else self._history_days # local midnights (a DST change inside the window must not cut off its first day) window = TimeRange(start=_local_start(today, window_days, tz), end=end) profile_start = _local_start(today, self._history_days, tz) @@ -428,8 +445,8 @@ async def refit(self, now: datetime | None = None) -> LoadModel | None: return self._model self._model, self._fitted_at = model, moment self._base, self._devices = base, devices - if self.ml_enabled: - self._fit_ml( + if ml_due: + await self._fit_ml( full, moment=moment, tz=tz, @@ -454,7 +471,7 @@ async def refit(self, now: datetime | None = None) -> LoadModel | None: ) return model - def _fit_ml( + async def _fit_ml( self, full: BaseLoad, *, @@ -464,9 +481,12 @@ def _fit_ml( temperature: TimeSeries | None, window_days: int, ) -> None: - """Fit the KNN model (a failure keeps the previous one; the profiles are unaffected).""" + """Fit the KNN model in a worker thread (pure NumPy/Python on immutable inputs, so the + event loop keeps running); a failure keeps the previous model (it turns stale after a + day) and leaves the profiles unaffected.""" try: - ml = fit_knn_model( + ml = await asyncio.to_thread( + fit_knn_model, full.series, now=moment, tz=tz, @@ -475,25 +495,35 @@ def _fit_ml( unmeasured=full.unmeasured, history_days=window_days, ) + except asyncio.CancelledError: + raise except Exception: log.exception("forecast.load.ml_fit_failed") return - self._ml = ml + self._ml, self._ml_refit = ml, False log.info("forecast.load.ml_fitted", days=ml.days, samples=ml.samples) async def _load_scores(self) -> None: - """The stored accuracy of both models (once; the nightly job keeps it current).""" + """The stored accuracy of both models and their paired comparison (once; the nightly + job keeps them current). A stored comparison older than ``SCORE_MAX_AGE`` is ignored by + the model choice.""" if self._scores_loaded or self._learned is None or not self.ml_enabled: self._scores_loaded = True return self._scores_loaded = True try: stored = await load_accuracy(self._learned, self.source_ids) + comparison = None + for entry in await self._learned.in_namespace(NS_ML_STATE): + if entry.key == COMPARISON_KEY and entry.schema_version == COMPARISON_SCHEMA: + comparison = _decode_comparison(entry.payload) except Exception: log.exception("forecast.load.scores_unreadable") return for source_id, item in stored.items(): self._scores.setdefault(source_id, item.records) + if self._comparison is None: + self._comparison = comparison # --- temperature ------------------------------------------------------------------ @@ -546,8 +576,9 @@ def daily_temperatures(self, now: datetime) -> dict[date, float]: # --- refresh ---------------------------------------------------------------------- def request_refit(self) -> None: - """Refit at the next refresh (e.g. after the history was rebuilt).""" + """Refit at the next refresh (e.g. after the history was rebuilt), the KNN model too.""" self._fitted_at = None + self._ml_refit = True def _forecast_days(self, now: datetime, tz: tzinfo) -> list[date]: first = now.astimezone(tz).date() @@ -569,10 +600,11 @@ def profile_forecast(self, now: datetime | None = None) -> QuantileSeries | None def ml_forecast(self, now: datetime | None = None) -> QuantileSeries | None: """The KNN forecast on the same grid (``None``: mode ``off`` or too little history).""" - ml = self._ml - if ml is None or not ml.ready or not self.ml_enabled: - return None moment = self._clock() if now is None else now + if not self._ml_current(moment): + return None + ml = self._ml + assert ml is not None tz = self._tz() grid = planning_grid(moment, self._horizon) today = moment.astimezone(tz).date() @@ -584,39 +616,60 @@ def ml_forecast(self, now: datetime | None = None) -> QuantileSeries | None: log.exception("forecast.load.ml_forecast_failed") return None + def _ml_current(self, now: datetime) -> bool: + """The KNN model is on, ready and not stale (fitted today or yesterday).""" + ml = self._ml + if ml is None or not ml.ready or not self.ml_enabled: + return False + today = now.astimezone(self._tz()).date() + return ml.fitted_for >= today - timedelta(days=ML_MAX_AGE_DAYS) + def set_scores(self, scores: Mapping[str, Sequence[AccuracyRecord]]) -> None: - """Replace the rolling accuracy the model choice uses (the nightly job's result).""" + """Replace the per-source accuracy shown next to the comparison (display only).""" for source_id, records in scores.items(): self._scores[source_id] = tuple(records) - def choice(self, ml_available: bool | None = None) -> LoadModelChoice: + def set_comparison(self, comparison: PairedScore | None) -> None: + """The paired comparison the model choice uses (the nightly job's result).""" + self._comparison = comparison + + def choice( + self, now: datetime | None = None, *, ml_available: bool | None = None + ) -> LoadModelChoice: """Which model :meth:`forecast` returns and why (:func:`choose_load_model`). - ``ml_available`` defaults to "the KNN model is fitted and ready".""" + ``ml_available`` defaults to "the KNN model is on, ready and not stale".""" + moment = self._clock() if now is None else now ml = self._ml if ml_available is None: - ml_available = ml is not None and ml.ready + ml_available = self._ml_current(moment) return choose_load_model( self._inputs.ml_mode, ml_days=0 if ml is None else ml.days, ml_available=ml_available, min_history_days=self._inputs.min_history_days, - profile_records=self._scores.get(LOAD_SOURCE_ID, ()), - ml_records=self._scores.get(LOAD_ML_SOURCE_ID, ()), + score=self._comparison, + now=moment, + ml_in_use=self._ml_in_use, ) def selected( self, now: datetime | None = None ) -> tuple[QuantileSeries | None, LoadModelChoice]: """The forecast the planner uses and the choice behind it: the KNN forecast only when - the choice selects it and it exists, else the profile forecast.""" - choice = self.choice() + the choice selects it and it exists, else the profile forecast. Remembers the outcome + for the hysteresis of the next choice.""" + moment = self._clock() if now is None else now + choice = self.choice(moment) + forecast = None if choice.in_use == "ml": - ml = self.ml_forecast(now) - if ml is not None: - return ml, choice - choice = self.choice(ml_available=False) - return self.profile_forecast(now), choice + forecast = self.ml_forecast(moment) + if forecast is None: + choice = self.choice(moment, ml_available=False) + if forecast is None: + forecast = self.profile_forecast(moment) + self._ml_in_use = choice.in_use == "ml" + return forecast, choice def forecast(self, now: datetime | None = None) -> QuantileSeries | None: """The base load forecast the planner and the risk model use, from the current slot for @@ -648,15 +701,14 @@ async def refresh(self) -> QuantileSeries | None: else: await self._resolve(self._forecast_days(now, tz), now.astimezone(tz).date()) await self._update_weather(now) - profile = self.profile_forecast(now) - ml = self.ml_forecast(now) - if profile is not None: - await self._maybe_store(LOAD_SOURCE_ID, profile, now) - if ml is not None: - await self._maybe_store(LOAD_ML_SOURCE_ID, ml, now) - forecast = profile - if ml is not None and self.choice().in_use == "ml": - forecast = ml + forecast, choice = self.selected(now) + profile = forecast if choice.in_use == "profile" else self.profile_forecast(now) + if profile is not None and await self._maybe_store(LOAD_SOURCE_ID, profile, now): + # the KNN forecast is stored with the same issue time as the profiles, so the + # nightly comparison pairs every issue; computed only when due or in use + ml = forecast if choice.in_use == "ml" else self.ml_forecast(now) + if ml is not None: + await self._maybe_store(LOAD_ML_SOURCE_ID, ml, now, force=True) except asyncio.CancelledError: raise except Exception: @@ -683,20 +735,24 @@ def _notify(self) -> None: except Exception: log.exception("forecast.load.listener_failed") - async def _maybe_store(self, source_id: str, forecast: QuantileSeries, now: datetime) -> None: + async def _maybe_store( + self, source_id: str, forecast: QuantileSeries, now: datetime, *, force: bool = False + ) -> bool: + """Store one issue unless one was stored within the store interval; ``True`` if stored.""" if self._data is None: - return + return False last = self._last_stored_at.get(source_id) - if last is not None and now - last < self._store_interval: - return + if not force and last is not None and now - last < self._store_interval: + return False try: await self._data.store_forecast(source_id, "load", now, forecast) except asyncio.CancelledError: raise except Exception: log.exception("forecast.load.store_failed", source=source_id) - return + return False self._last_stored_at[source_id] = now + return True async def run(self) -> None: """Refresh now, then every ``refresh_interval_s``.""" @@ -722,11 +778,55 @@ async def nightly(self, now: datetime | None = None) -> list[AccuracyRecord]: actual_reader=self.base_load_series, ) self.set_scores(results) + if LOAD_ML_SOURCE_ID not in results: # failed or off: no stale row is shown + self._scores.pop(LOAD_ML_SOURCE_ID, None) records = results.get(LOAD_SOURCE_ID, []) + if self.ml_enabled: + await self._update_comparison(moment) async with self._lock: await self.refit(moment) return records + async def _update_comparison(self, now: datetime) -> None: + """The paired score of both models (same issues and quarter hours); on failure the old + one is dropped, so ``auto`` never decides on a stale or partial comparison.""" + assert self._data is not None + assert self._learned is not None + try: + paired = await compute_paired_accuracy( + self._data, + LOAD_SOURCE_ID, + LOAD_ML_SOURCE_ID, + now, + kind="load", + actual_reader=self.base_load_series, + ) + score = PairedScore( + profile_mae_w=paired.mae_a_w, + ml_mae_w=paired.mae_b_w, + n=paired.n, + computed_at=paired.computed_at, + ) + await self._learned.put( + NS_ML_STATE, + COMPARISON_KEY, + _encode_comparison(score), + schema_version=COMPARISON_SCHEMA, + ) + except asyncio.CancelledError: + raise + except Exception: + log.exception("forecast.load.comparison_failed") + self._comparison = None + return + self._comparison = score + log.info( + "forecast.load.comparison", + profile_mae_w=round(score.profile_mae_w, 1), + ml_mae_w=round(score.ml_mae_w, 1), + n=score.n, + ) + # --- view ---------------------------------------------------------------------------- def view(self, hours: int = 48) -> LoadForecastView: @@ -798,21 +898,46 @@ def view(self, hours: int = 48) -> LoadForecastView: ) def model_view(self, choice: LoadModelChoice | None = None) -> LoadModelView: - """ "ML vs profile": the model in use, why, and both rolling errors.""" + """ "ML vs profile": the model in use, why, and the paired comparison it rests on.""" choice = self.choice() if choice is None else choice - ml = self._ml + score = choice.score return LoadModelView( mode=choice.mode, in_use=choice.in_use, reason=choice.reason, - ml_ready=self.ml_enabled and ml is not None and ml.ready, + ml_ready=self._ml_current(self._clock()), ml_days=choice.ml_days, min_history_days=choice.min_history_days, - profile_mae_w=None if choice.profile is None else round(choice.profile.mae_w, 1), - ml_mae_w=None if choice.ml is None else round(choice.ml.mae_w, 1), - profile_n=0 if choice.profile is None else choice.profile.n, - ml_n=0 if choice.ml is None else choice.ml.n, + profile_mae_w=None if score is None else round(score.profile_mae_w, 1), + ml_mae_w=None if score is None else round(score.ml_mae_w, 1), + compared_n=0 if score is None else score.n, + compared_at=None if score is None else score.computed_at, + ) + + +def _encode_comparison(score: PairedScore) -> dict[str, Any]: + return { + "profile_mae_w": score.profile_mae_w, + "ml_mae_w": score.ml_mae_w, + "n": score.n, + "computed_at": score.computed_at.isoformat(), + } + + +def _decode_comparison(payload: Mapping[str, Any]) -> PairedScore | None: + try: + computed = datetime.fromisoformat(str(payload["computed_at"])) + if computed.tzinfo is None: + return None + return PairedScore( + profile_mae_w=float(payload["profile_mae_w"]), + ml_mae_w=float(payload["ml_mae_w"]), + n=int(payload["n"]), + computed_at=computed, ) + except (KeyError, ValueError, TypeError): + log.warning("forecast.load.comparison_unreadable") + return None def _local_start(today: date, days: int, tz: tzinfo) -> datetime: diff --git a/suncore/forecast/views.py b/suncore/forecast/views.py index cbdfa69..0b6e86a 100644 --- a/suncore/forecast/views.py +++ b/suncore/forecast/views.py @@ -140,21 +140,24 @@ class LoadModelView(_View): """Which load model the planner gets: the profiles or the optional KNN model (WP12). ``mode`` is ``forecast.learning.load_ml_mode``; ``reason`` explains ``in_use`` - (``forecast.load_model.*``). The errors are the rolling 14-day MAE of ``load_profile`` and - ``load_ml`` pooled over the horizon buckets (``None``: not scored yet); ``ml_days`` counts the - usable history days of the KNN model, which ``auto`` needs ``min_history_days`` of.""" + (``forecast.load_model.*``). The two errors are the paired comparison ``auto`` decides on: + the 14-day MAE of ``load_profile`` and ``load_ml`` on the same issues and quarter hours + (``compared_n``), computed by the nightly job at ``compared_at`` (``None``: none yet, or + older than 36 h). ``ml_days`` counts the usable history days of the KNN model, which + ``auto`` needs ``min_history_days`` of.""" mode: LoadMlMode in_use: ModelInUse reason: Reason ml_ready: bool - """The KNN model has enough history to forecast (7 days) and produced a forecast.""" + """The KNN model is on, has its 7 days of history and was fitted today or yesterday, so it + can forecast.""" ml_days: int min_history_days: int profile_mae_w: float | None ml_mae_w: float | None - profile_n: int - ml_n: int + compared_n: int + compared_at: AwareDatetime | None class ForecastAccuracyView(_View): diff --git a/tests/api/test_forecast_api.py b/tests/api/test_forecast_api.py index 69bb33b..3f3ed7f 100644 --- a/tests/api/test_forecast_api.py +++ b/tests/api/test_forecast_api.py @@ -186,8 +186,8 @@ async def test_accuracy_compares_the_ml_model_with_the_profiles(services: FakeSe min_history_days=30, profile_mae_w=120.0, ml_mae_w=90.0, - profile_n=1344, - ml_n=1344, + compared_n=1344, + compared_at=NOW, ) } ) diff --git a/tests/forecast/test_load_ml.py b/tests/forecast/test_load_ml.py index 4152480..3ca5684 100644 --- a/tests/forecast/test_load_ml.py +++ b/tests/forecast/test_load_ml.py @@ -13,13 +13,16 @@ from hypothesis import given, settings from hypothesis import strategies as st -from suncore.core.forecast import ACCURACY_MIN_SAMPLES, AccuracyRecord, HorizonBucket +from suncore.core.forecast import AccuracyRecord, HorizonBucket from suncore.core.series import SeriesUnit, TimeSeries, planning_grid from suncore.forecast.load import DayType from suncore.forecast.load_ml import ( KNN_MIN_DAYS, ML_MARGIN, + ML_MIN_PAIRED_SAMPLES, + SCORE_MAX_AGE, LoadMlMode, + PairedScore, choose_load_model, fit_knn_model, forecast_knn, @@ -194,6 +197,34 @@ def test_spring_forward_day_has_a_forecast_for_every_slot() -> None: assert all(v is not None for v in forecast.p50.values) +def test_autumn_dst_day_is_a_sample_day_and_forecast() -> None: + """25 October 2026 has 25 hours: it counts as a day (the repeated hour averaged) and a + forecast across it has every slot.""" + start = datetime(2026, 9, 28, tzinfo=BERLIN).astimezone(UTC) # a Monday + end = datetime(2026, 11, 2, tzinfo=BERLIN).astimezone(UTC) + base = series([weekly(start + Q * i) for i in range((end - start) // Q)], start) + now = datetime(2026, 11, 2, 9, tzinfo=BERLIN).astimezone(UTC) + model = fit_knn_model(base, now=now, tz=BERLIN, day_types={}) + assert model.days == 35 + assert model.samples == 35 * 96 + before = datetime(2026, 10, 24, 10, tzinfo=BERLIN).astimezone(UTC) + grid = planning_grid(before, timedelta(hours=48)) # 24 Oct 10:00 to 26 Oct 09:00 + forecast = forecast_knn(model, grid, BERLIN, {}) + assert forecast is not None + assert len(forecast) == grid.count + assert all(v is not None for v in forecast.p50.values) + sunday_noon = datetime(2026, 10, 25, 11, 0, tzinfo=BERLIN).astimezone(UTC) + assert p50_at(forecast.p50.values, grid.start, sunday_noon) == pytest.approx(500, abs=1) + + +def test_the_forecast_level_ends_at_local_midnight_like_the_samples() -> None: + """Today's partial morning (a spike) does not move the level of the forecast days.""" + values: list[float | None] = [400.0] * (96 * 21) + values += [4000.0] * 40 # today until 10:00 + model = fit_knn_model(series(values), now=at(21, 10), tz=BERLIN, day_types={}) + assert model.recent_level_w == pytest.approx(400) + + # --- properties -------------------------------------------------------------------------- @@ -277,12 +308,15 @@ def test_weighted_quantiles_match_the_reference(pairs: list[tuple[float, float]] # --- model choice ------------------------------------------------------------------------ +SCORED_AT = datetime(2026, 9, 1, 1, 0, tzinfo=UTC) + -def records(source: str, mae: float, n: int = 1344) -> list[AccuracyRecord]: - return [ - AccuracyRecord(source_id=source, horizon=h, mae_w=mae, bias_w=0.0, n=n) - for h in HorizonBucket - ] +def score( + ml: float = 80.0, profile: float = 100.0, n: int = 1344, age_h: float = 2.0 +) -> PairedScore: + return PairedScore( + profile_mae_w=profile, ml_mae_w=ml, n=n, computed_at=SCORED_AT - timedelta(hours=age_h) + ) def choose( @@ -291,16 +325,18 @@ def choose( ml_days: int = 45, ml_available: bool = True, min_history_days: int = 30, - profile: list[AccuracyRecord] | None = None, - ml: list[AccuracyRecord] | None = None, + paired: PairedScore | None = None, + no_score: bool = False, + ml_in_use: bool = False, ) -> tuple[str, str]: choice = choose_load_model( mode, ml_days=ml_days, ml_available=ml_available, min_history_days=min_history_days, - profile_records=records("load_profile", 100.0) if profile is None else profile, - ml_records=records("load_ml", 80.0) if ml is None else ml, + score=None if no_score else (paired or score()), + now=SCORED_AT, + ml_in_use=ml_in_use, ) return choice.in_use, choice.reason.key @@ -308,16 +344,27 @@ def choose( def test_off_and_shadow_never_use_the_model() -> None: assert choose("off") == ("profile", "forecast.load_model.ml_off") assert choose("shadow") == ("profile", "forecast.load_model.ml_shadow") + assert choose("shadow", ml_in_use=True) == ("profile", "forecast.load_model.ml_shadow") def test_auto_uses_the_model_only_when_it_is_measurably_better() -> None: assert choose() == ("ml", "forecast.load_model.ml_in_use") - better_by_less_than_the_margin = 100.0 * (1 - ML_MARGIN / 2) - assert choose(ml=records("load_ml", better_by_less_than_the_margin)) == ( + within_margin = 100.0 * (1 - ML_MARGIN / 2) + assert choose(paired=score(ml=within_margin)) == ( + "profile", + "forecast.load_model.ml_not_better", + ) + assert choose(paired=score(ml=120.0)) == ("profile", "forecast.load_model.ml_not_better") + + +def test_hysteresis_keeps_the_model_until_it_is_no_longer_better() -> None: + within_margin = score(ml=100.0 * (1 - ML_MARGIN / 2)) + assert choose(paired=within_margin, ml_in_use=True) == ("ml", "forecast.load_model.ml_in_use") + assert choose(paired=within_margin, ml_in_use=False)[0] == "profile" + assert choose(paired=score(ml=100.0), ml_in_use=True) == ( "profile", "forecast.load_model.ml_not_better", ) - assert choose(ml=records("load_ml", 120.0)) == ("profile", "forecast.load_model.ml_not_better") def test_auto_needs_enough_history() -> None: @@ -326,20 +373,28 @@ def test_auto_needs_enough_history() -> None: assert choose(ml_days=10, min_history_days=10) == ("ml", "forecast.load_model.ml_in_use") -def test_auto_needs_a_forecast_and_a_comparable_score() -> None: +def test_auto_needs_a_forecast_and_a_fresh_comparable_score() -> None: assert choose(ml_available=False) == ("profile", "forecast.load_model.ml_unavailable") - assert choose(ml=[]) == ("profile", "forecast.load_model.ml_not_scored") - assert choose(profile=[]) == ("profile", "forecast.load_model.ml_not_scored") - # scored on 3 of the profiles' 14 days (stored only for the last few days) - assert choose(ml=records("load_ml", 50.0, n=288)) == ( - "profile", - "forecast.load_model.ml_not_scored", - ) - thin = records("load_ml", 50.0, n=ACCURACY_MIN_SAMPLES // 6) - assert choose(profile=records("load_profile", 100.0, n=10), ml=thin) == ( - "profile", - "forecast.load_model.ml_not_scored", + assert choose(no_score=True) == ("profile", "forecast.load_model.ml_not_scored") + thin = score(ml=10.0, n=ML_MIN_PAIRED_SAMPLES - 1) + assert choose(paired=thin) == ("profile", "forecast.load_model.ml_not_scored") + hours = SCORE_MAX_AGE.total_seconds() / 3600 + assert choose(paired=score(age_h=hours)) == ("ml", "forecast.load_model.ml_in_use") + stale = score(ml=10.0, age_h=hours + 0.25) # a nightly job failed: not used, even if better + assert choose(paired=stale) == ("profile", "forecast.load_model.ml_not_scored") + + +def test_a_stale_score_is_not_reported() -> None: + choice = choose_load_model( + "auto", + ml_days=45, + ml_available=True, + min_history_days=30, + score=score(age_h=48), + now=SCORED_AT, ) + assert choice.score is None + assert choice.reason.args == {"n": 0, "needed": ML_MIN_PAIRED_SAMPLES} @given( @@ -349,8 +404,9 @@ def test_auto_needs_a_forecast_and_a_comparable_score() -> None: available=st.booleans(), profile_mae=st.floats(0, 5000), ml_mae=st.floats(0, 5000), - profile_n=st.integers(0, 5000), - ml_n=st.integers(0, 5000), + n=st.integers(0, 5000), + age_h=st.floats(0, 100), + in_use=st.booleans(), ) def test_the_model_is_used_only_under_every_rule( *, @@ -360,27 +416,29 @@ def test_the_model_is_used_only_under_every_rule( available: bool, profile_mae: float, ml_mae: float, - profile_n: int, - ml_n: int, + n: int, + age_h: float, + in_use: bool, ) -> None: choice = choose_load_model( mode, ml_days=ml_days, ml_available=available, min_history_days=min_days, - profile_records=records("load_profile", profile_mae, profile_n) if profile_n else [], - ml_records=records("load_ml", ml_mae, ml_n) if ml_n else [], + score=score(ml_mae, profile_mae, n, age_h), + now=SCORED_AT, + ml_in_use=in_use, ) assert choice.reason.key.startswith("forecast.load_model.") if choice.in_use == "ml": assert mode == "auto" assert available assert ml_days >= min_days - assert ml_n >= profile_n * 0.9 - assert 3 * ml_n >= ACCURACY_MIN_SAMPLES + assert n >= ML_MIN_PAIRED_SAMPLES + assert age_h <= SCORE_MAX_AGE.total_seconds() / 3600 assert ml_mae < profile_mae - assert choice.ml_days == ml_days - assert choice.min_history_days == min_days + if not in_use: + assert ml_mae < (1 - ML_MARGIN) * profile_mae def test_pooled_error_weights_the_buckets_by_their_quarter_hours() -> None: diff --git a/tests/forecast/test_load_ml_service.py b/tests/forecast/test_load_ml_service.py index 7dc434b..dacdbdd 100644 --- a/tests/forecast/test_load_ml_service.py +++ b/tests/forecast/test_load_ml_service.py @@ -10,13 +10,24 @@ import pytest -from suncore.core.forecast import AccuracyRecord, HorizonBucket +from suncore.core.forecast import HorizonBucket from suncore.core.measurements import SITE_HOUSE_W, device_power_key from suncore.core.series import SeriesUnit, TimeSeries -from suncore.forecast.accuracy import load_accuracy, save_accuracy +from suncore.forecast import accuracy as accuracy_module +from suncore.forecast.accuracy import load_accuracy from suncore.forecast.load import LOAD_SOURCE_ID -from suncore.forecast.load_ml import LOAD_ML_SOURCE_ID, LoadMlMode -from suncore.forecast.load_service import LoadForecastService, LoadInputs +from suncore.forecast.load_ml import ( + LOAD_ML_SOURCE_ID, + ML_MIN_PAIRED_SAMPLES, + LoadMlMode, + PairedScore, +) +from suncore.forecast.load_service import ( + COMPARISON_KEY, + NS_ML_STATE, + LoadForecastService, + LoadInputs, +) from suncore.storage import Database, LearnedParamStore from suncore.storage.timeseries import TimeseriesStore, open_timeseries @@ -85,34 +96,36 @@ def service( ) -def records(source: str, mae: float) -> list[AccuracyRecord]: - return [ - AccuracyRecord(source_id=source, horizon=h, mae_w=mae, bias_w=0.0, n=1344) - for h in HorizonBucket - ] +def paired(ml: float, profile: float = 100.0, at: datetime | None = None) -> PairedScore: + return PairedScore( + profile_mae_w=profile, + ml_mae_w=ml, + n=ML_MIN_PAIRED_SAMPLES * 2, + computed_at=at or moment(35, 3), + ) -async def test_shadow_is_stored_and_shown_but_never_used( +async def test_shadow_is_stored_with_the_profiles_and_never_used( store: TimeseriesStore, learned: LearnedParamStore ) -> None: await fill_history(store, 35) - now = moment(35) - svc = service(store, learned, Clock(now), "shadow") - svc.set_scores( - { - LOAD_SOURCE_ID: records(LOAD_SOURCE_ID, 100), - LOAD_ML_SOURCE_ID: records(LOAD_ML_SOURCE_ID, 10), - } - ) + clock = Clock(moment(35)) + svc = service(store, learned, clock, "shadow") + svc.set_comparison(paired(10)) forecast = await svc.refresh() assert forecast is not None - assert forecast == svc.profile_forecast(now) # even though the ML model scored better - ml = svc.ml_forecast(now) + assert forecast == svc.profile_forecast() # even though the ML model scored better + ml = svc.ml_forecast() assert ml is not None assert ml.start == forecast.start assert len(ml) == len(forecast) - assert await store.forecast_issues(LOAD_SOURCE_ID, "load") == [now] - assert await store.forecast_issues(LOAD_ML_SOURCE_ID, "load") == [now] + clock.now = moment(35, 10, 35) + await svc.refresh() # not due + clock.now = moment(35, 11, 5) + await svc.refresh() + profile_issues = await store.forecast_issues(LOAD_SOURCE_ID, "load") + assert profile_issues == [moment(35), moment(35, 11, 5)] + assert await store.forecast_issues(LOAD_ML_SOURCE_ID, "load") == profile_issues # paired assert svc.source_ids == (LOAD_SOURCE_ID, LOAD_ML_SOURCE_ID) assert svc.planning_reasons() == () view = svc.view() @@ -126,6 +139,7 @@ async def test_shadow_is_stored_and_shown_but_never_used( assert view.model.reason.args == {"days": 35, "min_days": 30} assert view.model.ml_mae_w == 10 assert view.model.profile_mae_w == 100 + assert view.model.compared_n == ML_MIN_PAIRED_SAMPLES * 2 async def test_off_computes_nothing(store: TimeseriesStore, learned: LearnedParamStore) -> None: @@ -149,12 +163,7 @@ async def test_auto_with_too_little_history_keeps_the_profiles( await fill_history(store, 20) now = moment(20) svc = service(store, learned, Clock(now), "auto") - svc.set_scores( - { - LOAD_SOURCE_ID: records(LOAD_SOURCE_ID, 100), - LOAD_ML_SOURCE_ID: records(LOAD_ML_SOURCE_ID, 10), - } - ) + svc.set_comparison(paired(10, at=now)) forecast = await svc.refresh() assert forecast == svc.profile_forecast(now) (reason,) = svc.planning_reasons() @@ -162,7 +171,7 @@ async def test_auto_with_too_little_history_keeps_the_profiles( assert reason.args == {"days": 20, "min_days": 30} -async def test_auto_uses_the_model_while_it_is_better_and_falls_back( +async def test_auto_uses_the_model_while_it_is_better_with_hysteresis( store: TimeseriesStore, learned: LearnedParamStore ) -> None: await fill_history(store, 35) @@ -172,15 +181,11 @@ async def test_auto_uses_the_model_while_it_is_better_and_falls_back( assert svc.choice().reason.key == "forecast.load_model.ml_not_scored" # nothing scored yet assert svc.forecast(now) == svc.profile_forecast(now) - svc.set_scores( - { - LOAD_SOURCE_ID: records(LOAD_SOURCE_ID, 100), - LOAD_ML_SOURCE_ID: records(LOAD_ML_SOURCE_ID, 60), - } - ) + svc.set_comparison(paired(99)) # better, but by less than the margin + assert svc.forecast(now) == svc.profile_forecast(now) + svc.set_comparison(paired(60)) ml = svc.ml_forecast(now) assert ml is not None - assert svc.forecast(now) == ml load, choice = svc.selected(now) assert load == ml assert svc.planning_reasons(choice)[0].key == "forecast.load_model.ml_in_use" @@ -188,9 +193,27 @@ async def test_auto_uses_the_model_while_it_is_better_and_falls_back( assert view.source_id == LOAD_ML_SOURCE_ID assert view.p50_w == list(ml.p50.values[: 48 * 4]) - svc.set_scores({LOAD_ML_SOURCE_ID: records(LOAD_ML_SOURCE_ID, 120)}) # worse now + svc.set_comparison(paired(99)) # still better: it stays in use (hysteresis) + assert svc.forecast(now) == ml + svc.set_comparison(paired(100)) # no longer better assert svc.forecast(now) == svc.profile_forecast(now) assert svc.choice().reason.key == "forecast.load_model.ml_not_better" + svc.set_comparison(paired(99)) # entering again needs the margin + assert svc.forecast(now) == svc.profile_forecast(now) + + +async def test_auto_ignores_a_stale_comparison( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store, 35) + now = moment(35) + svc = service(store, learned, Clock(now), "auto") + await svc.refresh() + svc.set_comparison(paired(10, at=now - timedelta(hours=37))) # last night's job failed + load, choice = svc.selected(now) + assert load == svc.profile_forecast(now) + assert choice.reason.key == "forecast.load_model.ml_not_scored" + assert svc.model_view().compared_n == 0 async def test_auto_falls_back_when_the_model_has_no_forecast( @@ -200,12 +223,7 @@ async def test_auto_falls_back_when_the_model_has_no_forecast( now = moment(35) svc = service(store, learned, Clock(now), "auto") await svc.refresh() - svc.set_scores( - { - LOAD_SOURCE_ID: records(LOAD_SOURCE_ID, 100), - LOAD_ML_SOURCE_ID: records(LOAD_ML_SOURCE_ID, 60), - } - ) + svc.set_comparison(paired(60)) def broken(*_args: object, **_kwargs: object) -> None: msg = "boom" @@ -218,6 +236,58 @@ def broken(*_args: object, **_kwargs: object) -> None: assert choice.reason.key == "forecast.load_model.ml_unavailable" +async def test_a_stale_model_is_not_used( + store: TimeseriesStore, learned: LearnedParamStore, monkeypatch: pytest.MonkeyPatch +) -> None: + """The KNN fit keeps failing: yesterday's model still forecasts, an older one does not.""" + await fill_history(store, 35) + clock = Clock(moment(35)) + svc = service(store, learned, clock, "auto") + await svc.refresh() + fitted = svc.ml_model + assert fitted is not None + + def broken(*_args: object, **_kwargs: object) -> None: + msg = "boom" + raise RuntimeError(msg) + + monkeypatch.setattr("suncore.forecast.load_service.fit_knn_model", broken) + clock.now = moment(36) + svc.set_comparison(paired(60, at=clock.now)) + await svc.refresh() + assert svc.ml_model is fitted # kept + assert svc.view().fitted_at == clock.now # the profiles refit regardless + assert svc.choice().in_use == "ml" + clock.now = moment(37) + svc.set_comparison(paired(60, at=clock.now)) + await svc.refresh() + load, choice = svc.selected() + assert choice.reason.key == "forecast.load_model.ml_unavailable" + assert load == svc.profile_forecast() + assert svc.ml_forecast() is None + assert not svc.model_view().ml_ready + + +async def test_the_model_is_fitted_once_per_day_unless_asked( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + await fill_history(store, 35) + now = moment(35) + svc = service(store, learned, Clock(now), "shadow") + await svc.refit(now) + first = svc.ml_model + await svc.refit(now + timedelta(hours=1)) + assert svc.ml_model is first + svc.request_refit() + await svc.refit(now + timedelta(hours=2)) + assert svc.ml_model is not first + second = svc.ml_model + await svc.refit(moment(36, 0, 30)) + assert svc.ml_model is not second + assert svc.ml_model is not None + assert svc.ml_model.fitted_for == moment(36).astimezone(BERLIN).date() + + async def test_the_profiles_keep_their_own_window( store: TimeseriesStore, learned: LearnedParamStore ) -> None: @@ -237,6 +307,20 @@ async def test_the_profiles_keep_their_own_window( assert with_ml.model_view().ml_days == 70 +async def test_autumn_dst_inside_the_window_keeps_56_profile_days( + store: TimeseriesStore, learned: LearnedParamStore +) -> None: + """3 Aug + 100 days = 11 Nov: 25 Oct (25 hours) lies inside both windows.""" + await fill_history(store, 100) + now = moment(100) + svc = service(store, learned, Clock(now), "shadow") + await svc.refit(now) + assert svc.model is not None + assert svc.model.days == 56 + assert svc.ml_model is not None + assert svc.ml_model.days == 91 + + async def test_long_min_history_widens_the_window( store: TimeseriesStore, learned: LearnedParamStore ) -> None: @@ -248,13 +332,13 @@ async def test_long_min_history_widens_the_window( assert svc.ml_model.days == 95 -async def test_nightly_scores_both_models_and_keeps_the_result( +async def test_nightly_scores_both_models_on_the_same_quarter_hours( store: TimeseriesStore, learned: LearnedParamStore ) -> None: await fill_history(store, 16) clock = Clock(moment(14)) svc = service(store, learned, clock, "shadow") - await svc.refresh() # issues of both models + await svc.refresh() # issues of both models, same time clock.now = moment(16, 3, 0) records_ = await svc.nightly() assert {r.source_id for r in records_} == {LOAD_SOURCE_ID} @@ -263,23 +347,48 @@ async def test_nightly_scores_both_models_and_keeps_the_result( ml = {r.horizon: r for r in stored[LOAD_ML_SOURCE_ID].records} profile = {r.horizon: r for r in stored[LOAD_SOURCE_ID].records} assert ml[HorizonBucket.H0_6].n == profile[HorizonBucket.H0_6].n > 0 - assert ml[HorizonBucket.H0_6].mae_w == pytest.approx(0, abs=1) # same metric, same actuals view = svc.model_view() - assert view.ml_n == sum(r.n for r in ml.values()) - assert view.profile_n == sum(r.n for r in profile.values()) + assert view.compared_n == sum(r.n for r in profile.values()) # every quarter hour paired + assert view.compared_at == clock.now + assert view.ml_mae_w == pytest.approx(0, abs=1) # the house is the same every day + assert view.profile_mae_w is not None + assert view.profile_mae_w < 20 # the profiles' kernel smoothing blurs the evening step + rows = await learned.in_namespace(NS_ML_STATE) + assert [r.key for r in rows] == [COMPARISON_KEY] + # a later service (restart) reads the comparison back + again = service(store, learned, clock, "shadow") + await again.refit() + assert again.model_view().compared_n == view.compared_n -async def test_scores_of_an_earlier_run_are_read_at_the_first_fit( - store: TimeseriesStore, learned: LearnedParamStore + +async def test_a_failed_comparison_or_ml_score_leaves_nothing_stale( + store: TimeseriesStore, learned: LearnedParamStore, monkeypatch: pytest.MonkeyPatch ) -> None: - await fill_history(store, 35) - now = moment(35) - await save_accuracy(learned, LOAD_SOURCE_ID, records(LOAD_SOURCE_ID, 100), now=now) - await save_accuracy(learned, LOAD_ML_SOURCE_ID, records(LOAD_ML_SOURCE_ID, 50), now=now) - svc = service(store, learned, Clock(now), "auto") + await fill_history(store, 16) + clock = Clock(moment(14)) + svc = service(store, learned, clock, "auto", min_history_days=7) await svc.refresh() - assert svc.choice().in_use == "ml" - assert svc.model_view().ml_mae_w == 50 + svc.set_comparison(paired(10, at=moment(16, 2))) + + async def broken(*_args: object, **_kwargs: object) -> None: + msg = "boom" + raise RuntimeError(msg) + + monkeypatch.setattr("suncore.forecast.load_service.compute_paired_accuracy", broken) + + async def only_profile(data, learned_, source_ids, now, **kwargs): # type: ignore[no-untyped-def] + return await accuracy_module.update_accuracy( + data, learned_, [LOAD_SOURCE_ID], now, **kwargs + ) + + monkeypatch.setattr("suncore.forecast.load_service.update_accuracy", only_profile) + svc.set_scores({LOAD_ML_SOURCE_ID: []}) + clock.now = moment(16, 3) + await svc.nightly() + assert svc.choice().reason.key == "forecast.load_model.ml_not_scored" + assert svc.model_view().compared_n == 0 + assert LOAD_ML_SOURCE_ID not in svc._scores async def test_without_storage_nothing_breaks() -> None: