From 08507dd89ab016f6f0a5a61730d18ff1efb6de0e Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 12:41:26 -0700 Subject: [PATCH 01/30] Add design spec for synthetic history generation When cloned panels lack HA recorder data, the simulator will project one year of historical data using existing circuit/BESS/EVSE config and modulation infrastructure, stored in a companion SQLite file mirroring HA's recorder schema. --- ...-26-synthetic-history-generation-design.md | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 docs/superpowers/specs/2026-03-26-synthetic-history-generation-design.md diff --git a/docs/superpowers/specs/2026-03-26-synthetic-history-generation-design.md b/docs/superpowers/specs/2026-03-26-synthetic-history-generation-design.md new file mode 100644 index 0000000..e5f4713 --- /dev/null +++ b/docs/superpowers/specs/2026-03-26-synthetic-history-generation-design.md @@ -0,0 +1,175 @@ +# Synthetic History Generation for Cloned Panels + +## Problem + +When a user clones a template to create a panel config and no HA recorder data is available, the simulator has no historical data to replay. The simulator should project what the history would have been using the circuit data, BESS, and EVSE configuration it already has, producing a year of synthetic recorder data at the same granularity a real HA instance would retain. + +## Architecture + +### Data Source Abstraction + +The recorder playback layer (`RecorderDataSource`) consumes data through a provider interface (`HistoryProvider` protocol). The provider abstracts the source from the playback. Two peer provider implementations exist: + +- **HA provider** -- reads from a live Home Assistant instance via its statistics API +- **SQLite provider** (`SqliteHistoryProvider`) -- reads from a local SQLite file + +Both implement the same interface. Which one is used is a configuration choice, not a quality or priority distinction. The playback layer receives a provider and replays data identically regardless of source. + +``` + +---------------------+ + | RecorderDataSource | <- single playback abstraction + | (get_power, merge, | + | interpolation) | + +---------+-----------+ + | + +---------+-----------+ + | HistoryProvider | <- provider interface + +-----+--------+------+ + | | + +------+--+ +--+----------+ + | HA | | SQLite | + | Provider | | Provider | + +---------+ +-------------+ +``` + +No changes to `RecorderDataSource`, `RealisticBehaviorEngine.get_power()`, or `engine.py` playback logic. + +### Startup Source Selection + +``` +1. Load panel config YAML +2. Determine configured data source: + - If recorder_entity mappings point to HA -> use HA provider + - If history_db configured/discovered -> use SQLite provider + - If neither -> no recorder data, synthetic per-tick generation (existing fallback) +``` + +No precedence between sources. The config determines which provider is used. + +## New Components + +### 1. SqliteHistoryProvider + +Implements the existing `HistoryProvider` protocol. Reads from a companion SQLite file using HA's recorder schema. Returns data in the same format as the live HA provider. + +- Reads `statistics` table for hourly data +- Reads `statistics_short_term` table for 5-minute data +- `RecorderDataSource` merges the two tiers using its existing logic + +### 2. SyntheticHistoryGenerator + +Standalone module that takes a panel config YAML, runs the projection, and writes the companion SQLite. + +**Inputs:** +- Completed panel config YAML (circuits, templates, BESS config, EVSE profiles) +- Panel latitude/longitude (for solar model and weather data) +- Generation anchor: "now" (or configurable timestamp for testing) + +**Time windows generated:** +- `[anchor - 1 year, anchor - 10 days]`: hourly rows written to `statistics` table +- `[anchor - 10 days, anchor]`: 5-minute rows written to `statistics_short_term` table + +This mirrors HA's actual retention model: hourly data is permanent, 5-minute data exists only for the most recent 10 days. + +**Per-circuit generation strategy:** + +| Circuit Type | Generation Approach | +|---|---| +| Consumer (loads) | `typical_power` x time-of-day profile x monthly/seasonal factors x noise | +| Producer (solar/PV) | Solar production model x weather degradation (Open-Meteo) x panel capacity | +| BESS | Charge/discharge/idle schedule from `battery_behavior` config, respecting `max_charge_power`, `max_discharge_power`, SOE constraints | +| EVSE | `time_of_day_profile.hour_factors` x rated power, with session randomization | +| HVAC | Temperature-aware seasonal model (existing `hvac_type` logic) x duty cycle | +| Cycling loads | `cycling_pattern` (duty cycle or on/off durations) applied per period | + +**Per statistics row, fields populated:** +- `start_ts`: period start epoch +- `mean`: computed power for that period +- `min`: `mean x (1 - noise_factor)` +- `max`: `mean x (1 + noise_factor)` +- `created_ts`: `start_ts` (synthetic but plausible) + +Reuses the existing modulation infrastructure from `RealisticBehaviorEngine`: solar curves (`solar.py`), weather degradation (`weather.py`), seasonal/monthly factors, time-of-day profiles, cycling patterns, and HVAC modeling. + +### 3. Provider Selection Logic + +In `PanelInstance`/`app.py` startup, the configured data source determines which provider is instantiated and passed to `RecorderDataSource`. + +## SQLite Schema + +File convention: `configs/_history.db` alongside the panel YAML. + +Discovery: `SqliteHistoryProvider` receives the DB path at construction. `PanelInstance` resolves it by convention (swap `.yaml` to `_history.db`) or from an explicit `history_db` field in `panel_config`. + +```sql +CREATE TABLE statistics_meta ( + id INTEGER PRIMARY KEY, + statistic_id TEXT UNIQUE NOT NULL, + source TEXT NOT NULL DEFAULT 'simulator', + unit_of_measurement TEXT, + has_mean INTEGER DEFAULT 1, + has_sum INTEGER DEFAULT 0, + name TEXT +); + +CREATE TABLE statistics ( + id INTEGER PRIMARY KEY, + metadata_id INTEGER NOT NULL REFERENCES statistics_meta(id), + created_ts REAL NOT NULL, + start_ts REAL NOT NULL, + mean REAL, + min REAL, + max REAL, + last_reset_ts REAL, + state REAL, + sum REAL, + UNIQUE(metadata_id, start_ts) +); + +CREATE TABLE statistics_short_term ( + id INTEGER PRIMARY KEY, + metadata_id INTEGER NOT NULL REFERENCES statistics_meta(id), + created_ts REAL NOT NULL, + start_ts REAL NOT NULL, + mean REAL, + min REAL, + max REAL, + last_reset_ts REAL, + state REAL, + sum REAL, + UNIQUE(metadata_id, start_ts) +); +``` + +Entity naming in `statistics_meta`: `sensor.__power` -- matches what the SPAN HA integration produces, so `recorder_entity` mappings work identically. + +## Clone Pipeline Integration + +After `translate_scraped_panel()` produces the config dict and writes the YAML: +1. Call `SyntheticHistoryGenerator.generate(config_path)` +2. Generator reads the YAML, runs the projection, writes the companion `_history.db` +3. Clone output includes both files + +## Standalone CLI + +``` +python -m span_panel_simulator.history_generator configs/my_panel.yaml +``` + +- Reads the YAML, generates (or regenerates) the companion SQLite +- Useful for hand-authored configs, regenerating after config edits, testing +- Optional flags: `--anchor-time` (default: now), `--years` (default: 1) + +## Testing Strategy + +**Unit tests:** +- `SyntheticHistoryGenerator`: given a known config, verify correct number of rows, correct time ranges, power values within expected bounds per circuit type +- `SqliteHistoryProvider`: given a pre-built SQLite, verify it returns data in the same format as `HistoryProvider` +- Round-trip: generate -> load via `SqliteHistoryProvider` -> verify `RecorderDataSource.get_power()` returns interpolated values consistent with generation inputs + +**Integration tests:** +- Clone a template panel -> verify companion `_history.db` is created alongside YAML +- Start a `PanelInstance` from that config -> verify it replays synthetic history identically to how it would replay HA data + +**CLI test:** +- Run the standalone generator against a test config, verify the SQLite output schema and row counts From b6d8bff79fa16ccaa79c8b343bec5d8256c81ee1 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 12:44:35 -0700 Subject: [PATCH 02/30] Address spec review findings for synthetic history generation Adds: configurable lookback in RecorderDataSource, record format spec for SqliteHistoryProvider, deterministic noise model, timezone handling, BESS SOE tracking constraints, async generation boundary, weather fallback, app.py/config_types.py change notes. --- ...-26-synthetic-history-generation-design.md | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/2026-03-26-synthetic-history-generation-design.md b/docs/superpowers/specs/2026-03-26-synthetic-history-generation-design.md index e5f4713..ff71e58 100644 --- a/docs/superpowers/specs/2026-03-26-synthetic-history-generation-design.md +++ b/docs/superpowers/specs/2026-03-26-synthetic-history-generation-design.md @@ -32,7 +32,9 @@ Both implement the same interface. Which one is used is a configuration choice, +---------+ +-------------+ ``` -No changes to `RecorderDataSource`, `RealisticBehaviorEngine.get_power()`, or `engine.py` playback logic. +**Changes to `RecorderDataSource`:** The `_HOURLY_LOOKBACK` constant (currently 90 days) must be configurable or increased to 365 days when loading from SQLite, since the generated history spans a full year. Without this, `RecorderDataSource.load()` would silently discard 9 months of generated data. The cleanest approach is to accept a `lookback_days` parameter at load time, defaulting to 90 for HA (matching current behavior) and 365 for SQLite. + +No changes to `RealisticBehaviorEngine.get_power()` or `engine.py` playback logic. ### Startup Source Selection @@ -54,6 +56,7 @@ Implements the existing `HistoryProvider` protocol. Reads from a companion SQLit - Reads `statistics` table for hourly data - Reads `statistics_short_term` table for 5-minute data +- Returns records with `start` as epoch seconds (float) matching SQLite's `start_ts` column directly -- `RecorderDataSource._parse_timestamp` already handles this format - `RecorderDataSource` merges the two tiers using its existing logic ### 2. SyntheticHistoryGenerator @@ -88,13 +91,24 @@ This mirrors HA's actual retention model: hourly data is permanent, 5-minute dat - `min`: `mean x (1 - noise_factor)` - `max`: `mean x (1 + noise_factor)` - `created_ts`: `start_ts` (synthetic but plausible) +- `sum`: NULL for v1 (power sensors use `mean`/`min`/`max`; energy accumulation via `sum` can be added later if dashboard kWh charts require it) Reuses the existing modulation infrastructure from `RealisticBehaviorEngine`: solar curves (`solar.py`), weather degradation (`weather.py`), seasonal/monthly factors, time-of-day profiles, cycling patterns, and HVAC modeling. +**Noise model:** Per-row noise is deterministic, seeded from a hash of `(panel_serial, circuit_id, start_ts)`. This ensures regenerating the DB produces identical data, matching the approach already used by `daily_weather_factor` in `solar.py`. + +**Timezone handling:** The generator uses the panel's configured timezone (`panel_config.time_zone`, or derived from lat/lon) to convert UTC epoch timestamps to local time when applying time-of-day profiles and BESS charge/discharge schedules. + +**BESS SOE tracking:** BESS circuits are generated in strict chronological order, carrying state-of-energy forward across all rows. Initial SOE starts at `backup_reserve_pct`. This means BESS generation cannot be parallelized per-circuit. + ### 3. Provider Selection Logic In `PanelInstance`/`app.py` startup, the configured data source determines which provider is instantiated and passed to `RecorderDataSource`. +**Changes to existing files:** +- **`app.py`**: `_load_recorder_data()` currently only creates a `RecorderDataSource` when an HA client is available. A second code path is needed: when a companion `_history.db` exists, instantiate `SqliteHistoryProvider` and pass it to `RecorderDataSource` with `lookback_days=365`. +- **`config_types.py`**: Add `history_db: NotRequired[str]` to the `PanelConfig` TypedDict for explicit path override (convention-based discovery is the default). + ## SQLite Schema File convention: `configs/_history.db` alongside the panel YAML. @@ -145,10 +159,13 @@ Entity naming in `statistics_meta`: `sensor.__power` ## Clone Pipeline Integration -After `translate_scraped_panel()` produces the config dict and writes the YAML: -1. Call `SyntheticHistoryGenerator.generate(config_path)` +After `translate_scraped_panel()` produces the config dict and `write_clone_config()` writes the YAML: +1. Call `await SyntheticHistoryGenerator.generate(config_path)` -- async because weather data fetching (`weather.py`) uses async HTTP 2. Generator reads the YAML, runs the projection, writes the companion `_history.db` 3. Clone output includes both files +4. If generation fails (e.g., network unavailable for Open-Meteo), the clone still succeeds with the YAML. The generator falls back to the deterministic weather model in `solar.py` (no-network fallback). The SQLite is still produced, just with less accurate weather variation. + +Generation is invoked from both the dashboard clone endpoint and the CLI clone command. ## Standalone CLI From 769cfca17816cb75a4e8a10782b35321d3551f33 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 12:55:15 -0700 Subject: [PATCH 03/30] plan: synthetic history generation implementation plan --- ...2026-03-26-synthetic-history-generation.md | 1690 +++++++++++++++++ 1 file changed, 1690 insertions(+) create mode 100644 docs/superpowers/plans/2026-03-26-synthetic-history-generation.md diff --git a/docs/superpowers/plans/2026-03-26-synthetic-history-generation.md b/docs/superpowers/plans/2026-03-26-synthetic-history-generation.md new file mode 100644 index 0000000..a2988af --- /dev/null +++ b/docs/superpowers/plans/2026-03-26-synthetic-history-generation.md @@ -0,0 +1,1690 @@ +# Synthetic History Generation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Enable cloned panels to replay synthetic historical data from a companion SQLite file, matching HA recorder schema, so panels work without a live HA instance. + +**Architecture:** A new `SqliteHistoryProvider` implements the existing `HistoryProvider` protocol, reading from a local SQLite file with HA-compatible schema. A `SyntheticHistoryGenerator` produces 1 year of hourly + 10 days of 5-minute data per circuit using the existing modulation infrastructure (solar, weather, HVAC, cycling, BESS). At startup, `app.py` checks for a companion `_history.db` file alongside the YAML config and uses it when no HA client is available. + +**Tech Stack:** Python 3.12, sqlite3 (stdlib), aiohttp (weather fetch), pytest, mypy strict mode + +--- + +## File Structure + +| Action | Path | Responsibility | +|--------|------|---------------| +| Create | `src/span_panel_simulator/sqlite_history.py` | `SqliteHistoryProvider` — reads SQLite files via `HistoryProvider` protocol | +| Create | `src/span_panel_simulator/history_generator.py` | `SyntheticHistoryGenerator` — builds companion SQLite from config YAML | +| Modify | `src/span_panel_simulator/config_types.py` | Add `history_db: NotRequired[str]` to `PanelConfig` | +| Modify | `src/span_panel_simulator/app.py` | Add SQLite provider path in `_load_recorder_data()` | +| Modify | `src/span_panel_simulator/dashboard/routes.py` | Invoke generator after clone-from-panel | +| Modify | `src/span_panel_simulator/__main__.py` | No changes needed (generator is invoked by dashboard/clone, not CLI entry) | +| Create | `tests/test_sqlite_history.py` | Tests for `SqliteHistoryProvider` | +| Create | `tests/test_history_generator.py` | Tests for `SyntheticHistoryGenerator` | +| Create | `tests/test_sqlite_app_integration.py` | Integration test: SQLite provider used at panel startup | + +--- + +### Task 1: SqliteHistoryProvider + +**Files:** +- Create: `src/span_panel_simulator/sqlite_history.py` +- Create: `tests/test_sqlite_history.py` +- Modify: `tests/test_history.py` (add protocol conformance test) + +- [ ] **Step 1: Write failing test — protocol conformance** + +Add to `tests/test_history.py`: + +```python +from span_panel_simulator.sqlite_history import SqliteHistoryProvider + + +class TestSqliteHistoryProvider: + def test_satisfies_protocol(self) -> None: + from span_panel_simulator.history import HistoryProvider + + provider: HistoryProvider = SqliteHistoryProvider(":memory:") + assert hasattr(provider, "async_get_statistics") +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest tests/test_history.py::TestSqliteHistoryProvider::test_satisfies_protocol -v` +Expected: FAIL — `ModuleNotFoundError: No module named 'span_panel_simulator.sqlite_history'` + +- [ ] **Step 3: Write minimal SqliteHistoryProvider skeleton** + +Create `src/span_panel_simulator/sqlite_history.py`: + +```python +"""SQLite-backed history provider — reads companion _history.db files. + +Implements the ``HistoryProvider`` protocol by querying ``statistics`` and +``statistics_short_term`` tables in the HA-compatible schema written by +``SyntheticHistoryGenerator``. +""" + +from __future__ import annotations + +import logging +import sqlite3 +from pathlib import Path + +_LOGGER = logging.getLogger(__name__) + +# SQL schema for the companion history database. +SCHEMA_SQL = """\ +CREATE TABLE IF NOT EXISTS statistics_meta ( + id INTEGER PRIMARY KEY, + statistic_id TEXT UNIQUE NOT NULL, + source TEXT NOT NULL DEFAULT 'simulator', + unit_of_measurement TEXT, + has_mean INTEGER DEFAULT 1, + has_sum INTEGER DEFAULT 0, + name TEXT +); + +CREATE TABLE IF NOT EXISTS statistics ( + id INTEGER PRIMARY KEY, + metadata_id INTEGER NOT NULL REFERENCES statistics_meta(id), + created_ts REAL NOT NULL, + start_ts REAL NOT NULL, + mean REAL, + min REAL, + max REAL, + last_reset_ts REAL, + state REAL, + sum REAL, + UNIQUE(metadata_id, start_ts) +); + +CREATE TABLE IF NOT EXISTS statistics_short_term ( + id INTEGER PRIMARY KEY, + metadata_id INTEGER NOT NULL REFERENCES statistics_meta(id), + created_ts REAL NOT NULL, + start_ts REAL NOT NULL, + mean REAL, + min REAL, + max REAL, + last_reset_ts REAL, + state REAL, + sum REAL, + UNIQUE(metadata_id, start_ts) +); +""" + +# Period name -> table name mapping +_PERIOD_TABLE: dict[str, str] = { + "hour": "statistics", + "5minute": "statistics_short_term", +} + + +class SqliteHistoryProvider: + """Read-only history provider backed by a local SQLite file. + + The database uses HA's recorder schema: ``statistics_meta`` maps + statistic IDs to integer keys, and ``statistics`` / ``statistics_short_term`` + store hourly and 5-minute aggregated rows respectively. + + Timestamps are stored as epoch seconds (``start_ts`` column) and returned + in the same format that ``RecorderDataSource._parse_timestamp`` expects. + """ + + def __init__(self, db_path: str | Path) -> None: + self._db_path = str(db_path) + + async def async_get_statistics( + self, + statistic_ids: list[str], + *, + period: str = "hour", + start_time: str | None = None, + end_time: str | None = None, + ) -> dict[str, list[dict[str, object]]]: + """Query statistics from the SQLite database. + + Returns data in the same format as the HA provider: a dict mapping + statistic IDs to lists of records with ``start``, ``mean``, ``min``, + ``max`` fields. + """ + table = _PERIOD_TABLE.get(period) + if table is None: + return {} + + if not statistic_ids: + return {} + + result: dict[str, list[dict[str, object]]] = {} + + try: + con = sqlite3.connect(self._db_path) + except sqlite3.Error: + _LOGGER.warning("Could not open history database: %s", self._db_path) + return {} + + try: + cur = con.cursor() + + # Resolve statistic_id -> metadata_id + placeholders = ",".join("?" for _ in statistic_ids) + cur.execute( + f"SELECT id, statistic_id FROM statistics_meta " # noqa: S608 + f"WHERE statistic_id IN ({placeholders})", + statistic_ids, + ) + meta_rows = cur.fetchall() + meta_map: dict[int, str] = {row[0]: row[1] for row in meta_rows} + + if not meta_map: + return {} + + for metadata_id, statistic_id in meta_map.items(): + query = ( + f"SELECT start_ts, mean, min, max FROM {table} " # noqa: S608 + f"WHERE metadata_id = ?" + ) + params: list[object] = [metadata_id] + + if start_time is not None: + # start_time is ISO 8601 string; convert to epoch for comparison + from datetime import UTC, datetime + + try: + dt = datetime.fromisoformat(start_time) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + query += " AND start_ts >= ?" + params.append(dt.timestamp()) + except ValueError: + pass + + if end_time is not None: + from datetime import UTC, datetime + + try: + dt = datetime.fromisoformat(end_time) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + query += " AND start_ts <= ?" + params.append(dt.timestamp()) + except ValueError: + pass + + query += " ORDER BY start_ts" + cur.execute(query, params) + + records: list[dict[str, object]] = [] + for row in cur.fetchall(): + records.append({ + "start": row[0], # epoch seconds (float) + "mean": row[1], + "min": row[2], + "max": row[3], + }) + + if records: + result[statistic_id] = records + finally: + con.close() + + return result +``` + +- [ ] **Step 4: Run protocol conformance test** + +Run: `python -m pytest tests/test_history.py::TestSqliteHistoryProvider -v` +Expected: PASS + +- [ ] **Step 5: Write test — reads hourly data from pre-populated DB** + +Create `tests/test_sqlite_history.py`: + +```python +"""Tests for SqliteHistoryProvider.""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from span_panel_simulator.sqlite_history import SCHEMA_SQL, SqliteHistoryProvider + + +def _create_test_db(path: Path, entity_id: str, rows: list[tuple[float, float]]) -> None: + """Create a test SQLite DB with statistics_meta and statistics rows.""" + con = sqlite3.connect(str(path)) + con.executescript(SCHEMA_SQL) + con.execute( + "INSERT INTO statistics_meta (id, statistic_id, unit_of_measurement) " + "VALUES (1, ?, 'W')", + (entity_id,), + ) + for start_ts, mean in rows: + con.execute( + "INSERT INTO statistics (metadata_id, created_ts, start_ts, mean, min, max) " + "VALUES (1, ?, ?, ?, ?, ?)", + (start_ts, start_ts, mean, mean * 0.9, mean * 1.1), + ) + con.commit() + con.close() + + +class TestSqliteHistoryProvider: + @pytest.mark.asyncio + async def test_reads_hourly_data(self, tmp_path: Path) -> None: + db_path = tmp_path / "test.db" + entity = "sensor.sim_panel_kitchen_power" + rows = [(1000.0, 500.0), (4600.0, 600.0), (8200.0, 550.0)] + _create_test_db(db_path, entity, rows) + + provider = SqliteHistoryProvider(db_path) + result = await provider.async_get_statistics([entity], period="hour") + + assert entity in result + assert len(result[entity]) == 3 + assert result[entity][0]["start"] == 1000.0 + assert result[entity][0]["mean"] == 500.0 + + @pytest.mark.asyncio + async def test_reads_short_term_data(self, tmp_path: Path) -> None: + db_path = tmp_path / "test.db" + entity = "sensor.sim_panel_kitchen_power" + con = sqlite3.connect(str(db_path)) + con.executescript(SCHEMA_SQL) + con.execute( + "INSERT INTO statistics_meta (id, statistic_id) VALUES (1, ?)", + (entity,), + ) + con.execute( + "INSERT INTO statistics_short_term " + "(metadata_id, created_ts, start_ts, mean, min, max) " + "VALUES (1, 1000.0, 1000.0, 200.0, 180.0, 220.0)", + ) + con.commit() + con.close() + + provider = SqliteHistoryProvider(db_path) + result = await provider.async_get_statistics([entity], period="5minute") + + assert entity in result + assert len(result[entity]) == 1 + assert result[entity][0]["mean"] == 200.0 + + @pytest.mark.asyncio + async def test_filters_by_start_time(self, tmp_path: Path) -> None: + db_path = tmp_path / "test.db" + entity = "sensor.test_power" + rows = [(1000.0, 100.0), (5000.0, 200.0), (9000.0, 300.0)] + _create_test_db(db_path, entity, rows) + + provider = SqliteHistoryProvider(db_path) + result = await provider.async_get_statistics( + [entity], + period="hour", + start_time="1970-01-01T01:00:00+00:00", # 3600 epoch + ) + + assert entity in result + assert len(result[entity]) == 2 # only 5000 and 9000 + + @pytest.mark.asyncio + async def test_unknown_entity_returns_empty(self, tmp_path: Path) -> None: + db_path = tmp_path / "test.db" + _create_test_db(db_path, "sensor.real", [(1000.0, 100.0)]) + + provider = SqliteHistoryProvider(db_path) + result = await provider.async_get_statistics( + ["sensor.does_not_exist"], period="hour" + ) + + assert result == {} + + @pytest.mark.asyncio + async def test_missing_db_returns_empty(self, tmp_path: Path) -> None: + provider = SqliteHistoryProvider(tmp_path / "nonexistent.db") + result = await provider.async_get_statistics(["sensor.x"], period="hour") + assert result == {} + + @pytest.mark.asyncio + async def test_unknown_period_returns_empty(self, tmp_path: Path) -> None: + db_path = tmp_path / "test.db" + _create_test_db(db_path, "sensor.x", [(1000.0, 100.0)]) + + provider = SqliteHistoryProvider(db_path) + result = await provider.async_get_statistics(["sensor.x"], period="month") + assert result == {} +``` + +- [ ] **Step 6: Run all SqliteHistoryProvider tests** + +Run: `python -m pytest tests/test_sqlite_history.py -v` +Expected: All PASS + +- [ ] **Step 7: Run mypy** + +Run: `python -m mypy src/span_panel_simulator/sqlite_history.py --strict` +Expected: PASS with no errors + +- [ ] **Step 8: Commit** + +```bash +git add src/span_panel_simulator/sqlite_history.py tests/test_sqlite_history.py tests/test_history.py +git commit -m "feat: add SqliteHistoryProvider for local history replay" +``` + +--- + +### Task 2: Add `history_db` to PanelConfig TypedDict + +**Files:** +- Modify: `src/span_panel_simulator/config_types.py:22-33` + +- [ ] **Step 1: Add the field** + +In `src/span_panel_simulator/config_types.py`, add `history_db` to `PanelConfig`: + +```python +class PanelConfig(TypedDict): + """Panel configuration.""" + + serial_number: str + total_tabs: int + main_size: int # Main breaker size in Amps + latitude: NotRequired[float] # degrees north, default 37.7 + longitude: NotRequired[float] # degrees east, default -122.4 + soc_shed_threshold: NotRequired[float] # SOC % below which SOC_THRESHOLD circuits are shed + postal_code: NotRequired[str] # ZIP / postal code, default "94103" + time_zone: NotRequired[str] # IANA timezone, default "America/Los_Angeles" + history_db: NotRequired[str] # path to companion SQLite history file (overrides convention) +``` + +- [ ] **Step 2: Run mypy on config_types** + +Run: `python -m mypy src/span_panel_simulator/config_types.py --strict` +Expected: PASS + +- [ ] **Step 3: Run existing tests to confirm no regressions** + +Run: `python -m pytest tests/ -x -q` +Expected: All PASS + +- [ ] **Step 4: Commit** + +```bash +git add src/span_panel_simulator/config_types.py +git commit -m "feat: add history_db field to PanelConfig for explicit SQLite path" +``` + +--- + +### Task 3: Wire SqliteHistoryProvider into app.py startup + +**Files:** +- Modify: `src/span_panel_simulator/app.py:366-419` +- Create: `tests/test_sqlite_app_integration.py` + +- [ ] **Step 1: Write failing integration test** + +Create `tests/test_sqlite_app_integration.py`: + +```python +"""Integration test: SqliteHistoryProvider used at panel startup.""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from span_panel_simulator.recorder import RecorderDataSource +from span_panel_simulator.sqlite_history import SCHEMA_SQL, SqliteHistoryProvider + + +class TestSqliteRecorderRoundTrip: + """Verify that SqliteHistoryProvider feeds RecorderDataSource correctly.""" + + @pytest.mark.asyncio + async def test_load_and_get_power(self, tmp_path: Path) -> None: + """Generate rows, load via SqliteHistoryProvider, query via RecorderDataSource.""" + db_path = tmp_path / "panel_history.db" + entity = "sensor.sim_panel_kitchen_power" + + # Create DB with 24 hourly rows + con = sqlite3.connect(str(db_path)) + con.executescript(SCHEMA_SQL) + con.execute( + "INSERT INTO statistics_meta (id, statistic_id, unit_of_measurement) " + "VALUES (1, ?, 'W')", + (entity,), + ) + base_ts = 1_700_000_000.0 # ~Nov 2023 + for i in range(24): + ts = base_ts + i * 3600 + mean = 500.0 + i * 10.0 + con.execute( + "INSERT INTO statistics (metadata_id, created_ts, start_ts, mean, min, max) " + "VALUES (1, ?, ?, ?, ?, ?)", + (ts, ts, mean, mean * 0.9, mean * 1.1), + ) + con.commit() + con.close() + + provider = SqliteHistoryProvider(db_path) + recorder = RecorderDataSource() + loaded = await recorder.load(provider, [entity], lookback_days=365) + + assert loaded == 1 + assert recorder.has_entity(entity) + + # Query a timestamp in the middle — should interpolate + mid_ts = base_ts + 12 * 3600 + power = recorder.get_power(entity, mid_ts) + assert power is not None + assert 600.0 < power < 640.0 # 500 + 12*10 = 620, interpolation close + + @pytest.mark.asyncio + async def test_no_db_file_returns_none(self, tmp_path: Path) -> None: + """When companion DB does not exist, provider returns empty.""" + provider = SqliteHistoryProvider(tmp_path / "missing.db") + recorder = RecorderDataSource() + loaded = await recorder.load(provider, ["sensor.x"], lookback_days=365) + assert loaded == 0 +``` + +- [ ] **Step 2: Run test to verify it passes (provider already works)** + +Run: `python -m pytest tests/test_sqlite_app_integration.py -v` +Expected: PASS (this validates the round-trip, no app.py changes needed yet) + +- [ ] **Step 3: Modify `_load_recorder_data` in app.py** + +Replace the existing `_load_recorder_data` method in `src/span_panel_simulator/app.py` (lines 366-419). The new version checks for a companion SQLite file when no HA client is available: + +```python + async def _load_recorder_data(self, config_path: Path) -> RecorderDataSource | None: + """Create and populate a RecorderDataSource from config + history source. + + Source selection: + 1. If HA client is available and config has recorder_entity mappings → HA provider + 2. If a companion ``_history.db`` file exists (or ``history_db`` is set) → SQLite provider + 3. Otherwise → None (engine uses synthetic per-tick generation) + + Failures are logged and swallowed so the panel still starts in synthetic mode. + """ + try: + raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) + except Exception: + return None + + if not isinstance(raw, dict): + return None + + templates = raw.get("circuit_templates") + if not isinstance(templates, dict): + return None + + entity_ids: list[str] = [] + for tmpl in templates.values(): + if isinstance(tmpl, dict): + entity_id = tmpl.get("recorder_entity") + if isinstance(entity_id, str) and entity_id: + entity_ids.append(entity_id) + + if not entity_ids: + return None + + # Source 1: HA client available → use HA provider + if self._ha_client is not None: + _LOGGER.info( + "Loading recorder data for %s (%d entities) from HA", + config_path.name, + len(entity_ids), + ) + recorder = RecorderDataSource() + try: + loaded = await recorder.load(self._ha_client, entity_ids) + except Exception: + _LOGGER.warning( + "Recorder data loading failed for %s — using synthetic", + config_path.name, + exc_info=True, + ) + return None + + if loaded == 0: + _LOGGER.warning( + "Recorder returned no data for %s — using synthetic", + config_path.name, + ) + return recorder if loaded > 0 else None + + # Source 2: companion SQLite file + db_path = self._resolve_history_db(config_path, raw) + if db_path is not None: + from span_panel_simulator.sqlite_history import SqliteHistoryProvider + + _LOGGER.info( + "Loading recorder data for %s (%d entities) from %s", + config_path.name, + len(entity_ids), + db_path.name, + ) + provider = SqliteHistoryProvider(db_path) + recorder = RecorderDataSource() + try: + loaded = await recorder.load(provider, entity_ids, lookback_days=365) + except Exception: + _LOGGER.warning( + "SQLite history loading failed for %s — using synthetic", + config_path.name, + exc_info=True, + ) + return None + + if loaded == 0: + _LOGGER.warning( + "SQLite history returned no data for %s — using synthetic", + config_path.name, + ) + return recorder if loaded > 0 else None + + return None + + @staticmethod + def _resolve_history_db(config_path: Path, raw: dict[str, object]) -> Path | None: + """Find the companion SQLite history DB for a config file. + + Checks explicit ``panel_config.history_db`` first, then falls back + to the convention: ``_history.db`` in the same directory. + """ + panel_config = raw.get("panel_config") + if isinstance(panel_config, dict): + explicit = panel_config.get("history_db") + if isinstance(explicit, str) and explicit: + explicit_path = Path(explicit) + if not explicit_path.is_absolute(): + explicit_path = config_path.parent / explicit_path + if explicit_path.exists(): + return explicit_path + + convention_path = config_path.with_name(config_path.stem + "_history.db") + if convention_path.exists(): + return convention_path + + return None +``` + +- [ ] **Step 4: Write test for _resolve_history_db** + +Add to `tests/test_sqlite_app_integration.py`: + +```python +from span_panel_simulator.app import SimulatorApp + + +class TestResolveHistoryDb: + def test_convention_path(self, tmp_path: Path) -> None: + config_path = tmp_path / "my_panel.yaml" + config_path.write_text("panel_config:\n serial_number: x\n") + db_path = tmp_path / "my_panel_history.db" + db_path.write_text("") # just needs to exist + + result = SimulatorApp._resolve_history_db(config_path, {}) + assert result == db_path + + def test_explicit_path(self, tmp_path: Path) -> None: + config_path = tmp_path / "my_panel.yaml" + config_path.write_text("") + db_path = tmp_path / "custom.db" + db_path.write_text("") + + raw = {"panel_config": {"history_db": "custom.db"}} + result = SimulatorApp._resolve_history_db(config_path, raw) + assert result == db_path + + def test_no_db_returns_none(self, tmp_path: Path) -> None: + config_path = tmp_path / "my_panel.yaml" + config_path.write_text("") + + result = SimulatorApp._resolve_history_db(config_path, {}) + assert result is None + + def test_explicit_overrides_convention(self, tmp_path: Path) -> None: + config_path = tmp_path / "my_panel.yaml" + config_path.write_text("") + # Both exist + (tmp_path / "my_panel_history.db").write_text("") + custom = tmp_path / "custom.db" + custom.write_text("") + + raw = {"panel_config": {"history_db": "custom.db"}} + result = SimulatorApp._resolve_history_db(config_path, raw) + assert result == custom +``` + +- [ ] **Step 5: Run all integration tests** + +Run: `python -m pytest tests/test_sqlite_app_integration.py -v` +Expected: All PASS + +- [ ] **Step 6: Run full test suite** + +Run: `python -m pytest tests/ -x -q` +Expected: All PASS + +- [ ] **Step 7: Run mypy on app.py** + +Run: `python -m mypy src/span_panel_simulator/app.py --strict` +Expected: PASS (or only pre-existing issues) + +- [ ] **Step 8: Commit** + +```bash +git add src/span_panel_simulator/app.py tests/test_sqlite_app_integration.py +git commit -m "feat: wire SqliteHistoryProvider into panel startup" +``` + +--- + +### Task 4: SyntheticHistoryGenerator — core generation logic + +**Files:** +- Create: `src/span_panel_simulator/history_generator.py` +- Create: `tests/test_history_generator.py` + +This is the largest task. The generator reads a panel config YAML, computes per-circuit power for every time step, and writes the companion SQLite DB. + +- [ ] **Step 1: Write failing test — generator produces correct row counts** + +Create `tests/test_history_generator.py`: + +```python +"""Tests for SyntheticHistoryGenerator.""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest +import yaml + +from span_panel_simulator.history_generator import SyntheticHistoryGenerator + +# Minimal config with one consumer circuit +_MINIMAL_CONFIG: dict[str, object] = { + "panel_config": { + "serial_number": "sim-test-gen", + "total_tabs": 16, + "main_size": 200, + "latitude": 37.7, + "longitude": -122.4, + }, + "circuit_templates": { + "kitchen": { + "energy_profile": { + "mode": "consumer", + "power_range": [0, 2400], + "typical_power": 800.0, + "power_variation": 0.1, + }, + "relay_behavior": "controllable", + "priority": "MUST_HAVE", + "recorder_entity": "sensor.sim_test_gen_kitchen_power", + }, + }, + "circuits": [ + {"id": "circuit_1", "name": "Kitchen", "template": "kitchen", "tabs": [1]}, + ], + "unmapped_tabs": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + "simulation_params": { + "update_interval": 5, + "time_acceleration": 1.0, + "noise_factor": 0.02, + "enable_realistic_behaviors": True, + }, +} + + +class TestSyntheticHistoryGenerator: + @pytest.mark.asyncio + async def test_generates_correct_tables(self, tmp_path: Path) -> None: + config_path = tmp_path / "test_panel.yaml" + config_path.write_text(yaml.dump(_MINIMAL_CONFIG)) + + gen = SyntheticHistoryGenerator() + db_path = await gen.generate(config_path) + + assert db_path.exists() + assert db_path.name == "test_panel_history.db" + + con = sqlite3.connect(str(db_path)) + # Check statistics_meta has the entity + meta = con.execute( + "SELECT statistic_id FROM statistics_meta" + ).fetchall() + assert len(meta) == 1 + assert meta[0][0] == "sensor.sim_test_gen_kitchen_power" + + # Check hourly rows exist (roughly 365 days * 24 hours - 10 days * 24) + hourly_count = con.execute( + "SELECT COUNT(*) FROM statistics" + ).fetchone()[0] + # ~355 days * 24 = 8520, allow some tolerance + assert hourly_count > 8000 + assert hourly_count < 9000 + + # Check short-term rows exist (10 days * 24 hours * 12 per hour) + short_count = con.execute( + "SELECT COUNT(*) FROM statistics_short_term" + ).fetchone()[0] + # 10 days * 288 five-minute slots = 2880 + assert short_count > 2800 + assert short_count < 3000 + + con.close() + + @pytest.mark.asyncio + async def test_deterministic_output(self, tmp_path: Path) -> None: + """Same config + anchor produces identical DBs.""" + config_path = tmp_path / "test_panel.yaml" + config_path.write_text(yaml.dump(_MINIMAL_CONFIG)) + + anchor = 1_700_000_000.0 # fixed anchor + + gen = SyntheticHistoryGenerator() + db1 = await gen.generate(config_path, anchor_time=anchor) + + # Rename to avoid overwrite + db1_copy = tmp_path / "db1.db" + db1.rename(db1_copy) + + db2 = await gen.generate(config_path, anchor_time=anchor) + + con1 = sqlite3.connect(str(db1_copy)) + con2 = sqlite3.connect(str(db2)) + + rows1 = con1.execute( + "SELECT start_ts, mean FROM statistics ORDER BY start_ts" + ).fetchall() + rows2 = con2.execute( + "SELECT start_ts, mean FROM statistics ORDER BY start_ts" + ).fetchall() + + assert rows1 == rows2 + con1.close() + con2.close() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest tests/test_history_generator.py::TestSyntheticHistoryGenerator::test_generates_correct_tables -v` +Expected: FAIL — `ModuleNotFoundError` + +- [ ] **Step 3: Implement SyntheticHistoryGenerator** + +Create `src/span_panel_simulator/history_generator.py`: + +```python +"""Synthetic history generator — builds companion SQLite databases. + +Given a panel config YAML, generates a year of synthetic power statistics +matching HA's recorder schema. The output SQLite file can be read by +``SqliteHistoryProvider`` and fed to ``RecorderDataSource`` for replay. + +Time windows: + - ``[anchor - 1 year, anchor - 10 days]``: hourly rows in ``statistics`` + - ``[anchor - 10 days, anchor]``: 5-minute rows in ``statistics_short_term`` + +Uses the same modulation infrastructure as the live simulation engine: +solar curves, weather degradation, HVAC seasonal model, time-of-day +profiles, cycling patterns, and monthly factors. +""" + +from __future__ import annotations + +import hashlib +import logging +import math +import sqlite3 +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING +from zoneinfo import ZoneInfo + +import yaml + +from span_panel_simulator.hvac import hvac_seasonal_factor +from span_panel_simulator.solar import daily_weather_factor, solar_production_factor +from span_panel_simulator.sqlite_history import SCHEMA_SQL +from span_panel_simulator.weather import fetch_historical_weather, get_cached_weather + +if TYPE_CHECKING: + from span_panel_simulator.config_types import ( + CircuitTemplateExtended, + SimulationConfig, + ) + +_LOGGER = logging.getLogger(__name__) + +_SECONDS_PER_HOUR = 3600 +_SECONDS_PER_5MIN = 300 +_DAYS_SHORT_TERM = 10 +_DAYS_TOTAL = 365 + + +def _deterministic_noise(panel_serial: str, circuit_id: str, start_ts: float) -> float: + """Deterministic per-row noise in [-1, 1], seeded from identity + timestamp.""" + raw = f"{panel_serial}:{circuit_id}:{start_ts}".encode() + h = int(hashlib.sha256(raw).hexdigest()[:8], 16) + return (h % 20000 - 10000) / 10000.0 + + +def _resolve_timezone(config: dict[str, object]) -> ZoneInfo: + """Resolve panel timezone from config, matching engine logic.""" + panel = config.get("panel_config", {}) + if not isinstance(panel, dict): + return ZoneInfo("America/Los_Angeles") + + explicit = panel.get("time_zone") + if isinstance(explicit, str) and explicit: + try: + return ZoneInfo(explicit) + except (KeyError, ValueError): + pass + + lat = panel.get("latitude") + lon = panel.get("longitude") + if lat is not None and lon is not None: + from timezonefinder import TimezoneFinder + + tz_name = TimezoneFinder().timezone_at(lat=float(lat), lng=float(lon)) + if tz_name is not None: + return ZoneInfo(tz_name) + + return ZoneInfo("America/Los_Angeles") + + +class SyntheticHistoryGenerator: + """Generate companion SQLite history databases from panel config YAMLs.""" + + async def generate( + self, + config_path: Path, + *, + anchor_time: float | None = None, + ) -> Path: + """Generate the companion history DB for a config file. + + Args: + config_path: Path to the panel YAML config. + anchor_time: Unix epoch for the "now" end of the window. + Defaults to current time. + + Returns: + Path to the generated ``_history.db`` file. + """ + raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + msg = f"Invalid config: {config_path}" + raise ValueError(msg) + + anchor = anchor_time if anchor_time is not None else time.time() + db_path = config_path.with_name(config_path.stem + "_history.db") + + panel_config = raw.get("panel_config", {}) + if not isinstance(panel_config, dict): + msg = "Missing panel_config" + raise ValueError(msg) + + serial = str(panel_config.get("serial_number", "unknown")) + lat = float(panel_config.get("latitude", 37.7)) + lon = float(panel_config.get("longitude", -122.4)) + tz = _resolve_timezone(raw) + noise_factor = float( + raw.get("simulation_params", {}).get("noise_factor", 0.02) + if isinstance(raw.get("simulation_params"), dict) + else 0.02 + ) + + # Fetch weather data for solar degradation (best-effort) + weather_monthly: dict[int, float] | None = None + cached = get_cached_weather(lat, lon) + if cached is not None: + weather_monthly = cached.monthly_factors + else: + try: + wd = await fetch_historical_weather(lat, lon) + weather_monthly = wd.monthly_factors + except Exception: + _LOGGER.debug("Weather fetch failed; using deterministic model", exc_info=True) + + # Collect circuits with recorder_entity mappings + templates = raw.get("circuit_templates", {}) + if not isinstance(templates, dict): + templates = {} + + circuits_to_generate: list[tuple[str, str, dict[str, object]]] = [] + for tmpl_name, tmpl in templates.items(): + if not isinstance(tmpl, dict): + continue + entity = tmpl.get("recorder_entity") + if isinstance(entity, str) and entity: + circuits_to_generate.append((tmpl_name, entity, tmpl)) + + if not circuits_to_generate: + _LOGGER.warning("No recorder_entity mappings in %s — nothing to generate", config_path.name) + # Still create DB with schema for consistency + con = sqlite3.connect(str(db_path)) + con.executescript(SCHEMA_SQL) + con.close() + return db_path + + # Compute time boundaries + hourly_start = anchor - _DAYS_TOTAL * 86400 + short_term_start = anchor - _DAYS_SHORT_TERM * 86400 + hourly_end = short_term_start # hourly stops where short-term begins + + # Generate + _LOGGER.info( + "Generating synthetic history for %s: %d circuits, anchor=%s", + config_path.name, + len(circuits_to_generate), + datetime.fromtimestamp(anchor, tz=UTC).isoformat(), + ) + + con = sqlite3.connect(str(db_path)) + con.executescript(SCHEMA_SQL) + + # Clear any existing data (regeneration case) + con.execute("DELETE FROM statistics") + con.execute("DELETE FROM statistics_short_term") + con.execute("DELETE FROM statistics_meta") + + try: + for idx, (tmpl_name, entity_id, tmpl) in enumerate(circuits_to_generate, start=1): + con.execute( + "INSERT INTO statistics_meta (id, statistic_id, source, unit_of_measurement, name) " + "VALUES (?, ?, 'simulator', 'W', ?)", + (idx, entity_id, tmpl_name), + ) + + # Generate hourly rows + self._generate_rows( + con=con, + table="statistics", + metadata_id=idx, + entity_id=entity_id, + template=tmpl, + start_ts=hourly_start, + end_ts=hourly_end, + step_seconds=_SECONDS_PER_HOUR, + serial=serial, + lat=lat, + lon=lon, + tz=tz, + noise_factor=noise_factor, + weather_monthly=weather_monthly, + ) + + # Generate 5-minute rows + self._generate_rows( + con=con, + table="statistics_short_term", + metadata_id=idx, + entity_id=entity_id, + template=tmpl, + start_ts=short_term_start, + end_ts=anchor, + step_seconds=_SECONDS_PER_5MIN, + serial=serial, + lat=lat, + lon=lon, + tz=tz, + noise_factor=noise_factor, + weather_monthly=weather_monthly, + ) + + con.commit() + finally: + con.close() + + _LOGGER.info("Wrote synthetic history to %s", db_path.name) + return db_path + + def _generate_rows( + self, + *, + con: sqlite3.Connection, + table: str, + metadata_id: int, + entity_id: str, + template: dict[str, object], + start_ts: float, + end_ts: float, + step_seconds: int, + serial: str, + lat: float, + lon: float, + tz: ZoneInfo, + noise_factor: float, + weather_monthly: dict[int, float] | None, + ) -> None: + """Generate statistics rows for one circuit into the given table.""" + ep = template.get("energy_profile", {}) + if not isinstance(ep, dict): + return + + mode = str(ep.get("mode", "consumer")) + typical_power = float(ep.get("typical_power", 0.0)) + nameplate_w = ep.get("nameplate_capacity_w") + nameplate = float(nameplate_w) if nameplate_w is not None else None + + # Time-of-day profile + tod_profile = template.get("time_of_day_profile", {}) + tod_enabled = isinstance(tod_profile, dict) and tod_profile.get("enabled", False) + hour_factors: dict[int, float] = {} + if isinstance(tod_profile, dict): + raw_hf = tod_profile.get("hour_factors", {}) + if isinstance(raw_hf, dict): + hour_factors = {int(k): float(v) for k, v in raw_hf.items()} + + # Monthly factors + monthly_factors: dict[int, float] = {} + raw_mf = template.get("monthly_factors") + if isinstance(raw_mf, dict): + monthly_factors = {int(k): float(v) for k, v in raw_mf.items()} + + # HVAC type + hvac_type = template.get("hvac_type") + hvac_type_str = str(hvac_type) if isinstance(hvac_type, str) else None + + # Cycling pattern + cycling = template.get("cycling_pattern") + duty_cycle: float | None = None + cycle_period = 2700 + if isinstance(cycling, dict): + dc = cycling.get("duty_cycle") + if dc is not None: + duty_cycle = float(dc) + else: + on_dur = cycling.get("on_duration") + off_dur = cycling.get("off_duration") + if on_dur is not None and off_dur is not None: + total = int(on_dur) + int(off_dur) + if total > 0: + duty_cycle = int(on_dur) / total + cycle_period = total + cp = cycling.get("period") + if cp is not None: + cycle_period = int(cp) + + # Active days from time_of_day_profile + active_days: list[int] = [] + if isinstance(tod_profile, dict): + ad = tod_profile.get("active_days", []) + if isinstance(ad, list): + active_days = [int(d) for d in ad] + + # Power range for clamping + power_range = ep.get("power_range", [0, 10000]) + if isinstance(power_range, list) and len(power_range) == 2: + min_power, max_power = float(power_range[0]), float(power_range[1]) + else: + min_power, max_power = 0.0, 10000.0 + + # Mean of hour factors for normalisation + mean_hf = ( + sum(hour_factors.values()) / len(hour_factors) + if hour_factors + else 1.0 + ) + + # Mean of monthly factors for normalisation + mean_mf = ( + sum(monthly_factors.values()) / len(monthly_factors) + if monthly_factors + else 1.0 + ) + + batch: list[tuple[object, ...]] = [] + ts = start_ts + while ts < end_ts: + power = self._compute_power_at( + ts=ts, + mode=mode, + typical_power=typical_power, + nameplate=nameplate, + lat=lat, + lon=lon, + tz=tz, + serial=serial, + hour_factors=hour_factors, + mean_hf=mean_hf, + tod_enabled=tod_enabled, + monthly_factors=monthly_factors, + mean_mf=mean_mf, + hvac_type=hvac_type_str, + duty_cycle=duty_cycle, + cycle_period=cycle_period, + active_days=active_days, + weather_monthly=weather_monthly, + ) + + # Apply deterministic noise + noise = _deterministic_noise(serial, entity_id, ts) + noisy_power = power * (1.0 + noise * noise_factor) + + # Clamp + if mode == "producer": + noisy_power = max(0.0, min(abs(min_power), noisy_power)) + else: + noisy_power = max(min_power, min(max_power, noisy_power)) + + mean_val = noisy_power + min_val = mean_val * (1.0 - noise_factor) + max_val = mean_val * (1.0 + noise_factor) + + batch.append((metadata_id, ts, ts, mean_val, min_val, max_val)) + + if len(batch) >= 1000: + con.executemany( + f"INSERT INTO {table} " # noqa: S608 + "(metadata_id, created_ts, start_ts, mean, min, max) " + "VALUES (?, ?, ?, ?, ?, ?)", + batch, + ) + batch.clear() + + ts += step_seconds + + if batch: + con.executemany( + f"INSERT INTO {table} " # noqa: S608 + "(metadata_id, created_ts, start_ts, mean, min, max) " + "VALUES (?, ?, ?, ?, ?, ?)", + batch, + ) + + def _compute_power_at( + self, + *, + ts: float, + mode: str, + typical_power: float, + nameplate: float | None, + lat: float, + lon: float, + tz: ZoneInfo, + serial: str, + hour_factors: dict[int, float], + mean_hf: float, + tod_enabled: bool, + monthly_factors: dict[int, float], + mean_mf: float, + hvac_type: str | None, + duty_cycle: float | None, + cycle_period: int, + active_days: list[int], + weather_monthly: dict[int, float] | None, + ) -> float: + """Compute synthetic power for one time step.""" + dt = datetime.fromtimestamp(ts, tz=tz) + hour = dt.hour + weekday = dt.weekday() + month = dt.month + + # Check active days + if active_days and weekday not in active_days: + return 0.0 + + base = typical_power + + # Mode-specific modulation + if mode == "producer": + # Solar: use nameplate or typical_power as scale + scale = abs(nameplate) if nameplate is not None and nameplate > 0 else abs(base) + solar = solar_production_factor(ts, lat, lon) + weather = daily_weather_factor( + ts, seed=hash(serial), monthly_factors=weather_monthly + ) + return scale * solar * weather + + # Time-of-day for consumers + if hour_factors and tod_enabled: + factor = hour_factors.get(hour, 0.0) + if mean_hf > 0: + base = typical_power / mean_hf * factor + else: + base = 0.0 + elif tod_enabled: + # Basic peak/off-peak + if hour >= 22 or hour <= 6: + base = typical_power * 0.3 + elif hour in range(7, 22): + base = typical_power + + # Monthly/seasonal modulation + if monthly_factors: + mf = monthly_factors.get(month, 1.0) + if mean_mf > 0: + base = base / mean_mf * mf + elif hvac_type is not None: + base = base * hvac_seasonal_factor(ts, lat, hvac_type, tz=tz) + + # Cycling: reduce by duty cycle + if duty_cycle is not None and duty_cycle < 1.0: + # For hourly/5min aggregation, duty cycle reduces mean power + base = base * duty_cycle + + return base +``` + +- [ ] **Step 4: Run tests** + +Run: `python -m pytest tests/test_history_generator.py -v` +Expected: All PASS + +- [ ] **Step 5: Write test — solar circuit produces day/night pattern** + +Add to `tests/test_history_generator.py`: + +```python + @pytest.mark.asyncio + async def test_solar_circuit_has_day_night_pattern(self, tmp_path: Path) -> None: + """Solar circuits should produce zero power at night, nonzero during day.""" + solar_config = { + **_MINIMAL_CONFIG, + "circuit_templates": { + "solar": { + "energy_profile": { + "mode": "producer", + "power_range": [-5000, 0], + "typical_power": -3000.0, + "power_variation": 0.05, + "nameplate_capacity_w": 5000.0, + }, + "relay_behavior": "non_controllable", + "priority": "NEVER", + "recorder_entity": "sensor.sim_test_gen_solar_power", + }, + }, + "circuits": [ + {"id": "circuit_1", "name": "Solar", "template": "solar", "tabs": [1]}, + ], + } + + config_path = tmp_path / "solar_panel.yaml" + config_path.write_text(yaml.dump(solar_config)) + + gen = SyntheticHistoryGenerator() + # Use a fixed anchor in summer for reliable daylight + db_path = await gen.generate(config_path, anchor_time=1_719_792_000.0) # ~Jul 2024 + + con = sqlite3.connect(str(db_path)) + rows = con.execute( + "SELECT start_ts, mean FROM statistics ORDER BY start_ts LIMIT 48" + ).fetchall() + con.close() + + # Among the first 48 hourly rows, some should be zero (night) + # and some should be nonzero (day) + values = [r[1] for r in rows] + assert any(v == 0.0 for v in values), "Expected some zero (nighttime) rows" + assert any(v > 0.0 for v in values), "Expected some nonzero (daytime) rows" +``` + +- [ ] **Step 6: Run all generator tests** + +Run: `python -m pytest tests/test_history_generator.py -v` +Expected: All PASS + +- [ ] **Step 7: Run mypy** + +Run: `python -m mypy src/span_panel_simulator/history_generator.py --strict` +Expected: PASS (or only pre-existing issues from imported modules) + +- [ ] **Step 8: Commit** + +```bash +git add src/span_panel_simulator/history_generator.py tests/test_history_generator.py +git commit -m "feat: add SyntheticHistoryGenerator for offline history creation" +``` + +--- + +### Task 5: Standalone CLI entry point for generator + +**Files:** +- Modify: `src/span_panel_simulator/history_generator.py` (add `__main__` block at bottom) + +- [ ] **Step 1: Add CLI entry point to history_generator.py** + +Append to the bottom of `src/span_panel_simulator/history_generator.py`: + +```python +async def _cli_main() -> None: + """CLI entry point for standalone generation.""" + import argparse + + parser = argparse.ArgumentParser( + description="Generate synthetic history DB from a panel config YAML", + ) + parser.add_argument("config", type=Path, help="Path to the panel YAML config") + parser.add_argument( + "--anchor-time", + type=float, + default=None, + help="Unix epoch for the anchor (default: now)", + ) + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", + ) + + gen = SyntheticHistoryGenerator() + db_path = await gen.generate(args.config, anchor_time=args.anchor_time) + print(f"Generated: {db_path}") + + +if __name__ == "__main__": + import asyncio + + asyncio.run(_cli_main()) +``` + +- [ ] **Step 2: Test the CLI manually** + +Run: `python -m span_panel_simulator.history_generator configs/default_MAIN_16.yaml --anchor-time 1700000000` +Expected: Prints `Generated: configs/default_MAIN_16_history.db` (or warns about no recorder_entity mappings if the default config lacks them) + +- [ ] **Step 3: Commit** + +```bash +git add src/span_panel_simulator/history_generator.py +git commit -m "feat: add standalone CLI for synthetic history generation" +``` + +--- + +### Task 6: Integrate generator into clone-from-panel flow + +**Files:** +- Modify: `src/span_panel_simulator/dashboard/routes.py` (in `handle_clone_from_panel`) + +- [ ] **Step 1: Write test — clone flow generates companion DB** + +Add to `tests/test_history_generator.py`: + +```python +class TestCloneIntegration: + @pytest.mark.asyncio + async def test_generate_after_clone_config(self, tmp_path: Path) -> None: + """After writing a clone config, the generator should produce a companion DB.""" + # Write a clone-like config with recorder_entity mappings + config = { + "panel_config": { + "serial_number": "sim-ABC123-clone", + "total_tabs": 32, + "main_size": 200, + "latitude": 37.7, + "longitude": -122.4, + }, + "circuit_templates": { + "clone_1": { + "energy_profile": { + "mode": "consumer", + "power_range": [0, 2400], + "typical_power": 500.0, + "power_variation": 0.1, + }, + "relay_behavior": "controllable", + "priority": "MUST_HAVE", + "recorder_entity": "sensor.span_panel_kitchen_power", + }, + "clone_3": { + "energy_profile": { + "mode": "producer", + "power_range": [-5000, 0], + "typical_power": -3000.0, + "power_variation": 0.05, + "nameplate_capacity_w": 5000.0, + }, + "relay_behavior": "non_controllable", + "priority": "NEVER", + "recorder_entity": "sensor.span_panel_solar_power", + }, + }, + "circuits": [ + {"id": "circuit_1", "name": "Kitchen", "template": "clone_1", "tabs": [1]}, + {"id": "circuit_3", "name": "Solar", "template": "clone_3", "tabs": [3]}, + ], + "unmapped_tabs": list(range(4, 33)), + "simulation_params": { + "update_interval": 5, + "time_acceleration": 1.0, + "noise_factor": 0.02, + "enable_realistic_behaviors": True, + }, + } + + config_path = tmp_path / "ABC123-clone.yaml" + config_path.write_text(yaml.dump(config)) + + gen = SyntheticHistoryGenerator() + db_path = await gen.generate(config_path, anchor_time=1_700_000_000.0) + + assert db_path.exists() + + # Verify both entities are in the DB + con = sqlite3.connect(str(db_path)) + meta = con.execute("SELECT statistic_id FROM statistics_meta ORDER BY id").fetchall() + assert len(meta) == 2 + assert meta[0][0] == "sensor.span_panel_kitchen_power" + assert meta[1][0] == "sensor.span_panel_solar_power" + + # Both should have hourly data + for entity_idx in (1, 2): + count = con.execute( + "SELECT COUNT(*) FROM statistics WHERE metadata_id = ?", + (entity_idx,), + ).fetchone()[0] + assert count > 8000 + con.close() +``` + +- [ ] **Step 2: Run test** + +Run: `python -m pytest tests/test_history_generator.py::TestCloneIntegration -v` +Expected: PASS + +- [ ] **Step 3: Add generator call to handle_clone_from_panel in routes.py** + +In `src/span_panel_simulator/dashboard/routes.py`, find the `handle_clone_from_panel` function. After the line `clone_path = write_clone_config(config, ctx.config_dir, scraped.serial_number)` (around line 1558), add the generator invocation: + +```python + clone_path = write_clone_config(config, ctx.config_dir, scraped.serial_number) + + # Generate synthetic history companion DB for offline replay + try: + from span_panel_simulator.history_generator import SyntheticHistoryGenerator + + gen = SyntheticHistoryGenerator() + history_db = await gen.generate(clone_path) + _LOGGER.info("Generated synthetic history: %s", history_db.name) + except Exception: + _LOGGER.warning("Synthetic history generation failed — panel will use per-tick synthesis", exc_info=True) +``` + +- [ ] **Step 4: Run existing clone tests to verify no regressions** + +Run: `python -m pytest tests/test_clone.py -v` +Expected: All PASS + +- [ ] **Step 5: Run full test suite** + +Run: `python -m pytest tests/ -x -q` +Expected: All PASS + +- [ ] **Step 6: Commit** + +```bash +git add src/span_panel_simulator/dashboard/routes.py tests/test_history_generator.py +git commit -m "feat: generate synthetic history DB on clone-from-panel" +``` + +--- + +### Task 7: End-to-end round-trip test + +**Files:** +- Modify: `tests/test_sqlite_app_integration.py` + +- [ ] **Step 1: Write end-to-end test — generate then load via provider** + +Add to `tests/test_sqlite_app_integration.py`: + +```python +from span_panel_simulator.history_generator import SyntheticHistoryGenerator + +# Same minimal config as test_history_generator +_ROUNDTRIP_CONFIG: dict[str, object] = { + "panel_config": { + "serial_number": "sim-roundtrip", + "total_tabs": 16, + "main_size": 200, + "latitude": 37.7, + "longitude": -122.4, + }, + "circuit_templates": { + "kitchen": { + "energy_profile": { + "mode": "consumer", + "power_range": [0, 2400], + "typical_power": 800.0, + "power_variation": 0.1, + }, + "relay_behavior": "controllable", + "priority": "MUST_HAVE", + "recorder_entity": "sensor.sim_roundtrip_kitchen_power", + }, + }, + "circuits": [ + {"id": "circuit_1", "name": "Kitchen", "template": "kitchen", "tabs": [1]}, + ], + "unmapped_tabs": list(range(2, 17)), + "simulation_params": { + "update_interval": 5, + "time_acceleration": 1.0, + "noise_factor": 0.02, + "enable_realistic_behaviors": True, + }, +} + + +class TestEndToEndRoundTrip: + @pytest.mark.asyncio + async def test_generate_then_load_then_query(self, tmp_path: Path) -> None: + """Full pipeline: generate DB -> load via SqliteHistoryProvider -> query via RecorderDataSource.""" + import yaml + + config_path = tmp_path / "roundtrip.yaml" + config_path.write_text(yaml.dump(_ROUNDTRIP_CONFIG)) + + anchor = 1_700_000_000.0 + entity = "sensor.sim_roundtrip_kitchen_power" + + # Step 1: Generate + gen = SyntheticHistoryGenerator() + db_path = await gen.generate(config_path, anchor_time=anchor) + assert db_path.exists() + + # Step 2: Load + provider = SqliteHistoryProvider(db_path) + recorder = RecorderDataSource() + loaded = await recorder.load(provider, [entity], lookback_days=365) + assert loaded == 1 + + # Step 3: Query + bounds = recorder.time_bounds() + assert bounds is not None + start, end = bounds + + # Coverage should be close to 365 days + coverage_days = (end - start) / 86400 + assert coverage_days > 360 + + # Query multiple points — all should return non-None + import random + rng = random.Random(42) + for _ in range(100): + ts = rng.uniform(start, end) + power = recorder.get_power(entity, ts) + assert power is not None + assert power >= 0.0 # consumer circuit, always >= 0 + + @pytest.mark.asyncio + async def test_convention_discovery_works(self, tmp_path: Path) -> None: + """Verify that _resolve_history_db finds the generated companion file.""" + import yaml + + config_path = tmp_path / "discovery.yaml" + config_path.write_text(yaml.dump(_ROUNDTRIP_CONFIG)) + + gen = SyntheticHistoryGenerator() + db_path = await gen.generate(config_path, anchor_time=1_700_000_000.0) + + # The generated file should match the convention + assert db_path.name == "discovery_history.db" + + # SimulatorApp._resolve_history_db should find it + result = SimulatorApp._resolve_history_db(config_path, _ROUNDTRIP_CONFIG) + assert result == db_path +``` + +- [ ] **Step 2: Run end-to-end tests** + +Run: `python -m pytest tests/test_sqlite_app_integration.py -v` +Expected: All PASS + +- [ ] **Step 3: Run full test suite + mypy** + +Run: `python -m pytest tests/ -x -q && python -m mypy src/span_panel_simulator/sqlite_history.py src/span_panel_simulator/history_generator.py src/span_panel_simulator/app.py --strict` +Expected: All PASS + +- [ ] **Step 4: Commit** + +```bash +git add tests/test_sqlite_app_integration.py +git commit -m "test: add end-to-end round-trip test for synthetic history pipeline" +``` + +--- + +### Task 8: Run ruff and final cleanup + +- [ ] **Step 1: Run ruff on all new/modified files** + +Run: `python -m ruff check src/span_panel_simulator/sqlite_history.py src/span_panel_simulator/history_generator.py src/span_panel_simulator/app.py src/span_panel_simulator/config_types.py src/span_panel_simulator/dashboard/routes.py` +Expected: No errors (fix any that appear) + +- [ ] **Step 2: Run full test suite one final time** + +Run: `python -m pytest tests/ -v` +Expected: All PASS + +- [ ] **Step 3: Commit any cleanup** + +```bash +git add -u +git commit -m "chore: lint and cleanup for synthetic history feature" +``` From 937174d3507ecf1efe5ef880561a6c366294bf01 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 12:59:01 -0700 Subject: [PATCH 04/30] feat: add SqliteHistoryProvider for local history replay --- src/span_panel_simulator/sqlite_history.py | 174 +++++++++++++++++++++ tests/test_history.py | 7 + tests/test_sqlite_history.py | 115 ++++++++++++++ 3 files changed, 296 insertions(+) create mode 100644 src/span_panel_simulator/sqlite_history.py create mode 100644 tests/test_sqlite_history.py diff --git a/src/span_panel_simulator/sqlite_history.py b/src/span_panel_simulator/sqlite_history.py new file mode 100644 index 0000000..ffaaa36 --- /dev/null +++ b/src/span_panel_simulator/sqlite_history.py @@ -0,0 +1,174 @@ +"""SQLite-backed history provider — reads companion _history.db files. + +Implements the ``HistoryProvider`` protocol by querying ``statistics`` and +``statistics_short_term`` tables in the HA-compatible schema written by +``SyntheticHistoryGenerator``. +""" + +from __future__ import annotations + +import logging +import sqlite3 +from pathlib import Path + +_LOGGER = logging.getLogger(__name__) + +# SQL schema for the companion history database. +SCHEMA_SQL = """\ +CREATE TABLE IF NOT EXISTS statistics_meta ( + id INTEGER PRIMARY KEY, + statistic_id TEXT UNIQUE NOT NULL, + source TEXT NOT NULL DEFAULT 'simulator', + unit_of_measurement TEXT, + has_mean INTEGER DEFAULT 1, + has_sum INTEGER DEFAULT 0, + name TEXT +); + +CREATE TABLE IF NOT EXISTS statistics ( + id INTEGER PRIMARY KEY, + metadata_id INTEGER NOT NULL REFERENCES statistics_meta(id), + created_ts REAL NOT NULL, + start_ts REAL NOT NULL, + mean REAL, + min REAL, + max REAL, + last_reset_ts REAL, + state REAL, + sum REAL, + UNIQUE(metadata_id, start_ts) +); + +CREATE TABLE IF NOT EXISTS statistics_short_term ( + id INTEGER PRIMARY KEY, + metadata_id INTEGER NOT NULL REFERENCES statistics_meta(id), + created_ts REAL NOT NULL, + start_ts REAL NOT NULL, + mean REAL, + min REAL, + max REAL, + last_reset_ts REAL, + state REAL, + sum REAL, + UNIQUE(metadata_id, start_ts) +); +""" + +# Period name -> table name mapping +_PERIOD_TABLE: dict[str, str] = { + "hour": "statistics", + "5minute": "statistics_short_term", +} + + +class SqliteHistoryProvider: + """Read-only history provider backed by a local SQLite file. + + The database uses HA's recorder schema: ``statistics_meta`` maps + statistic IDs to integer keys, and ``statistics`` / ``statistics_short_term`` + store hourly and 5-minute aggregated rows respectively. + + Timestamps are stored as epoch seconds (``start_ts`` column) and returned + in the same format that ``RecorderDataSource._parse_timestamp`` expects. + """ + + def __init__(self, db_path: str | Path) -> None: + self._db_path = str(db_path) + + async def async_get_statistics( + self, + statistic_ids: list[str], + *, + period: str = "hour", + start_time: str | None = None, + end_time: str | None = None, + ) -> dict[str, list[dict[str, object]]]: + """Query statistics from the SQLite database. + + Returns data in the same format as the HA provider: a dict mapping + statistic IDs to lists of records with ``start``, ``mean``, ``min``, + ``max`` fields. + """ + table = _PERIOD_TABLE.get(period) + if table is None: + return {} + + if not statistic_ids: + return {} + + result: dict[str, list[dict[str, object]]] = {} + + if self._db_path != ":memory:" and not Path(self._db_path).exists(): + _LOGGER.warning("History database not found: %s", self._db_path) + return {} + + try: + con = sqlite3.connect(self._db_path) + except sqlite3.Error: + _LOGGER.warning("Could not open history database: %s", self._db_path) + return {} + + try: + cur = con.cursor() + + # Resolve statistic_id -> metadata_id + placeholders = ",".join("?" for _ in statistic_ids) + cur.execute( + f"SELECT id, statistic_id FROM statistics_meta " + f"WHERE statistic_id IN ({placeholders})", + statistic_ids, + ) + meta_rows = cur.fetchall() + meta_map: dict[int, str] = {row[0]: row[1] for row in meta_rows} + + if not meta_map: + return {} + + for metadata_id, statistic_id in meta_map.items(): + query = f"SELECT start_ts, mean, min, max FROM {table} WHERE metadata_id = ?" + params: list[object] = [metadata_id] + + if start_time is not None: + from datetime import UTC, datetime + + try: + dt = datetime.fromisoformat(start_time) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + query += " AND start_ts >= ?" + params.append(dt.timestamp()) + except ValueError: + pass + + if end_time is not None: + from datetime import UTC, datetime + + try: + dt = datetime.fromisoformat(end_time) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + query += " AND start_ts <= ?" + params.append(dt.timestamp()) + except ValueError: + pass + + query += " ORDER BY start_ts" + cur.execute(query, params) + + records: list[dict[str, object]] = [] + for row in cur.fetchall(): + records.append( + { + "start": row[0], + "mean": row[1], + "min": row[2], + "max": row[3], + } + ) + + if records: + result[statistic_id] = records + finally: + con.close() + + return result diff --git a/tests/test_history.py b/tests/test_history.py index e1180c3..d9b1ac7 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -9,6 +9,7 @@ HistoryProvider, NullHistoryProvider, ) +from span_panel_simulator.sqlite_history import SqliteHistoryProvider class TestNullHistoryProvider: @@ -33,3 +34,9 @@ async def test_returns_empty(self) -> None: def test_satisfies_protocol(self) -> None: provider: HistoryProvider = EBusHistoryProvider() assert hasattr(provider, "async_get_statistics") + + +class TestSqliteHistoryProvider: + def test_satisfies_protocol(self) -> None: + provider: HistoryProvider = SqliteHistoryProvider(":memory:") + assert hasattr(provider, "async_get_statistics") diff --git a/tests/test_sqlite_history.py b/tests/test_sqlite_history.py new file mode 100644 index 0000000..bb5e05a --- /dev/null +++ b/tests/test_sqlite_history.py @@ -0,0 +1,115 @@ +"""Tests for SqliteHistoryProvider.""" + +from __future__ import annotations + +import sqlite3 +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +from span_panel_simulator.sqlite_history import SCHEMA_SQL, SqliteHistoryProvider + + +def _create_test_db(path: Path, entity_id: str, rows: list[tuple[float, float]]) -> None: + """Create a test SQLite DB with statistics_meta and statistics rows.""" + con = sqlite3.connect(str(path)) + con.executescript(SCHEMA_SQL) + con.execute( + "INSERT INTO statistics_meta (id, statistic_id, unit_of_measurement) VALUES (1, ?, 'W')", + (entity_id,), + ) + for start_ts, mean in rows: + con.execute( + "INSERT INTO statistics (metadata_id, created_ts, start_ts, mean, min, max) " + "VALUES (1, ?, ?, ?, ?, ?)", + (start_ts, start_ts, mean, mean * 0.9, mean * 1.1), + ) + con.commit() + con.close() + + +class TestSqliteHistoryProvider: + @pytest.mark.asyncio + async def test_reads_hourly_data(self, tmp_path: Path) -> None: + db_path = tmp_path / "test.db" + entity = "sensor.sim_panel_kitchen_power" + rows = [(1000.0, 500.0), (4600.0, 600.0), (8200.0, 550.0)] + _create_test_db(db_path, entity, rows) + + provider = SqliteHistoryProvider(db_path) + result = await provider.async_get_statistics([entity], period="hour") + + assert entity in result + assert len(result[entity]) == 3 + assert result[entity][0]["start"] == 1000.0 + assert result[entity][0]["mean"] == 500.0 + + @pytest.mark.asyncio + async def test_reads_short_term_data(self, tmp_path: Path) -> None: + db_path = tmp_path / "test.db" + entity = "sensor.sim_panel_kitchen_power" + con = sqlite3.connect(str(db_path)) + con.executescript(SCHEMA_SQL) + con.execute( + "INSERT INTO statistics_meta (id, statistic_id) VALUES (1, ?)", + (entity,), + ) + con.execute( + "INSERT INTO statistics_short_term " + "(metadata_id, created_ts, start_ts, mean, min, max) " + "VALUES (1, 1000.0, 1000.0, 200.0, 180.0, 220.0)", + ) + con.commit() + con.close() + + provider = SqliteHistoryProvider(db_path) + result = await provider.async_get_statistics([entity], period="5minute") + + assert entity in result + assert len(result[entity]) == 1 + assert result[entity][0]["mean"] == 200.0 + + @pytest.mark.asyncio + async def test_filters_by_start_time(self, tmp_path: Path) -> None: + db_path = tmp_path / "test.db" + entity = "sensor.test_power" + rows = [(1000.0, 100.0), (5000.0, 200.0), (9000.0, 300.0)] + _create_test_db(db_path, entity, rows) + + provider = SqliteHistoryProvider(db_path) + result = await provider.async_get_statistics( + [entity], + period="hour", + start_time="1970-01-01T01:00:00+00:00", + ) + + assert entity in result + assert len(result[entity]) == 2 + + @pytest.mark.asyncio + async def test_unknown_entity_returns_empty(self, tmp_path: Path) -> None: + db_path = tmp_path / "test.db" + _create_test_db(db_path, "sensor.real", [(1000.0, 100.0)]) + + provider = SqliteHistoryProvider(db_path) + result = await provider.async_get_statistics(["sensor.does_not_exist"], period="hour") + + assert result == {} + + @pytest.mark.asyncio + async def test_missing_db_returns_empty(self, tmp_path: Path) -> None: + provider = SqliteHistoryProvider(tmp_path / "nonexistent.db") + result = await provider.async_get_statistics(["sensor.x"], period="hour") + assert result == {} + + @pytest.mark.asyncio + async def test_unknown_period_returns_empty(self, tmp_path: Path) -> None: + db_path = tmp_path / "test.db" + _create_test_db(db_path, "sensor.x", [(1000.0, 100.0)]) + + provider = SqliteHistoryProvider(db_path) + result = await provider.async_get_statistics(["sensor.x"], period="month") + assert result == {} From 37c1aa95a522871508942807491203ca449523d6 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:00:04 -0700 Subject: [PATCH 05/30] feat: add history_db field to PanelConfig for explicit SQLite path --- src/span_panel_simulator/config_types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/span_panel_simulator/config_types.py b/src/span_panel_simulator/config_types.py index afa8ba2..5e74b29 100644 --- a/src/span_panel_simulator/config_types.py +++ b/src/span_panel_simulator/config_types.py @@ -30,6 +30,7 @@ class PanelConfig(TypedDict): soc_shed_threshold: NotRequired[float] # SOC % below which SOC_THRESHOLD circuits are shed postal_code: NotRequired[str] # ZIP / postal code, default "94103" time_zone: NotRequired[str] # IANA timezone, default "America/Los_Angeles" + history_db: NotRequired[str] # path to companion SQLite history file (overrides convention) class CyclingPattern(TypedDict, total=False): From 3744e4cc9682b42755e7379053b8e051feb63cb7 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:02:44 -0700 Subject: [PATCH 06/30] Wire SqliteHistoryProvider into app.py startup as second history source _load_recorder_data now checks for a companion SQLite history DB when no HA client is configured, using SqliteHistoryProvider with a 365-day lookback. A new _resolve_history_db static method resolves the DB path via explicit panel_config.history_db or the _history.db convention. --- src/span_panel_simulator/app.py | 105 ++++++++++++++++++++------ tests/test_sqlite_app_integration.py | 108 +++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 24 deletions(-) create mode 100644 tests/test_sqlite_app_integration.py diff --git a/src/span_panel_simulator/app.py b/src/span_panel_simulator/app.py index 23344c8..a13c548 100644 --- a/src/span_panel_simulator/app.py +++ b/src/span_panel_simulator/app.py @@ -364,15 +364,15 @@ async def _start_panel(self, config_path: Path) -> PanelInstance: return panel async def _load_recorder_data(self, config_path: Path) -> RecorderDataSource | None: - """Create and populate a RecorderDataSource from config + HA history. + """Create and populate a RecorderDataSource from config + history source. - Returns ``None`` if HA is unavailable or the config has no - ``recorder_entity`` mappings. Failures are logged and swallowed - so the panel still starts in synthetic mode. - """ - if self._ha_client is None: - return None + Source selection: + 1. If HA client is available and config has recorder_entity mappings → HA provider + 2. If a companion ``_history.db`` file exists (or ``history_db`` set) → SQLite provider + 3. Otherwise → None (engine uses synthetic per-tick generation) + Failures are logged and swallowed so the panel still starts in synthetic mode. + """ try: raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) except Exception: @@ -395,28 +395,85 @@ async def _load_recorder_data(self, config_path: Path) -> RecorderDataSource | N if not entity_ids: return None - _LOGGER.info( - "Loading recorder data for %s (%d entities)", - config_path.name, - len(entity_ids), - ) - recorder = RecorderDataSource() - try: - loaded = await recorder.load(self._ha_client, entity_ids) - except Exception: - _LOGGER.warning( - "Recorder data loading failed for %s — using synthetic", + # Source 1: HA client available → use HA provider + if self._ha_client is not None: + _LOGGER.info( + "Loading recorder data for %s (%d entities) from HA", config_path.name, - exc_info=True, + len(entity_ids), ) - return None + recorder = RecorderDataSource() + try: + loaded = await recorder.load(self._ha_client, entity_ids) + except Exception: + _LOGGER.warning( + "Recorder data loading failed for %s — using synthetic", + config_path.name, + exc_info=True, + ) + return None - if loaded == 0: - _LOGGER.warning( - "Recorder returned no data for %s — using synthetic", + if loaded == 0: + _LOGGER.warning( + "Recorder returned no data for %s — using synthetic", + config_path.name, + ) + return recorder if loaded > 0 else None + + # Source 2: companion SQLite file + db_path = self._resolve_history_db(config_path, raw) + if db_path is not None: + from span_panel_simulator.sqlite_history import SqliteHistoryProvider + + _LOGGER.info( + "Loading recorder data for %s (%d entities) from %s", config_path.name, + len(entity_ids), + db_path.name, ) - return recorder if loaded > 0 else None + provider = SqliteHistoryProvider(db_path) + recorder = RecorderDataSource() + try: + loaded = await recorder.load(provider, entity_ids, lookback_days=365) + except Exception: + _LOGGER.warning( + "SQLite history loading failed for %s — using synthetic", + config_path.name, + exc_info=True, + ) + return None + + if loaded == 0: + _LOGGER.warning( + "SQLite history returned no data for %s — using synthetic", + config_path.name, + ) + return recorder if loaded > 0 else None + + return None + + @staticmethod + def _resolve_history_db(config_path: Path, raw: dict[str, object]) -> Path | None: + """Find the companion SQLite history DB for a config file. + + Checks explicit ``panel_config.history_db`` first, then falls back + to the convention: ``_history.db`` in the same directory. + """ + panel_config = raw.get("panel_config") + if isinstance(panel_config, dict): + explicit = panel_config.get("history_db") + if isinstance(explicit, str) and explicit: + explicit_path = Path(explicit) + if not explicit_path.is_absolute(): + explicit_path = config_path.parent / explicit_path + if explicit_path.exists(): + return explicit_path + + convention_path = config_path.with_name(config_path.stem + "_history.db") + if convention_path.exists(): + return convention_path + + return None async def _stop_panel(self, config_path: Path) -> None: """Stop and unregister a panel.""" diff --git a/tests/test_sqlite_app_integration.py b/tests/test_sqlite_app_integration.py new file mode 100644 index 0000000..7fab23b --- /dev/null +++ b/tests/test_sqlite_app_integration.py @@ -0,0 +1,108 @@ +"""Integration test: SqliteHistoryProvider used at panel startup.""" + +from __future__ import annotations + +import sqlite3 +import time +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +from span_panel_simulator.app import SimulatorApp +from span_panel_simulator.recorder import RecorderDataSource +from span_panel_simulator.sqlite_history import SCHEMA_SQL, SqliteHistoryProvider + + +class TestSqliteRecorderRoundTrip: + """Verify that SqliteHistoryProvider feeds RecorderDataSource correctly.""" + + @pytest.mark.asyncio + async def test_load_and_get_power(self, tmp_path: Path) -> None: + """Generate rows, load via SqliteHistoryProvider, query via RecorderDataSource.""" + db_path = tmp_path / "panel_history.db" + entity = "sensor.sim_panel_kitchen_power" + + con = sqlite3.connect(str(db_path)) + con.executescript(SCHEMA_SQL) + con.execute( + "INSERT INTO statistics_meta (id, statistic_id, unit_of_measurement) " + "VALUES (1, ?, 'W')", + (entity,), + ) + # Use a base timestamp within the past 90 days so it falls inside + # the lookback window when recorder.load() computes start_time. + base_ts = time.time() - 7 * 86400 # 7 days ago + base_ts = base_ts - (base_ts % 3600) # align to hour boundary + for i in range(24): + ts = base_ts + i * 3600 + mean = 500.0 + i * 10.0 + con.execute( + "INSERT INTO statistics (metadata_id, created_ts, start_ts, mean, min, max) " + "VALUES (1, ?, ?, ?, ?, ?)", + (ts, ts, mean, mean * 0.9, mean * 1.1), + ) + con.commit() + con.close() + + provider = SqliteHistoryProvider(db_path) + recorder = RecorderDataSource() + loaded = await recorder.load(provider, [entity], lookback_days=365) + + assert loaded == 1 + assert recorder.has_entity(entity) + + # At i=12: mean = 500 + 12*10 = 620 W; at i=13: mean = 630 W + # Query exactly at i=12 to get 620 W + mid_ts = base_ts + 12 * 3600 + power = recorder.get_power(entity, mid_ts) + assert power is not None + assert 619.0 < power < 621.0 + + @pytest.mark.asyncio + async def test_no_db_file_returns_none(self, tmp_path: Path) -> None: + provider = SqliteHistoryProvider(tmp_path / "missing.db") + recorder = RecorderDataSource() + loaded = await recorder.load(provider, ["sensor.x"], lookback_days=365) + assert loaded == 0 + + +class TestResolveHistoryDb: + def test_convention_path(self, tmp_path: Path) -> None: + config_path = tmp_path / "my_panel.yaml" + config_path.write_text("panel_config:\n serial_number: x\n") + db_path = tmp_path / "my_panel_history.db" + db_path.write_text("") + + result = SimulatorApp._resolve_history_db(config_path, {}) + assert result == db_path + + def test_explicit_path(self, tmp_path: Path) -> None: + config_path = tmp_path / "my_panel.yaml" + config_path.write_text("") + db_path = tmp_path / "custom.db" + db_path.write_text("") + + raw = {"panel_config": {"history_db": "custom.db"}} + result = SimulatorApp._resolve_history_db(config_path, raw) + assert result == db_path + + def test_no_db_returns_none(self, tmp_path: Path) -> None: + config_path = tmp_path / "my_panel.yaml" + config_path.write_text("") + + result = SimulatorApp._resolve_history_db(config_path, {}) + assert result is None + + def test_explicit_overrides_convention(self, tmp_path: Path) -> None: + config_path = tmp_path / "my_panel.yaml" + config_path.write_text("") + (tmp_path / "my_panel_history.db").write_text("") + custom = tmp_path / "custom.db" + custom.write_text("") + + raw = {"panel_config": {"history_db": "custom.db"}} + result = SimulatorApp._resolve_history_db(config_path, raw) + assert result == custom From 704b937bbfbf105c9ce71d36af04f22d96eb2c0e Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:07:36 -0700 Subject: [PATCH 07/30] feat: add SyntheticHistoryGenerator for companion SQLite history databases Generates a year of synthetic power statistics from panel config YAMLs, writing hourly rows (statistics) and 5-minute rows (statistics_short_term) into a companion _history.db file compatible with SqliteHistoryProvider. Uses solar, weather, HVAC seasonal, time-of-day, cycling, and monthly factor modulation with deterministic per-row noise seeded from (panel_serial, circuit_id, start_ts) for reproducible output. --- src/span_panel_simulator/history_generator.py | 422 ++++++++++++++++++ tests/test_history_generator.py | 143 ++++++ 2 files changed, 565 insertions(+) create mode 100644 src/span_panel_simulator/history_generator.py create mode 100644 tests/test_history_generator.py diff --git a/src/span_panel_simulator/history_generator.py b/src/span_panel_simulator/history_generator.py new file mode 100644 index 0000000..4333f84 --- /dev/null +++ b/src/span_panel_simulator/history_generator.py @@ -0,0 +1,422 @@ +"""Synthetic history generator — builds companion SQLite databases. + +Given a panel config YAML, generates a year of synthetic power statistics +matching HA's recorder schema. The output SQLite file can be read by +``SqliteHistoryProvider`` and fed to ``RecorderDataSource`` for replay. + +Time windows: + - ``[anchor - 1 year, anchor - 10 days]``: hourly rows in ``statistics`` + - ``[anchor - 10 days, anchor]``: 5-minute rows in ``statistics_short_term`` + +Uses the same modulation infrastructure as the live simulation engine: +solar curves, weather degradation, HVAC seasonal model, time-of-day +profiles, cycling patterns, and monthly factors. +""" + +from __future__ import annotations + +import hashlib +import logging +import sqlite3 +import time +from datetime import UTC, datetime +from typing import TYPE_CHECKING +from zoneinfo import ZoneInfo + +if TYPE_CHECKING: + from pathlib import Path + +import yaml + +from span_panel_simulator.hvac import hvac_seasonal_factor +from span_panel_simulator.solar import daily_weather_factor, solar_production_factor +from span_panel_simulator.sqlite_history import SCHEMA_SQL +from span_panel_simulator.weather import fetch_historical_weather, get_cached_weather + +_LOGGER = logging.getLogger(__name__) + +_SECONDS_PER_HOUR = 3600 +_SECONDS_PER_5MIN = 300 +_DAYS_SHORT_TERM = 10 +_DAYS_TOTAL = 365 + + +def _deterministic_noise(panel_serial: str, circuit_id: str, start_ts: float) -> float: + """Deterministic per-row noise in [-1, 1], seeded from identity + timestamp.""" + raw = f"{panel_serial}:{circuit_id}:{start_ts}".encode() + h = int(hashlib.sha256(raw).hexdigest()[:8], 16) + return (h % 20000 - 10000) / 10000.0 + + +def _resolve_timezone(config: dict[str, object]) -> ZoneInfo: + """Resolve panel timezone from config, matching engine logic.""" + panel = config.get("panel_config", {}) + if not isinstance(panel, dict): + return ZoneInfo("America/Los_Angeles") + + explicit = panel.get("time_zone") + if isinstance(explicit, str) and explicit: + try: + return ZoneInfo(explicit) + except (KeyError, ValueError): + pass + + lat = panel.get("latitude") + lon = panel.get("longitude") + if lat is not None and lon is not None: + from timezonefinder import TimezoneFinder + + tz_name = TimezoneFinder().timezone_at(lat=float(lat), lng=float(lon)) + if tz_name is not None: + return ZoneInfo(tz_name) + + return ZoneInfo("America/Los_Angeles") + + +class SyntheticHistoryGenerator: + """Generate companion SQLite history databases from panel config YAMLs.""" + + async def generate( + self, + config_path: Path, + *, + anchor_time: float | None = None, + ) -> Path: + """Generate the companion history DB for a config file. + + Args: + config_path: Path to the panel YAML config. + anchor_time: Unix epoch for the "now" end of the window. + Defaults to current time. + + Returns: + Path to the generated ``_history.db`` file. + """ + raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + msg = f"Invalid config: {config_path}" + raise ValueError(msg) + + anchor = anchor_time if anchor_time is not None else time.time() + db_path = config_path.with_name(config_path.stem + "_history.db") + + panel_config = raw.get("panel_config", {}) + if not isinstance(panel_config, dict): + msg = "Missing panel_config" + raise ValueError(msg) + + serial = str(panel_config.get("serial_number", "unknown")) + lat = float(panel_config.get("latitude", 37.7)) + lon = float(panel_config.get("longitude", -122.4)) + tz = _resolve_timezone(raw) + sim_params = raw.get("simulation_params", {}) + noise_factor = float( + sim_params.get("noise_factor", 0.02) if isinstance(sim_params, dict) else 0.02 + ) + + # Fetch weather data for solar degradation (best-effort) + weather_monthly: dict[int, float] | None = None + cached = get_cached_weather(lat, lon) + if cached is not None: + weather_monthly = cached.monthly_factors + else: + try: + wd = await fetch_historical_weather(lat, lon) + weather_monthly = wd.monthly_factors + except Exception: + _LOGGER.debug("Weather fetch failed; using deterministic model", exc_info=True) + + # Collect circuits with recorder_entity mappings + templates = raw.get("circuit_templates", {}) + if not isinstance(templates, dict): + templates = {} + + circuits_to_generate: list[tuple[str, str, dict[str, object]]] = [] + for tmpl_name, tmpl in templates.items(): + if not isinstance(tmpl, dict): + continue + entity = tmpl.get("recorder_entity") + if isinstance(entity, str) and entity: + circuits_to_generate.append((tmpl_name, entity, tmpl)) + + if not circuits_to_generate: + _LOGGER.warning( + "No recorder_entity mappings in %s — nothing to generate", + config_path.name, + ) + con = sqlite3.connect(str(db_path)) + con.executescript(SCHEMA_SQL) + con.close() + return db_path + + # Compute time boundaries + hourly_start = anchor - _DAYS_TOTAL * 86400 + short_term_start = anchor - _DAYS_SHORT_TERM * 86400 + hourly_end = short_term_start + + _LOGGER.info( + "Generating synthetic history for %s: %d circuits, anchor=%s", + config_path.name, + len(circuits_to_generate), + datetime.fromtimestamp(anchor, tz=UTC).isoformat(), + ) + + con = sqlite3.connect(str(db_path)) + con.executescript(SCHEMA_SQL) + + # Clear any existing data (regeneration case) + con.execute("DELETE FROM statistics") + con.execute("DELETE FROM statistics_short_term") + con.execute("DELETE FROM statistics_meta") + + try: + for idx, (tmpl_name, entity_id, tmpl) in enumerate(circuits_to_generate, start=1): + con.execute( + "INSERT INTO statistics_meta " + "(id, statistic_id, source, unit_of_measurement, name) " + "VALUES (?, ?, 'simulator', 'W', ?)", + (idx, entity_id, tmpl_name), + ) + + self._generate_rows( + con=con, + table="statistics", + metadata_id=idx, + entity_id=entity_id, + template=tmpl, + start_ts=hourly_start, + end_ts=hourly_end, + step_seconds=_SECONDS_PER_HOUR, + serial=serial, + lat=lat, + lon=lon, + tz=tz, + noise_factor=noise_factor, + weather_monthly=weather_monthly, + ) + + self._generate_rows( + con=con, + table="statistics_short_term", + metadata_id=idx, + entity_id=entity_id, + template=tmpl, + start_ts=short_term_start, + end_ts=anchor, + step_seconds=_SECONDS_PER_5MIN, + serial=serial, + lat=lat, + lon=lon, + tz=tz, + noise_factor=noise_factor, + weather_monthly=weather_monthly, + ) + + con.commit() + finally: + con.close() + + _LOGGER.info("Wrote synthetic history to %s", db_path.name) + return db_path + + def _generate_rows( + self, + *, + con: sqlite3.Connection, + table: str, + metadata_id: int, + entity_id: str, + template: dict[str, object], + start_ts: float, + end_ts: float, + step_seconds: int, + serial: str, + lat: float, + lon: float, + tz: ZoneInfo, + noise_factor: float, + weather_monthly: dict[int, float] | None, + ) -> None: + """Generate statistics rows for one circuit into the given table.""" + ep = template.get("energy_profile", {}) + if not isinstance(ep, dict): + return + + mode = str(ep.get("mode", "consumer")) + typical_power = float(ep.get("typical_power", 0.0)) + nameplate_w = ep.get("nameplate_capacity_w") + nameplate = float(nameplate_w) if nameplate_w is not None else None + + # Time-of-day profile + tod_profile = template.get("time_of_day_profile", {}) + tod_enabled = isinstance(tod_profile, dict) and bool(tod_profile.get("enabled", False)) + hour_factors: dict[int, float] = {} + if isinstance(tod_profile, dict): + raw_hf = tod_profile.get("hour_factors", {}) + if isinstance(raw_hf, dict): + hour_factors = {int(k): float(v) for k, v in raw_hf.items()} + + # Monthly factors + monthly_factors: dict[int, float] = {} + raw_mf = template.get("monthly_factors") + if isinstance(raw_mf, dict): + monthly_factors = {int(k): float(v) for k, v in raw_mf.items()} + + # HVAC type + hvac_type = template.get("hvac_type") + hvac_type_str = str(hvac_type) if isinstance(hvac_type, str) else None + + # Cycling pattern + cycling = template.get("cycling_pattern") + duty_cycle: float | None = None + if isinstance(cycling, dict): + dc = cycling.get("duty_cycle") + if dc is not None: + duty_cycle = float(dc) + else: + on_dur = cycling.get("on_duration") + off_dur = cycling.get("off_duration") + if on_dur is not None and off_dur is not None: + total = int(on_dur) + int(off_dur) + if total > 0: + duty_cycle = int(on_dur) / total + + # Active days from time_of_day_profile + active_days: list[int] = [] + if isinstance(tod_profile, dict): + ad = tod_profile.get("active_days", []) + if isinstance(ad, list): + active_days = [int(d) for d in ad] + + # Power range for clamping + power_range = ep.get("power_range", [0, 10000]) + if isinstance(power_range, list) and len(power_range) == 2: + min_power, max_power = float(power_range[0]), float(power_range[1]) + else: + min_power, max_power = 0.0, 10000.0 + + # Mean of hour factors for normalisation + mean_hf = sum(hour_factors.values()) / len(hour_factors) if hour_factors else 1.0 + + # Mean of monthly factors for normalisation + mean_mf = sum(monthly_factors.values()) / len(monthly_factors) if monthly_factors else 1.0 + + batch: list[tuple[object, ...]] = [] + ts = start_ts + while ts < end_ts: + power = self._compute_power_at( + ts=ts, + mode=mode, + typical_power=typical_power, + nameplate=nameplate, + lat=lat, + lon=lon, + tz=tz, + serial=serial, + hour_factors=hour_factors, + mean_hf=mean_hf, + tod_enabled=tod_enabled, + monthly_factors=monthly_factors, + mean_mf=mean_mf, + hvac_type=hvac_type_str, + duty_cycle=duty_cycle, + active_days=active_days, + weather_monthly=weather_monthly, + ) + + # Apply deterministic noise + noise = _deterministic_noise(serial, entity_id, ts) + noisy_power = power * (1.0 + noise * noise_factor) + + # Clamp + if mode == "producer": + noisy_power = max(0.0, min(abs(min_power), noisy_power)) + else: + noisy_power = max(min_power, min(max_power, noisy_power)) + + mean_val = noisy_power + min_val = mean_val * (1.0 - noise_factor) + max_val = mean_val * (1.0 + noise_factor) + + batch.append((metadata_id, ts, ts, mean_val, min_val, max_val)) + + if len(batch) >= 1000: + con.executemany( + f"INSERT INTO {table} " + "(metadata_id, created_ts, start_ts, mean, min, max) " + "VALUES (?, ?, ?, ?, ?, ?)", + batch, + ) + batch.clear() + + ts += step_seconds + + if batch: + con.executemany( + f"INSERT INTO {table} " + "(metadata_id, created_ts, start_ts, mean, min, max) " + "VALUES (?, ?, ?, ?, ?, ?)", + batch, + ) + + def _compute_power_at( + self, + *, + ts: float, + mode: str, + typical_power: float, + nameplate: float | None, + lat: float, + lon: float, + tz: ZoneInfo, + serial: str, + hour_factors: dict[int, float], + mean_hf: float, + tod_enabled: bool, + monthly_factors: dict[int, float], + mean_mf: float, + hvac_type: str | None, + duty_cycle: float | None, + active_days: list[int], + weather_monthly: dict[int, float] | None, + ) -> float: + """Compute synthetic power for one time step.""" + dt = datetime.fromtimestamp(ts, tz=tz) + hour = dt.hour + weekday = dt.weekday() + month = dt.month + + if active_days and weekday not in active_days: + return 0.0 + + base = typical_power + + if mode == "producer": + scale = abs(nameplate) if nameplate is not None and nameplate > 0 else abs(base) + solar = solar_production_factor(ts, lat, lon) + weather = daily_weather_factor(ts, seed=hash(serial), monthly_factors=weather_monthly) + return scale * solar * weather + + # Time-of-day for consumers + if hour_factors and tod_enabled: + factor = hour_factors.get(hour, 0.0) + base = typical_power / mean_hf * factor if mean_hf > 0 else 0.0 + elif tod_enabled: + if hour >= 22 or hour <= 6: + base = typical_power * 0.3 + elif hour in range(7, 22): + base = typical_power + + # Monthly/seasonal modulation + if monthly_factors: + mf = monthly_factors.get(month, 1.0) + if mean_mf > 0: + base = base / mean_mf * mf + elif hvac_type is not None: + base = base * hvac_seasonal_factor(ts, lat, hvac_type, tz=tz) + + # Cycling: reduce by duty cycle + if duty_cycle is not None and duty_cycle < 1.0: + base = base * duty_cycle + + return base diff --git a/tests/test_history_generator.py b/tests/test_history_generator.py new file mode 100644 index 0000000..ae0f9f3 --- /dev/null +++ b/tests/test_history_generator.py @@ -0,0 +1,143 @@ +"""Tests for SyntheticHistoryGenerator.""" + +from __future__ import annotations + +import sqlite3 +from typing import TYPE_CHECKING + +import pytest +import yaml + +if TYPE_CHECKING: + from pathlib import Path + +from span_panel_simulator.history_generator import SyntheticHistoryGenerator + +_MINIMAL_CONFIG: dict[str, object] = { + "panel_config": { + "serial_number": "sim-test-gen", + "total_tabs": 16, + "main_size": 200, + "latitude": 37.7, + "longitude": -122.4, + }, + "circuit_templates": { + "kitchen": { + "energy_profile": { + "mode": "consumer", + "power_range": [0, 2400], + "typical_power": 800.0, + "power_variation": 0.1, + }, + "relay_behavior": "controllable", + "priority": "MUST_HAVE", + "recorder_entity": "sensor.sim_test_gen_kitchen_power", + }, + }, + "circuits": [ + {"id": "circuit_1", "name": "Kitchen", "template": "kitchen", "tabs": [1]}, + ], + "unmapped_tabs": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + "simulation_params": { + "update_interval": 5, + "time_acceleration": 1.0, + "noise_factor": 0.02, + "enable_realistic_behaviors": True, + }, +} + + +class TestSyntheticHistoryGenerator: + @pytest.mark.asyncio + async def test_generates_correct_tables(self, tmp_path: Path) -> None: + config_path = tmp_path / "test_panel.yaml" + config_path.write_text(yaml.dump(_MINIMAL_CONFIG)) + + gen = SyntheticHistoryGenerator() + db_path = await gen.generate(config_path) + + assert db_path.exists() + assert db_path.name == "test_panel_history.db" + + con = sqlite3.connect(str(db_path)) + meta = con.execute("SELECT statistic_id FROM statistics_meta").fetchall() + assert len(meta) == 1 + assert meta[0][0] == "sensor.sim_test_gen_kitchen_power" + + hourly_count = con.execute("SELECT COUNT(*) FROM statistics").fetchone()[0] + # ~355 days * 24 = 8520, allow some tolerance + assert hourly_count > 8000 + assert hourly_count < 9000 + + short_count = con.execute("SELECT COUNT(*) FROM statistics_short_term").fetchone()[0] + # 10 days * 288 five-minute slots = 2880 + assert short_count > 2800 + assert short_count < 3000 + + con.close() + + @pytest.mark.asyncio + async def test_deterministic_output(self, tmp_path: Path) -> None: + """Same config + anchor produces identical DBs.""" + config_path = tmp_path / "test_panel.yaml" + config_path.write_text(yaml.dump(_MINIMAL_CONFIG)) + + anchor = 1_700_000_000.0 + + gen = SyntheticHistoryGenerator() + db1 = await gen.generate(config_path, anchor_time=anchor) + + db1_copy = tmp_path / "db1.db" + db1.rename(db1_copy) + + db2 = await gen.generate(config_path, anchor_time=anchor) + + con1 = sqlite3.connect(str(db1_copy)) + con2 = sqlite3.connect(str(db2)) + + rows1 = con1.execute("SELECT start_ts, mean FROM statistics ORDER BY start_ts").fetchall() + rows2 = con2.execute("SELECT start_ts, mean FROM statistics ORDER BY start_ts").fetchall() + + assert rows1 == rows2 + con1.close() + con2.close() + + @pytest.mark.asyncio + async def test_solar_circuit_has_day_night_pattern(self, tmp_path: Path) -> None: + """Solar circuits should produce zero power at night, nonzero during day.""" + solar_config = { + **_MINIMAL_CONFIG, + "circuit_templates": { + "solar": { + "energy_profile": { + "mode": "producer", + "power_range": [-5000, 0], + "typical_power": -3000.0, + "power_variation": 0.05, + "nameplate_capacity_w": 5000.0, + }, + "relay_behavior": "non_controllable", + "priority": "NEVER", + "recorder_entity": "sensor.sim_test_gen_solar_power", + }, + }, + "circuits": [ + {"id": "circuit_1", "name": "Solar", "template": "solar", "tabs": [1]}, + ], + } + + config_path = tmp_path / "solar_panel.yaml" + config_path.write_text(yaml.dump(solar_config)) + + gen = SyntheticHistoryGenerator() + db_path = await gen.generate(config_path, anchor_time=1_719_792_000.0) + + con = sqlite3.connect(str(db_path)) + rows = con.execute( + "SELECT start_ts, mean FROM statistics ORDER BY start_ts LIMIT 48" + ).fetchall() + con.close() + + values = [r[1] for r in rows] + assert any(v == 0.0 for v in values), "Expected some zero (nighttime) rows" + assert any(v > 0.0 for v in values), "Expected some nonzero (daytime) rows" From f3ea36bd6c49962f0352407d35f31f124304de0e Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:08:35 -0700 Subject: [PATCH 08/30] feat: add standalone CLI for synthetic history generation --- src/span_panel_simulator/history_generator.py | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/src/span_panel_simulator/history_generator.py b/src/span_panel_simulator/history_generator.py index 4333f84..d4884a9 100644 --- a/src/span_panel_simulator/history_generator.py +++ b/src/span_panel_simulator/history_generator.py @@ -20,12 +20,9 @@ import sqlite3 import time from datetime import UTC, datetime -from typing import TYPE_CHECKING +from pathlib import Path from zoneinfo import ZoneInfo -if TYPE_CHECKING: - from pathlib import Path - import yaml from span_panel_simulator.hvac import hvac_seasonal_factor @@ -420,3 +417,35 @@ def _compute_power_at( base = base * duty_cycle return base + + +async def _cli_main() -> None: + """CLI entry point for standalone generation.""" + import argparse + + parser = argparse.ArgumentParser( + description="Generate synthetic history DB from a panel config YAML", + ) + parser.add_argument("config", type=Path, help="Path to the panel YAML config") + parser.add_argument( + "--anchor-time", + type=float, + default=None, + help="Unix epoch for the anchor (default: now)", + ) + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", + ) + + gen = SyntheticHistoryGenerator() + db_path = await gen.generate(args.config, anchor_time=args.anchor_time) + print(f"Generated: {db_path}") + + +if __name__ == "__main__": + import asyncio + + asyncio.run(_cli_main()) From 2939f3ddfb89f211401c842ef917312d10e2d1c3 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:09:04 -0700 Subject: [PATCH 09/30] feat: generate synthetic history DB on clone-from-panel --- src/span_panel_simulator/dashboard/routes.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/span_panel_simulator/dashboard/routes.py b/src/span_panel_simulator/dashboard/routes.py index f6ef28d..985e893 100644 --- a/src/span_panel_simulator/dashboard/routes.py +++ b/src/span_panel_simulator/dashboard/routes.py @@ -1557,6 +1557,19 @@ async def handle_clone_from_panel(request: web.Request) -> web.Response: clone_path = write_clone_config(config, ctx.config_dir, scraped.serial_number) + # Generate synthetic history companion DB for offline replay + try: + from span_panel_simulator.history_generator import SyntheticHistoryGenerator + + gen = SyntheticHistoryGenerator() + history_db = await gen.generate(clone_path) + _LOGGER.info("Generated synthetic history: %s", history_db.name) + except Exception: + _LOGGER.warning( + "Synthetic history generation failed — panel will use per-tick synthesis", + exc_info=True, + ) + # Load the clone config into the dashboard editor store = _store(request) store.load_from_file(clone_path) From 1a29c56b00bd8cf9eddecae921fd3957c392345f Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:11:54 -0700 Subject: [PATCH 10/30] feat: add end-to-end round-trip tests for synthetic history pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add TestEndToEndRoundTrip with two async tests that exercise the full generate → load → query pipeline. The lookback window is computed dynamically from the fixed anchor timestamp so the tests remain valid regardless of when they run. Coverage assertion matches the actual hourly window (365 - 10 short-term days = 355 days). --- tests/test_sqlite_app_integration.py | 99 ++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/tests/test_sqlite_app_integration.py b/tests/test_sqlite_app_integration.py index 7fab23b..d3ff6c0 100644 --- a/tests/test_sqlite_app_integration.py +++ b/tests/test_sqlite_app_integration.py @@ -12,6 +12,7 @@ from pathlib import Path from span_panel_simulator.app import SimulatorApp +from span_panel_simulator.history_generator import SyntheticHistoryGenerator from span_panel_simulator.recorder import RecorderDataSource from span_panel_simulator.sqlite_history import SCHEMA_SQL, SqliteHistoryProvider @@ -106,3 +107,101 @@ def test_explicit_overrides_convention(self, tmp_path: Path) -> None: raw = {"panel_config": {"history_db": "custom.db"}} result = SimulatorApp._resolve_history_db(config_path, raw) assert result == custom + + +_ROUNDTRIP_CONFIG: dict[str, object] = { + "panel_config": { + "serial_number": "sim-roundtrip", + "total_tabs": 16, + "main_size": 200, + "latitude": 37.7, + "longitude": -122.4, + }, + "circuit_templates": { + "kitchen": { + "energy_profile": { + "mode": "consumer", + "power_range": [0, 2400], + "typical_power": 800.0, + "power_variation": 0.1, + }, + "relay_behavior": "controllable", + "priority": "MUST_HAVE", + "recorder_entity": "sensor.sim_roundtrip_kitchen_power", + }, + }, + "circuits": [ + {"id": "circuit_1", "name": "Kitchen", "template": "kitchen", "tabs": [1]}, + ], + "unmapped_tabs": list(range(2, 17)), + "simulation_params": { + "update_interval": 5, + "time_acceleration": 1.0, + "noise_factor": 0.02, + "enable_realistic_behaviors": True, + }, +} + + +class TestEndToEndRoundTrip: + @pytest.mark.asyncio + async def test_generate_then_load_then_query(self, tmp_path: Path) -> None: + """Full pipeline: generate DB -> load via SqliteHistoryProvider -> query power values.""" + import yaml + + config_path = tmp_path / "roundtrip.yaml" + config_path.write_text(yaml.dump(_ROUNDTRIP_CONFIG)) + + anchor = 1_700_000_000.0 + entity = "sensor.sim_roundtrip_kitchen_power" + + # Step 1: Generate + gen = SyntheticHistoryGenerator() + db_path = await gen.generate(config_path, anchor_time=anchor) + assert db_path.exists() + + # Step 2: Load — compute lookback to cover anchor (fixed in past) + import time as _time + + days_since_anchor = int((_time.time() - anchor) / 86400) + 400 + provider = SqliteHistoryProvider(db_path) + recorder = RecorderDataSource() + loaded = await recorder.load(provider, [entity], lookback_days=days_since_anchor) + assert loaded == 1 + + # Step 3: Query + bounds = recorder.time_bounds() + assert bounds is not None + start, end = bounds + + # Coverage spans the hourly window (365 - 10 short-term days = 355 days) + coverage_days = (end - start) / 86400 + assert coverage_days > 350 + + # Query multiple points — all should return non-None + import random + + rng = random.Random(42) + for _ in range(100): + ts = rng.uniform(start, end) + power = recorder.get_power(entity, ts) + assert power is not None + assert power >= 0.0 # consumer circuit, always >= 0 + + @pytest.mark.asyncio + async def test_convention_discovery_works(self, tmp_path: Path) -> None: + """Verify that _resolve_history_db finds the generated companion file.""" + import yaml + + config_path = tmp_path / "discovery.yaml" + config_path.write_text(yaml.dump(_ROUNDTRIP_CONFIG)) + + gen = SyntheticHistoryGenerator() + db_path = await gen.generate(config_path, anchor_time=1_700_000_000.0) + + # The generated file should match the convention + assert db_path.name == "discovery_history.db" + + # SimulatorApp._resolve_history_db should find it + result = SimulatorApp._resolve_history_db(config_path, _ROUNDTRIP_CONFIG) + assert result == db_path From 6308932c8eadd4bf9f489716bcb8826bd4141103 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:17:52 -0700 Subject: [PATCH 11/30] fix: address code review findings for synthetic history - Move datetime imports to module level in sqlite_history.py - Wrap synchronous SQLite calls in asyncio.to_thread via _sync_get_statistics - Add --years CLI flag to history_generator.py (default 1); thread through to generate() - Add BESS schedule handling in _compute_power_at: charge/discharge/idle hours respected before consumer/producer logic --- ...-26-synthetic-history-generation-design.md | 74 +++++++++++++------ src/span_panel_simulator/history_generator.py | 53 ++++++++++++- src/span_panel_simulator/sqlite_history.py | 56 +++++++++----- 3 files changed, 138 insertions(+), 45 deletions(-) diff --git a/docs/superpowers/specs/2026-03-26-synthetic-history-generation-design.md b/docs/superpowers/specs/2026-03-26-synthetic-history-generation-design.md index ff71e58..bff9992 100644 --- a/docs/superpowers/specs/2026-03-26-synthetic-history-generation-design.md +++ b/docs/superpowers/specs/2026-03-26-synthetic-history-generation-design.md @@ -2,18 +2,22 @@ ## Problem -When a user clones a template to create a panel config and no HA recorder data is available, the simulator has no historical data to replay. The simulator should project what the history would have been using the circuit data, BESS, and EVSE configuration it already has, producing a year of synthetic recorder data at the same granularity a real HA instance would retain. +When a user clones a template to create a panel config and no HA recorder data is available, the simulator has no historical data to replay. The simulator +should project what the history would have been using the circuit data, BESS, and EVSE configuration it already has, producing a year of synthetic recorder data +at the same granularity a real HA instance would retain. ## Architecture ### Data Source Abstraction -The recorder playback layer (`RecorderDataSource`) consumes data through a provider interface (`HistoryProvider` protocol). The provider abstracts the source from the playback. Two peer provider implementations exist: +The recorder playback layer (`RecorderDataSource`) consumes data through a provider interface (`HistoryProvider` protocol). The provider abstracts the source +from the playback. Two peer provider implementations exist: - **HA provider** -- reads from a live Home Assistant instance via its statistics API - **SQLite provider** (`SqliteHistoryProvider`) -- reads from a local SQLite file -Both implement the same interface. Which one is used is a configuration choice, not a quality or priority distinction. The playback layer receives a provider and replays data identically regardless of source. +Both implement the same interface. Which one is used is a configuration choice, not a quality or priority distinction. The playback layer receives a provider +and replays data identically regardless of source. ``` +---------------------+ @@ -32,7 +36,9 @@ Both implement the same interface. Which one is used is a configuration choice, +---------+ +-------------+ ``` -**Changes to `RecorderDataSource`:** The `_HOURLY_LOOKBACK` constant (currently 90 days) must be configurable or increased to 365 days when loading from SQLite, since the generated history spans a full year. Without this, `RecorderDataSource.load()` would silently discard 9 months of generated data. The cleanest approach is to accept a `lookback_days` parameter at load time, defaulting to 90 for HA (matching current behavior) and 365 for SQLite. +**Changes to `RecorderDataSource`:** The `_HOURLY_LOOKBACK` constant (currently 90 days) must be configurable or increased to 365 days when loading from SQLite, +since the generated history spans a full year. Without this, `RecorderDataSource.load()` would silently discard 9 months of generated data. The cleanest +approach is to accept a `lookback_days` parameter at load time, defaulting to 90 for HA (matching current behavior) and 365 for SQLite. No changes to `RealisticBehaviorEngine.get_power()` or `engine.py` playback logic. @@ -52,11 +58,13 @@ No precedence between sources. The config determines which provider is used. ### 1. SqliteHistoryProvider -Implements the existing `HistoryProvider` protocol. Reads from a companion SQLite file using HA's recorder schema. Returns data in the same format as the live HA provider. +Implements the existing `HistoryProvider` protocol. Reads from a companion SQLite file using HA's recorder schema. Returns data in the same format as the live +HA provider. - Reads `statistics` table for hourly data - Reads `statistics_short_term` table for 5-minute data -- Returns records with `start` as epoch seconds (float) matching SQLite's `start_ts` column directly -- `RecorderDataSource._parse_timestamp` already handles this format +- Returns records with `start` as epoch seconds (float) matching SQLite's `start_ts` column directly -- `RecorderDataSource._parse_timestamp` already handles + this format - `RecorderDataSource` merges the two tiers using its existing logic ### 2. SyntheticHistoryGenerator @@ -64,11 +72,13 @@ Implements the existing `HistoryProvider` protocol. Reads from a companion SQLit Standalone module that takes a panel config YAML, runs the projection, and writes the companion SQLite. **Inputs:** + - Completed panel config YAML (circuits, templates, BESS config, EVSE profiles) - Panel latitude/longitude (for solar model and weather data) - Generation anchor: "now" (or configurable timestamp for testing) **Time windows generated:** + - `[anchor - 1 year, anchor - 10 days]`: hourly rows written to `statistics` table - `[anchor - 10 days, anchor]`: 5-minute rows written to `statistics_short_term` table @@ -76,16 +86,17 @@ This mirrors HA's actual retention model: hourly data is permanent, 5-minute dat **Per-circuit generation strategy:** -| Circuit Type | Generation Approach | -|---|---| -| Consumer (loads) | `typical_power` x time-of-day profile x monthly/seasonal factors x noise | -| Producer (solar/PV) | Solar production model x weather degradation (Open-Meteo) x panel capacity | -| BESS | Charge/discharge/idle schedule from `battery_behavior` config, respecting `max_charge_power`, `max_discharge_power`, SOE constraints | -| EVSE | `time_of_day_profile.hour_factors` x rated power, with session randomization | -| HVAC | Temperature-aware seasonal model (existing `hvac_type` logic) x duty cycle | -| Cycling loads | `cycling_pattern` (duty cycle or on/off durations) applied per period | +| Circuit Type | Generation Approach | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | +| Consumer (loads) | `typical_power` x time-of-day profile x monthly/seasonal factors x noise | +| Producer (solar/PV) | Solar production model x weather degradation (Open-Meteo) x panel capacity | +| BESS | Charge/discharge/idle schedule from `battery_behavior` config, respecting `max_charge_power`, `max_discharge_power`, SOE constraints | +| EVSE | `time_of_day_profile.hour_factors` x rated power, with session randomization | +| HVAC | Temperature-aware seasonal model (existing `hvac_type` logic) x duty cycle | +| Cycling loads | `cycling_pattern` (duty cycle or on/off durations) applied per period | **Per statistics row, fields populated:** + - `start_ts`: period start epoch - `mean`: computed power for that period - `min`: `mean x (1 - noise_factor)` @@ -93,27 +104,35 @@ This mirrors HA's actual retention model: hourly data is permanent, 5-minute dat - `created_ts`: `start_ts` (synthetic but plausible) - `sum`: NULL for v1 (power sensors use `mean`/`min`/`max`; energy accumulation via `sum` can be added later if dashboard kWh charts require it) -Reuses the existing modulation infrastructure from `RealisticBehaviorEngine`: solar curves (`solar.py`), weather degradation (`weather.py`), seasonal/monthly factors, time-of-day profiles, cycling patterns, and HVAC modeling. +Reuses the existing modulation infrastructure from `RealisticBehaviorEngine`: solar curves (`solar.py`), weather degradation (`weather.py`), seasonal/monthly +factors, time-of-day profiles, cycling patterns, and HVAC modeling. -**Noise model:** Per-row noise is deterministic, seeded from a hash of `(panel_serial, circuit_id, start_ts)`. This ensures regenerating the DB produces identical data, matching the approach already used by `daily_weather_factor` in `solar.py`. +**Noise model:** Per-row noise is deterministic, seeded from a hash of `(panel_serial, circuit_id, start_ts)`. This ensures regenerating the DB produces +identical data, matching the approach already used by `daily_weather_factor` in `solar.py`. -**Timezone handling:** The generator uses the panel's configured timezone (`panel_config.time_zone`, or derived from lat/lon) to convert UTC epoch timestamps to local time when applying time-of-day profiles and BESS charge/discharge schedules. +**Timezone handling:** The generator uses the panel's configured timezone (`panel_config.time_zone`, or derived from lat/lon) to convert UTC epoch timestamps to +local time when applying time-of-day profiles and BESS charge/discharge schedules. -**BESS SOE tracking:** BESS circuits are generated in strict chronological order, carrying state-of-energy forward across all rows. Initial SOE starts at `backup_reserve_pct`. This means BESS generation cannot be parallelized per-circuit. +**BESS SOE tracking:** BESS circuits are generated in strict chronological order, carrying state-of-energy forward across all rows. Initial SOE starts at +`backup_reserve_pct`. This means BESS generation cannot be parallelized per-circuit. ### 3. Provider Selection Logic In `PanelInstance`/`app.py` startup, the configured data source determines which provider is instantiated and passed to `RecorderDataSource`. **Changes to existing files:** -- **`app.py`**: `_load_recorder_data()` currently only creates a `RecorderDataSource` when an HA client is available. A second code path is needed: when a companion `_history.db` exists, instantiate `SqliteHistoryProvider` and pass it to `RecorderDataSource` with `lookback_days=365`. -- **`config_types.py`**: Add `history_db: NotRequired[str]` to the `PanelConfig` TypedDict for explicit path override (convention-based discovery is the default). + +- **`app.py`**: `_load_recorder_data()` currently only creates a `RecorderDataSource` when an HA client is available. A second code path is needed: when a + companion `_history.db` exists, instantiate `SqliteHistoryProvider` and pass it to `RecorderDataSource` with `lookback_days=365`. +- **`config_types.py`**: Add `history_db: NotRequired[str]` to the `PanelConfig` TypedDict for explicit path override (convention-based discovery is the + default). ## SQLite Schema File convention: `configs/_history.db` alongside the panel YAML. -Discovery: `SqliteHistoryProvider` receives the DB path at construction. `PanelInstance` resolves it by convention (swap `.yaml` to `_history.db`) or from an explicit `history_db` field in `panel_config`. +Discovery: `SqliteHistoryProvider` receives the DB path at construction. `PanelInstance` resolves it by convention (swap `.yaml` to `_history.db`) or from an +explicit `history_db` field in `panel_config`. ```sql CREATE TABLE statistics_meta ( @@ -155,15 +174,18 @@ CREATE TABLE statistics_short_term ( ); ``` -Entity naming in `statistics_meta`: `sensor.__power` -- matches what the SPAN HA integration produces, so `recorder_entity` mappings work identically. +Entity naming in `statistics_meta`: `sensor.__power` -- matches what the SPAN HA integration produces, so `recorder_entity` mappings +work identically. ## Clone Pipeline Integration After `translate_scraped_panel()` produces the config dict and `write_clone_config()` writes the YAML: + 1. Call `await SyntheticHistoryGenerator.generate(config_path)` -- async because weather data fetching (`weather.py`) uses async HTTP 2. Generator reads the YAML, runs the projection, writes the companion `_history.db` 3. Clone output includes both files -4. If generation fails (e.g., network unavailable for Open-Meteo), the clone still succeeds with the YAML. The generator falls back to the deterministic weather model in `solar.py` (no-network fallback). The SQLite is still produced, just with less accurate weather variation. +4. If generation fails (e.g., network unavailable for Open-Meteo), the clone still succeeds with the YAML. The generator falls back to the deterministic weather + model in `solar.py` (no-network fallback). The SQLite is still produced, just with less accurate weather variation. Generation is invoked from both the dashboard clone endpoint and the CLI clone command. @@ -180,13 +202,17 @@ python -m span_panel_simulator.history_generator configs/my_panel.yaml ## Testing Strategy **Unit tests:** + - `SyntheticHistoryGenerator`: given a known config, verify correct number of rows, correct time ranges, power values within expected bounds per circuit type - `SqliteHistoryProvider`: given a pre-built SQLite, verify it returns data in the same format as `HistoryProvider` -- Round-trip: generate -> load via `SqliteHistoryProvider` -> verify `RecorderDataSource.get_power()` returns interpolated values consistent with generation inputs +- Round-trip: generate -> load via `SqliteHistoryProvider` -> verify `RecorderDataSource.get_power()` returns interpolated values consistent with generation + inputs **Integration tests:** + - Clone a template panel -> verify companion `_history.db` is created alongside YAML - Start a `PanelInstance` from that config -> verify it replays synthetic history identically to how it would replay HA data **CLI test:** + - Run the standalone generator against a test config, verify the SQLite output schema and row counts diff --git a/src/span_panel_simulator/history_generator.py b/src/span_panel_simulator/history_generator.py index d4884a9..bbb2265 100644 --- a/src/span_panel_simulator/history_generator.py +++ b/src/span_panel_simulator/history_generator.py @@ -78,6 +78,7 @@ async def generate( config_path: Path, *, anchor_time: float | None = None, + years: int | None = None, ) -> Path: """Generate the companion history DB for a config file. @@ -85,6 +86,8 @@ async def generate( config_path: Path to the panel YAML config. anchor_time: Unix epoch for the "now" end of the window. Defaults to current time. + years: Number of years of history to generate. Overrides the + module-level ``_DAYS_TOTAL`` constant when provided. Returns: Path to the generated ``_history.db`` file. @@ -95,6 +98,7 @@ async def generate( raise ValueError(msg) anchor = anchor_time if anchor_time is not None else time.time() + days_total = (years * 365) if years is not None else _DAYS_TOTAL db_path = config_path.with_name(config_path.stem + "_history.db") panel_config = raw.get("panel_config", {}) @@ -147,7 +151,7 @@ async def generate( return db_path # Compute time boundaries - hourly_start = anchor - _DAYS_TOTAL * 86400 + hourly_start = anchor - days_total * 86400 short_term_start = anchor - _DAYS_SHORT_TERM * 86400 hourly_end = short_term_start @@ -278,6 +282,14 @@ def _generate_rows( if total > 0: duty_cycle = int(on_dur) / total + # Battery behavior (BESS schedule) + battery_behavior_raw = template.get("battery_behavior") + battery_behavior: dict[str, object] | None = None + if isinstance(battery_behavior_raw, dict) and bool( + battery_behavior_raw.get("enabled", False) + ): + battery_behavior = battery_behavior_raw + # Active days from time_of_day_profile active_days: list[int] = [] if isinstance(tod_profile, dict): @@ -319,6 +331,7 @@ def _generate_rows( duty_cycle=duty_cycle, active_days=active_days, weather_monthly=weather_monthly, + battery_behavior=battery_behavior, ) # Apply deterministic noise @@ -376,6 +389,7 @@ def _compute_power_at( duty_cycle: float | None, active_days: list[int], weather_monthly: dict[int, float] | None, + battery_behavior: dict[str, object] | None = None, ) -> float: """Compute synthetic power for one time step.""" dt = datetime.fromtimestamp(ts, tz=tz) @@ -386,6 +400,35 @@ def _compute_power_at( if active_days and weekday not in active_days: return 0.0 + # BESS schedule takes priority over consumer/producer logic + if battery_behavior is not None: + charge_hours_raw = battery_behavior.get("charge_hours", []) + discharge_hours_raw = battery_behavior.get("discharge_hours", []) + idle_hours_raw = battery_behavior.get("idle_hours", []) + charge_hours = list(charge_hours_raw) if isinstance(charge_hours_raw, list) else [] + discharge_hours = ( + list(discharge_hours_raw) if isinstance(discharge_hours_raw, list) else [] + ) + idle_hours = list(idle_hours_raw) if isinstance(idle_hours_raw, list) else [] + + if hour in charge_hours: + max_charge = battery_behavior.get("max_charge_power") + if isinstance(max_charge, int | float): + return -float(max_charge) + return -typical_power + + if hour in discharge_hours: + max_discharge = battery_behavior.get("max_discharge_power") + if isinstance(max_discharge, int | float): + return float(max_discharge) + return typical_power + + if hour in idle_hours: + idle_range = battery_behavior.get("idle_power_range") + if isinstance(idle_range, list) and len(idle_range) == 2: + return float(idle_range[0]) + return 0.0 + base = typical_power if mode == "producer": @@ -433,6 +476,12 @@ async def _cli_main() -> None: default=None, help="Unix epoch for the anchor (default: now)", ) + parser.add_argument( + "--years", + type=int, + default=1, + help="Number of years of history to generate (default: 1)", + ) args = parser.parse_args() logging.basicConfig( @@ -441,7 +490,7 @@ async def _cli_main() -> None: ) gen = SyntheticHistoryGenerator() - db_path = await gen.generate(args.config, anchor_time=args.anchor_time) + db_path = await gen.generate(args.config, anchor_time=args.anchor_time, years=args.years) print(f"Generated: {db_path}") diff --git a/src/span_panel_simulator/sqlite_history.py b/src/span_panel_simulator/sqlite_history.py index ffaaa36..65a7086 100644 --- a/src/span_panel_simulator/sqlite_history.py +++ b/src/span_panel_simulator/sqlite_history.py @@ -7,8 +7,10 @@ from __future__ import annotations +import asyncio import logging import sqlite3 +from datetime import UTC, datetime from pathlib import Path _LOGGER = logging.getLogger(__name__) @@ -75,27 +77,18 @@ class SqliteHistoryProvider: def __init__(self, db_path: str | Path) -> None: self._db_path = str(db_path) - async def async_get_statistics( + def _sync_get_statistics( self, statistic_ids: list[str], *, - period: str = "hour", - start_time: str | None = None, - end_time: str | None = None, + table: str, + start_time: str | None, + end_time: str | None, ) -> dict[str, list[dict[str, object]]]: - """Query statistics from the SQLite database. + """Synchronous SQLite query for statistics data. - Returns data in the same format as the HA provider: a dict mapping - statistic IDs to lists of records with ``start``, ``mean``, ``min``, - ``max`` fields. + Intended to be called via ``asyncio.to_thread``. """ - table = _PERIOD_TABLE.get(period) - if table is None: - return {} - - if not statistic_ids: - return {} - result: dict[str, list[dict[str, object]]] = {} if self._db_path != ":memory:" and not Path(self._db_path).exists(): @@ -129,8 +122,6 @@ async def async_get_statistics( params: list[object] = [metadata_id] if start_time is not None: - from datetime import UTC, datetime - try: dt = datetime.fromisoformat(start_time) if dt.tzinfo is None: @@ -141,8 +132,6 @@ async def async_get_statistics( pass if end_time is not None: - from datetime import UTC, datetime - try: dt = datetime.fromisoformat(end_time) if dt.tzinfo is None: @@ -172,3 +161,32 @@ async def async_get_statistics( con.close() return result + + async def async_get_statistics( + self, + statistic_ids: list[str], + *, + period: str = "hour", + start_time: str | None = None, + end_time: str | None = None, + ) -> dict[str, list[dict[str, object]]]: + """Query statistics from the SQLite database. + + Returns data in the same format as the HA provider: a dict mapping + statistic IDs to lists of records with ``start``, ``mean``, ``min``, + ``max`` fields. + """ + table = _PERIOD_TABLE.get(period) + if table is None: + return {} + + if not statistic_ids: + return {} + + return await asyncio.to_thread( + self._sync_get_statistics, + statistic_ids, + table=table, + start_time=start_time, + end_time=end_time, + ) From def64b7006df68954c0f39304eb2986e32de2d24 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:26:39 -0700 Subject: [PATCH 12/30] fix: generate history after profile import so recorder_entity mappings exist --- src/span_panel_simulator/dashboard/routes.py | 31 +++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/span_panel_simulator/dashboard/routes.py b/src/span_panel_simulator/dashboard/routes.py index 985e893..83b4b41 100644 --- a/src/span_panel_simulator/dashboard/routes.py +++ b/src/span_panel_simulator/dashboard/routes.py @@ -1557,19 +1557,6 @@ async def handle_clone_from_panel(request: web.Request) -> web.Response: clone_path = write_clone_config(config, ctx.config_dir, scraped.serial_number) - # Generate synthetic history companion DB for offline replay - try: - from span_panel_simulator.history_generator import SyntheticHistoryGenerator - - gen = SyntheticHistoryGenerator() - history_db = await gen.generate(clone_path) - _LOGGER.info("Generated synthetic history: %s", history_db.name) - except Exception: - _LOGGER.warning( - "Synthetic history generation failed — panel will use per-tick synthesis", - exc_info=True, - ) - # Load the clone config into the dashboard editor store = _store(request) store.load_from_file(clone_path) @@ -1577,7 +1564,9 @@ async def handle_clone_from_panel(request: web.Request) -> web.Response: _LOGGER.info("Panel cloned from %s -> %s", host, clone_path.name) - # Automatically import HA usage profiles for the cloned panel + # Automatically import HA usage profiles for the cloned panel. + # This must happen BEFORE history generation because it populates + # the recorder_entity mappings the generator needs. profiles_imported = 0 if ctx.ha_client is not None and ctx.history_provider is not None: try: @@ -1587,6 +1576,20 @@ async def handle_clone_from_panel(request: web.Request) -> web.Response: except Exception: _LOGGER.debug("HA profile import after clone failed", exc_info=True) + # Generate synthetic history companion DB for offline replay. + # Runs after profile import so recorder_entity mappings are present. + try: + from span_panel_simulator.history_generator import SyntheticHistoryGenerator + + gen = SyntheticHistoryGenerator() + history_db = await gen.generate(clone_path) + _LOGGER.info("Generated synthetic history: %s", history_db.name) + except Exception: + _LOGGER.warning( + "Synthetic history generation failed — panel will use per-tick synthesis", + exc_info=True, + ) + # Start the clone engine (also triggers reload) ctx.start_panel(clone_path.name) From fe25f5dd3710311b5cf82c6bb45dcbfe20c39248 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:41:32 -0700 Subject: [PATCH 13/30] fix: derive recorder_entity for configs without HA, generate history on simple clone --- src/span_panel_simulator/app.py | 19 +++++++- src/span_panel_simulator/dashboard/routes.py | 39 +++++++++++++++ src/span_panel_simulator/history_generator.py | 48 ++++++++++++++++++- 3 files changed, 103 insertions(+), 3 deletions(-) diff --git a/src/span_panel_simulator/app.py b/src/span_panel_simulator/app.py index a13c548..4b5b111 100644 --- a/src/span_panel_simulator/app.py +++ b/src/span_panel_simulator/app.py @@ -386,15 +386,27 @@ async def _load_recorder_data(self, config_path: Path) -> RecorderDataSource | N return None entity_ids: list[str] = [] - for tmpl in templates.values(): + for tmpl_name, tmpl in templates.items(): if isinstance(tmpl, dict): entity_id = tmpl.get("recorder_entity") if isinstance(entity_id, str) and entity_id: entity_ids.append(entity_id) + _LOGGER.debug(" recorder_entity: %s -> %s", tmpl_name, entity_id) if not entity_ids: + _LOGGER.info( + "Recorder: no recorder_entity mappings in %s — skipping", + config_path.name, + ) return None + _LOGGER.info( + "Recorder: %d entities found, ha_client=%s, checking sources for %s", + len(entity_ids), + self._ha_client is not None, + config_path.name, + ) + # Source 1: HA client available → use HA provider if self._ha_client is not None: _LOGGER.info( @@ -422,6 +434,11 @@ async def _load_recorder_data(self, config_path: Path) -> RecorderDataSource | N # Source 2: companion SQLite file db_path = self._resolve_history_db(config_path, raw) + _LOGGER.info( + "Recorder: SQLite companion for %s: %s", + config_path.name, + db_path, + ) if db_path is not None: from span_panel_simulator.sqlite_history import SqliteHistoryProvider diff --git a/src/span_panel_simulator/dashboard/routes.py b/src/span_panel_simulator/dashboard/routes.py index 83b4b41..e351d67 100644 --- a/src/span_panel_simulator/dashboard/routes.py +++ b/src/span_panel_simulator/dashboard/routes.py @@ -1095,6 +1095,21 @@ async def handle_clone(request: web.Request) -> web.Response: output_path.write_text(yaml_content, encoding="utf-8") _LOGGER.info("Config cloned to %s", output_path) + + # Generate synthetic history companion DB for the cloned config. + # The generator will derive recorder_entity mappings if needed. + try: + from span_panel_simulator.history_generator import SyntheticHistoryGenerator + + gen = SyntheticHistoryGenerator() + history_db = await gen.generate(output_path) + _LOGGER.info("Generated synthetic history: %s", history_db.name) + except Exception: + _LOGGER.warning( + "Synthetic history generation failed for clone", + exc_info=True, + ) + return web.Response( text=f'
Cloned to {filename}
', content_type="text/html", @@ -1568,14 +1583,38 @@ async def handle_clone_from_panel(request: web.Request) -> web.Response: # This must happen BEFORE history generation because it populates # the recorder_entity mappings the generator needs. profiles_imported = 0 + _LOGGER.debug( + "Clone profile import: ha_client=%s, history_provider=%s", + ctx.ha_client is not None, + ctx.history_provider is not None, + ) if ctx.ha_client is not None and ctx.history_provider is not None: try: profiles_imported = await _import_profiles_for_serial( ctx.ha_client, ctx.history_provider, clone_path, scraped.serial_number ) + _LOGGER.info("Imported %d profiles before history generation", profiles_imported) except Exception: _LOGGER.debug("HA profile import after clone failed", exc_info=True) + # Check if recorder_entity mappings now exist in the written config + import yaml as _yaml + + _check_raw = _yaml.safe_load(clone_path.read_text(encoding="utf-8")) + _check_entities = [] + if isinstance(_check_raw, dict): + _check_tmpls = _check_raw.get("circuit_templates", {}) + if isinstance(_check_tmpls, dict): + for _tn, _tv in _check_tmpls.items(): + if isinstance(_tv, dict) and _tv.get("recorder_entity"): + _check_entities.append(f"{_tn}={_tv['recorder_entity']}") + _LOGGER.info( + "Pre-generate check: %d recorder_entity mappings in %s: %s", + len(_check_entities), + clone_path.name, + _check_entities[:5], + ) + # Generate synthetic history companion DB for offline replay. # Runs after profile import so recorder_entity mappings are present. try: diff --git a/src/span_panel_simulator/history_generator.py b/src/span_panel_simulator/history_generator.py index bbb2265..3879539 100644 --- a/src/span_panel_simulator/history_generator.py +++ b/src/span_panel_simulator/history_generator.py @@ -127,22 +127,66 @@ async def generate( except Exception: _LOGGER.debug("Weather fetch failed; using deterministic model", exc_info=True) - # Collect circuits with recorder_entity mappings + # Collect circuits to generate history for. + # If a template has an explicit recorder_entity, use it. + # Otherwise, derive one from (serial, circuit_id) using the HA + # naming convention: sensor.__power. + # This also writes the derived entity back into the YAML so + # the engine's recorder replay path picks it up at startup. templates = raw.get("circuit_templates", {}) if not isinstance(templates, dict): templates = {} + circuit_defs = raw.get("circuits", []) + if not isinstance(circuit_defs, list): + circuit_defs = [] + + # Build a map: template_name -> list of circuit_ids using it + tmpl_to_circuits: dict[str, list[str]] = {} + for cdef in circuit_defs: + if isinstance(cdef, dict): + cid = cdef.get("id") + tname = cdef.get("template") + if isinstance(cid, str) and isinstance(tname, str): + tmpl_to_circuits.setdefault(tname, []).append(cid) + circuits_to_generate: list[tuple[str, str, dict[str, object]]] = [] + derived_entities = False for tmpl_name, tmpl in templates.items(): if not isinstance(tmpl, dict): continue entity = tmpl.get("recorder_entity") if isinstance(entity, str) and entity: circuits_to_generate.append((tmpl_name, entity, tmpl)) + continue + + # Derive entity from the first circuit using this template + circuit_ids = tmpl_to_circuits.get(tmpl_name, []) + if not circuit_ids: + continue + # Use first circuit ID for the entity name + cid = circuit_ids[0] + clean_serial = serial.replace("-", "_") + derived_entity = f"sensor.{clean_serial}_{cid}_power" + tmpl["recorder_entity"] = derived_entity + circuits_to_generate.append((tmpl_name, derived_entity, tmpl)) + derived_entities = True + + if derived_entities: + # Write updated config with recorder_entity mappings + config_path.write_text( + yaml.dump(raw, default_flow_style=False, sort_keys=False), + encoding="utf-8", + ) + _LOGGER.info( + "Derived %d recorder_entity mappings for %s", + sum(1 for _ in circuits_to_generate if derived_entities), + config_path.name, + ) if not circuits_to_generate: _LOGGER.warning( - "No recorder_entity mappings in %s — nothing to generate", + "No circuits to generate history for in %s", config_path.name, ) con = sqlite3.connect(str(db_path)) From a029ec36177290595b5bdd8100636c26c2c7da43 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 14:00:04 -0700 Subject: [PATCH 14/30] fix: fall through to SQLite when HA returns no recorder data --- src/span_panel_simulator/app.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/span_panel_simulator/app.py b/src/span_panel_simulator/app.py index 4b5b111..85f21ce 100644 --- a/src/span_panel_simulator/app.py +++ b/src/span_panel_simulator/app.py @@ -407,7 +407,7 @@ async def _load_recorder_data(self, config_path: Path) -> RecorderDataSource | N config_path.name, ) - # Source 1: HA client available → use HA provider + # Source 1: HA client available → try HA provider if self._ha_client is not None: _LOGGER.info( "Loading recorder data for %s (%d entities) from HA", @@ -419,18 +419,18 @@ async def _load_recorder_data(self, config_path: Path) -> RecorderDataSource | N loaded = await recorder.load(self._ha_client, entity_ids) except Exception: _LOGGER.warning( - "Recorder data loading failed for %s — using synthetic", + "HA recorder loading failed for %s — trying SQLite fallback", config_path.name, exc_info=True, ) - return None + loaded = 0 - if loaded == 0: - _LOGGER.warning( - "Recorder returned no data for %s — using synthetic", - config_path.name, - ) - return recorder if loaded > 0 else None + if loaded > 0: + return recorder + _LOGGER.info( + "HA returned no data for %s — trying SQLite fallback", + config_path.name, + ) # Source 2: companion SQLite file db_path = self._resolve_history_db(config_path, raw) From 24b22760301e0828d7252cba35bdbabe622b005c Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:00:39 -0700 Subject: [PATCH 15/30] debug: show JS error message in modeling view catch block --- .../dashboard/templates/partials/modeling_view.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html b/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html index 618f2f3..88f4adb 100644 --- a/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html +++ b/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html @@ -174,7 +174,8 @@

Modeling —

}) .catch(function(err) { spinner.style.display = 'none'; - chartsWrap.innerHTML = '

Error loading data

'; + console.error('Modeling error:', err); + chartsWrap.innerHTML = '

Error loading data: ' + err.message + '

'; chartsWrap.style.display = ''; }); } From 3231207b2cc71eb76bffdaa8111f6cda038b0b86 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:02:08 -0700 Subject: [PATCH 16/30] fix: BSEE schedule always authoritative for discharge/idle hours in modeling pass --- src/span_panel_simulator/bsee.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/span_panel_simulator/bsee.py b/src/span_panel_simulator/bsee.py index c9ec6cb..b4e510c 100644 --- a/src/span_panel_simulator/bsee.py +++ b/src/span_panel_simulator/bsee.py @@ -208,26 +208,42 @@ def _resolve_battery_state(self, current_time: float) -> str: Grid-forced-offline always overrides the schedule: the battery must discharge to supply loads during an outage. + + For non-custom charge modes (solar-gen, solar-excess) the live + engine sets ``last_battery_direction`` each tick. However, during + the modeling pass the battery circuit may be replayed from the + recorder (skipping ``_apply_battery_behavior``), leaving the + direction stale at "idle". The schedule is always authoritative + for discharge/idle hours regardless of charge mode, so we consult + it first and only defer to the behavior engine for charge-hour + decisions where the mode matters. """ if self._forced_offline: return "discharging" - charge_mode: str = self._battery_behavior.get("charge_mode", "custom") - if charge_mode != "custom" and self._behavior_engine is not None: - return self._behavior_engine.last_battery_direction - current_hour = datetime.fromtimestamp(current_time, tz=self._tz).hour - charge_hours: list[int] = self._battery_behavior.get("charge_hours", []) discharge_hours: list[int] = self._battery_behavior.get("discharge_hours", []) idle_hours: list[int] = self._battery_behavior.get("idle_hours", []) - if current_hour in charge_hours: - return "charging" + # Discharge and idle hours are always schedule-driven, + # regardless of charge mode. if current_hour in discharge_hours: return "discharging" if current_hour in idle_hours: return "idle" + + # Charge hours: for non-custom modes, let the behavior engine + # decide (solar-gen tracks the solar curve, solar-excess waits + # for surplus). For custom mode, use the schedule directly. + charge_mode: str = self._battery_behavior.get("charge_mode", "custom") + if charge_mode != "custom" and self._behavior_engine is not None: + return self._behavior_engine.last_battery_direction + + charge_hours: list[int] = self._battery_behavior.get("charge_hours", []) + if current_hour in charge_hours: + return "charging" + return "idle" def _integrate_energy(self, current_time: float, power_w: float) -> None: From e0fddc5bcdbcf7c58b165b71137b7071b600e128 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:13:11 -0700 Subject: [PATCH 17/30] fix: BSEE schedule always authoritative, apply BESS to both Before and After modeling passes --- src/span_panel_simulator/bsee.py | 39 ++++-------- .../templates/partials/modeling_view.html | 4 +- src/span_panel_simulator/engine.py | 61 +++++++++++++------ 3 files changed, 54 insertions(+), 50 deletions(-) diff --git a/src/span_panel_simulator/bsee.py b/src/span_panel_simulator/bsee.py index b4e510c..47303ca 100644 --- a/src/span_panel_simulator/bsee.py +++ b/src/span_panel_simulator/bsee.py @@ -204,46 +204,29 @@ def software_version(self) -> str: # ------------------------------------------------------------------ def _resolve_battery_state(self, current_time: float) -> str: - """Determine battery state from grid status, charge mode, or schedule. - - Grid-forced-offline always overrides the schedule: the battery - must discharge to supply loads during an outage. - - For non-custom charge modes (solar-gen, solar-excess) the live - engine sets ``last_battery_direction`` each tick. However, during - the modeling pass the battery circuit may be replayed from the - recorder (skipping ``_apply_battery_behavior``), leaving the - direction stale at "idle". The schedule is always authoritative - for discharge/idle hours regardless of charge mode, so we consult - it first and only defer to the behavior engine for charge-hour - decisions where the mode matters. + """Determine battery state from grid status or schedule. + + The schedule (charge/discharge/idle hours) is always authoritative + for state resolution. The charge mode (solar-gen, solar-excess, + custom) affects power *magnitude* via the behavior engine's + ``_apply_battery_behavior``, not the state. This separation + ensures correct behavior in both the live simulation (where the + behavior engine is active) and the modeling pass (where the + battery circuit may be replayed from recorder data, leaving + ``last_battery_direction`` stale). """ if self._forced_offline: return "discharging" current_hour = datetime.fromtimestamp(current_time, tz=self._tz).hour + charge_hours: list[int] = self._battery_behavior.get("charge_hours", []) discharge_hours: list[int] = self._battery_behavior.get("discharge_hours", []) - idle_hours: list[int] = self._battery_behavior.get("idle_hours", []) - # Discharge and idle hours are always schedule-driven, - # regardless of charge mode. if current_hour in discharge_hours: return "discharging" - if current_hour in idle_hours: - return "idle" - - # Charge hours: for non-custom modes, let the behavior engine - # decide (solar-gen tracks the solar curve, solar-excess waits - # for surplus). For custom mode, use the schedule directly. - charge_mode: str = self._battery_behavior.get("charge_mode", "custom") - if charge_mode != "custom" and self._behavior_engine is not None: - return self._behavior_engine.last_battery_direction - - charge_hours: list[int] = self._battery_behavior.get("charge_hours", []) if current_hour in charge_hours: return "charging" - return "idle" def _integrate_energy(self, current_time: float, power_w: float) -> None: diff --git a/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html b/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html index 88f4adb..9c8aa9a 100644 --- a/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html +++ b/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html @@ -38,7 +38,7 @@

Modeling —

Before - (Site Power — no BESS) + (Grid Power — recorder baseline)
@@ -50,7 +50,7 @@

Modeling —

After - (Grid Power — with BESS) + (Grid Power — current config)
diff --git a/src/span_panel_simulator/engine.py b/src/span_panel_simulator/engine.py index 36c770e..30f5aa3 100644 --- a/src/span_panel_simulator/engine.py +++ b/src/span_panel_simulator/engine.py @@ -1572,21 +1572,29 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: ): solar_excess_ids.add(cid) - # Create temporary BSEE if battery is configured (After pass only) + # Create temporary BSEE instances for Before and After passes. + # Both passes apply the battery schedule so the charts reflect + # BESS operation in both the baseline recorder data and the + # current (possibly edited) configuration. cloned_bsee: BatteryStorageEquipment | None = None + cloned_bsee_before: BatteryStorageEquipment | None = None battery_circuit = self._find_battery_circuit() if self._bsee is not None and battery_circuit is not None: battery_cfg = battery_circuit.template.get("battery_behavior", {}) if isinstance(battery_cfg, dict): battery_dict: dict[str, Any] = dict(battery_cfg) - cloned_bsee = BatteryStorageEquipment( - battery_behavior=battery_dict, - panel_serial=self._config["panel_config"]["serial_number"], - feed_circuit_id=battery_circuit.circuit_id, - nameplate_capacity_kwh=self._bsee.nameplate_capacity_kwh, - behavior_engine=cloned_behavior, - panel_timezone=(cloned_behavior.panel_timezone if cloned_behavior else None), - ) + bsee_args: dict[str, Any] = { + "battery_behavior": battery_dict, + "panel_serial": self._config["panel_config"]["serial_number"], + "feed_circuit_id": battery_circuit.circuit_id, + "nameplate_capacity_kwh": self._bsee.nameplate_capacity_kwh, + "behavior_engine": cloned_behavior, + "panel_timezone": ( + cloned_behavior.panel_timezone if cloned_behavior else None + ), + } + cloned_bsee = BatteryStorageEquipment(**bsee_args) + cloned_bsee_before = BatteryStorageEquipment(**bsee_args) if cloned_behavior is None: return {"error": "Simulation not initialised"} @@ -1605,7 +1613,7 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: # passes so cycling / solar-excess bookkeeping does not cross-contaminate. modeling_checkpoint = cloned_behavior.capture_mutable_state() - powers_b, site_b, prod_b, _raw_b = self._aggregate_modeling_at_ts( + powers_b, site_b, prod_b, raw_batt_b = self._aggregate_modeling_at_ts( ts, cloned_behavior, solar_excess_ids, @@ -1621,23 +1629,36 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: modeling_recorder_baseline=False, ) - signed_battery = 0.0 + # Apply BSEE to Before pass (recorder baseline + battery schedule) + signed_battery_before = 0.0 + if cloned_bsee_before is not None: + cloned_bsee_before.update(ts, raw_batt_b) + state_b = cloned_bsee_before.battery_state + eff_b = cloned_bsee_before.battery_power_w + if state_b == "discharging": + signed_battery_before = -eff_b + elif state_b == "charging": + signed_battery_before = eff_b + + # Apply BSEE to After pass (current config + battery schedule) + signed_battery_after = 0.0 if cloned_bsee is not None: cloned_bsee.update(ts, raw_batt_a) - effective_power = cloned_bsee.battery_power_w - state = cloned_bsee.battery_state - if state == "discharging": - signed_battery = -effective_power - elif state == "charging": - signed_battery = effective_power + state_a = cloned_bsee.battery_state + eff_a = cloned_bsee.battery_power_w + if state_a == "discharging": + signed_battery_after = -eff_a + elif state_a == "charging": + signed_battery_after = eff_a - grid_after = site_a + signed_battery + grid_before = site_b + signed_battery_before + grid_after = site_a + signed_battery_after - site_power_arr.append(round(site_b, 1)) + site_power_arr.append(round(grid_before, 1)) pv_before_arr.append(round(prod_b, 1)) grid_power_arr.append(round(grid_after, 1)) pv_after_arr.append(round(prod_a, 1)) - battery_power_arr.append(round(signed_battery, 1)) + battery_power_arr.append(round(signed_battery_after, 1)) for cid in self._circuits: circuit_arrays_before[cid].append(round(powers_b.get(cid, 0.0), 1)) From 58ba5611ff57615e1d7eb855d7e9063fb1fef3ca Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:17:38 -0700 Subject: [PATCH 18/30] fix: add battery trace to Before chart with separate battery_power_before data --- .../dashboard/templates/partials/modeling_view.html | 11 +++++++---- src/span_panel_simulator/engine.py | 3 +++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html b/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html index 9c8aa9a..259f792 100644 --- a/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html +++ b/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html @@ -223,6 +223,7 @@

Modeling —

var pvBefore = (d.pv_power_before || d.pv_power || []).slice(visibleStart, visibleEnd + 1); var pvAfter = (d.pv_power_after || d.pv_power || []).slice(visibleStart, visibleEnd + 1); var batteryPower = d.battery_power.slice(visibleStart, visibleEnd + 1); + var batteryBefore = (d.battery_power_before || d.battery_power || []).slice(visibleStart, visibleEnd + 1); // Energy summaries — split import/export for distinct cost treatment var beforeEnergy = computeEnergy(sitePower, d.resolution_s); @@ -291,20 +292,22 @@

Modeling —

// Before chart datasets (recorder baseline); After (current SYN / edited circuits) var pvDsBefore = { label: 'Solar', data: pvBefore, borderColor: '#f59e0b', borderWidth: 1.5, pointRadius: 0, fill: { target: 'origin', above: 'rgba(245,158,11,0.08)' }, tension: 0.3 }; var pvDsAfter = { label: 'Solar', data: pvAfter, borderColor: '#f59e0b', borderWidth: 1.5, pointRadius: 0, fill: { target: 'origin', above: 'rgba(245,158,11,0.08)' }, tension: 0.3 }; - var battDs = { label: 'Battery', data: batteryPower, borderColor: '#10b981', borderWidth: 1.5, pointRadius: 0, fill: { target: 'origin', above: 'rgba(16,185,129,0.08)' }, tension: 0.3 }; + var battDsBefore = { label: 'Battery', data: batteryBefore, borderColor: '#10b981', borderWidth: 1.5, pointRadius: 0, fill: { target: 'origin', above: 'rgba(16,185,129,0.08)' }, tension: 0.3 }; + var battDsAfter = { label: 'Battery', data: batteryPower, borderColor: '#10b981', borderWidth: 1.5, pointRadius: 0, fill: { target: 'origin', above: 'rgba(16,185,129,0.08)' }, tension: 0.3 }; var beforeDatasets = [ - { label: 'Site Power', data: sitePower, borderColor: '#ef4444', borderWidth: 1.5, pointRadius: 0, fill: { target: 'origin', above: 'rgba(239,68,68,0.08)' }, tension: 0.3 }, + { label: 'Grid', data: sitePower, borderColor: '#ef4444', borderWidth: 1.5, pointRadius: 0, fill: { target: 'origin', above: 'rgba(239,68,68,0.08)' }, tension: 0.3 }, ]; if (pvCheckbox.checked) beforeDatasets.push(Object.assign({}, pvDsBefore)); + if (batteryCheckbox.checked) beforeDatasets.push(Object.assign({}, battDsBefore)); beforeDatasets = beforeDatasets.concat(overlayBefore); // After chart datasets var afterDatasets = [ - { label: 'Grid Power', data: gridPower, borderColor: '#ef4444', borderWidth: 1.5, pointRadius: 0, fill: { target: 'origin', above: 'rgba(239,68,68,0.08)' }, tension: 0.3 }, + { label: 'Grid', data: gridPower, borderColor: '#ef4444', borderWidth: 1.5, pointRadius: 0, fill: { target: 'origin', above: 'rgba(239,68,68,0.08)' }, tension: 0.3 }, ]; if (pvCheckbox.checked) afterDatasets.push(Object.assign({}, pvDsAfter)); - if (batteryCheckbox.checked) afterDatasets.push(Object.assign({}, battDs)); + if (batteryCheckbox.checked) afterDatasets.push(Object.assign({}, battDsAfter)); afterDatasets = afterDatasets.concat(overlayAfter); // Tick configuration diff --git a/src/span_panel_simulator/engine.py b/src/span_panel_simulator/engine.py index 30f5aa3..358592f 100644 --- a/src/span_panel_simulator/engine.py +++ b/src/span_panel_simulator/engine.py @@ -1605,6 +1605,7 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: pv_before_arr: list[float] = [] pv_after_arr: list[float] = [] battery_power_arr: list[float] = [] + battery_before_arr: list[float] = [] circuit_arrays_before: dict[str, list[float]] = {cid: [] for cid in self._circuits} circuit_arrays_after: dict[str, list[float]] = {cid: [] for cid in self._circuits} @@ -1659,6 +1660,7 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: grid_power_arr.append(round(grid_after, 1)) pv_after_arr.append(round(prod_a, 1)) battery_power_arr.append(round(signed_battery_after, 1)) + battery_before_arr.append(round(signed_battery_before, 1)) for cid in self._circuits: circuit_arrays_before[cid].append(round(powers_b.get(cid, 0.0), 1)) @@ -1688,6 +1690,7 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: # Legacy alias — same series as ``pv_power_after`` (current / SYN view). "pv_power": pv_after_arr, "battery_power": battery_power_arr, + "battery_power_before": battery_before_arr, "circuits": circuits_response, } From 46fbf8f335d2a5786f0a290bffdf230d10d846da Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:22:22 -0700 Subject: [PATCH 19/30] fix: Before chart only includes BESS when battery was in original recorder baseline --- src/span_panel_simulator/engine.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/span_panel_simulator/engine.py b/src/span_panel_simulator/engine.py index 358592f..b618565 100644 --- a/src/span_panel_simulator/engine.py +++ b/src/span_panel_simulator/engine.py @@ -1572,10 +1572,13 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: ): solar_excess_ids.add(cid) - # Create temporary BSEE instances for Before and After passes. - # Both passes apply the battery schedule so the charts reflect - # BESS operation in both the baseline recorder data and the - # current (possibly edited) configuration. + # Create temporary BSEE instances for modeling passes. + # After always gets BSEE (current config). + # Before only gets BSEE if the battery existed in the recorder + # baseline (not user-added). A user-added battery has + # user_modified=True and no recorder data — the Before chart + # should show life *without* the battery so the user sees the + # impact of adding one. cloned_bsee: BatteryStorageEquipment | None = None cloned_bsee_before: BatteryStorageEquipment | None = None battery_circuit = self._find_battery_circuit() @@ -1594,7 +1597,14 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: ), } cloned_bsee = BatteryStorageEquipment(**bsee_args) - cloned_bsee_before = BatteryStorageEquipment(**bsee_args) + + # Only apply BESS to Before if the battery was part of + # the original config (has recorder data, not user-added). + battery_is_original = not battery_circuit.template.get( + "user_modified", False + ) and bool(battery_circuit.template.get("recorder_entity")) + if battery_is_original: + cloned_bsee_before = BatteryStorageEquipment(**bsee_args) if cloned_behavior is None: return {"error": "Simulation not initialised"} From cde98977e9370cf8b7c09689eb4cfb2edca9d561 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:25:28 -0700 Subject: [PATCH 20/30] fix: Before pass uses recorder battery data directly, no BSEE special-casing --- src/span_panel_simulator/engine.py | 58 ++++++++++-------------------- 1 file changed, 18 insertions(+), 40 deletions(-) diff --git a/src/span_panel_simulator/engine.py b/src/span_panel_simulator/engine.py index b618565..96fa2b7 100644 --- a/src/span_panel_simulator/engine.py +++ b/src/span_panel_simulator/engine.py @@ -1572,39 +1572,23 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: ): solar_excess_ids.add(cid) - # Create temporary BSEE instances for modeling passes. - # After always gets BSEE (current config). - # Before only gets BSEE if the battery existed in the recorder - # baseline (not user-added). A user-added battery has - # user_modified=True and no recorder data — the Before chart - # should show life *without* the battery so the user sees the - # impact of adding one. + # BSEE for the After pass (applies current config). + # The Before pass needs no BSEE — the recorder already contains + # the battery's charge/discharge power with correct sign + # (negative = charging, positive = discharging). cloned_bsee: BatteryStorageEquipment | None = None - cloned_bsee_before: BatteryStorageEquipment | None = None battery_circuit = self._find_battery_circuit() if self._bsee is not None and battery_circuit is not None: battery_cfg = battery_circuit.template.get("battery_behavior", {}) if isinstance(battery_cfg, dict): - battery_dict: dict[str, Any] = dict(battery_cfg) - bsee_args: dict[str, Any] = { - "battery_behavior": battery_dict, - "panel_serial": self._config["panel_config"]["serial_number"], - "feed_circuit_id": battery_circuit.circuit_id, - "nameplate_capacity_kwh": self._bsee.nameplate_capacity_kwh, - "behavior_engine": cloned_behavior, - "panel_timezone": ( - cloned_behavior.panel_timezone if cloned_behavior else None - ), - } - cloned_bsee = BatteryStorageEquipment(**bsee_args) - - # Only apply BESS to Before if the battery was part of - # the original config (has recorder data, not user-added). - battery_is_original = not battery_circuit.template.get( - "user_modified", False - ) and bool(battery_circuit.template.get("recorder_entity")) - if battery_is_original: - cloned_bsee_before = BatteryStorageEquipment(**bsee_args) + cloned_bsee = BatteryStorageEquipment( + battery_behavior=dict(battery_cfg), + panel_serial=self._config["panel_config"]["serial_number"], + feed_circuit_id=battery_circuit.circuit_id, + nameplate_capacity_kwh=self._bsee.nameplate_capacity_kwh, + behavior_engine=cloned_behavior, + panel_timezone=(cloned_behavior.panel_timezone if cloned_behavior else None), + ) if cloned_behavior is None: return {"error": "Simulation not initialised"} @@ -1640,18 +1624,12 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: modeling_recorder_baseline=False, ) - # Apply BSEE to Before pass (recorder baseline + battery schedule) - signed_battery_before = 0.0 - if cloned_bsee_before is not None: - cloned_bsee_before.update(ts, raw_batt_b) - state_b = cloned_bsee_before.battery_state - eff_b = cloned_bsee_before.battery_power_w - if state_b == "discharging": - signed_battery_before = -eff_b - elif state_b == "charging": - signed_battery_before = eff_b - - # Apply BSEE to After pass (current config + battery schedule) + # Before: recorder data already has correct battery sign + # (negative = charging, positive = discharging). Invert to + # match the grid convention (discharge reduces grid import). + signed_battery_before = -raw_batt_b + + # After: BSEE applies current config (user edits, new battery) signed_battery_after = 0.0 if cloned_bsee is not None: cloned_bsee.update(ts, raw_batt_a) From fe3bfeb600f0f4893c089027922b54704f1ee0f7 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:31:05 -0700 Subject: [PATCH 21/30] =?UTF-8?q?fix:=20After=20pass=20battery=20sign=20ma?= =?UTF-8?q?tches=20Before=20convention=20=E2=80=94=20negate=20raw=20power?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/span_panel_simulator/engine.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/span_panel_simulator/engine.py b/src/span_panel_simulator/engine.py index 96fa2b7..fc2c5d4 100644 --- a/src/span_panel_simulator/engine.py +++ b/src/span_panel_simulator/engine.py @@ -1629,16 +1629,15 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: # match the grid convention (discharge reduces grid import). signed_battery_before = -raw_batt_b - # After: BSEE applies current config (user edits, new battery) + # After: BSEE applies current config (SOE tracking, user edits). + # Same sign convention as Before: negate raw power so that + # discharge (positive raw) reduces grid and charge (negative + # raw) increases grid. BSEE may clamp power to 0 when SOE + # bounds are reached. signed_battery_after = 0.0 if cloned_bsee is not None: cloned_bsee.update(ts, raw_batt_a) - state_a = cloned_bsee.battery_state - eff_a = cloned_bsee.battery_power_w - if state_a == "discharging": - signed_battery_after = -eff_a - elif state_a == "charging": - signed_battery_after = eff_a + signed_battery_after = -cloned_bsee.battery_power_w grid_before = site_b + signed_battery_before grid_after = site_a + signed_battery_after From 353f6fc5eb0b5f4049ad84c598578d22146049ff Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:51:51 -0700 Subject: [PATCH 22/30] fix: After pass skips BSEE when battery unchanged, fix SOE sign handling When no user modifications exist on the battery template, the After modeling pass now uses recorder data directly (matching Before exactly) instead of routing through BSEE where SOE depletion corrupted the trace. Also fix _integrate_energy to use abs(power_w) so SOE tracking handles both recorder-signed and synthetic-positive power correctly when BSEE is active after user modification. --- src/span_panel_simulator/bsee.py | 14 ++++++++++---- src/span_panel_simulator/engine.py | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/span_panel_simulator/bsee.py b/src/span_panel_simulator/bsee.py index 47303ca..c409e31 100644 --- a/src/span_panel_simulator/bsee.py +++ b/src/span_panel_simulator/bsee.py @@ -246,13 +246,19 @@ def _integrate_energy(self, current_time: float, power_w: float) -> None: delta_s = min(delta_s, _MAX_INTEGRATION_DELTA_S) delta_hours = delta_s / 3600.0 - if self._battery_state == "charging" and power_w > 0: - energy_kwh = (power_w / 1000.0) * delta_hours * self._charge_efficiency + # Use abs(power_w) so integration works regardless of sign + # convention. Recorder data is signed (negative = charging, + # positive = discharging) while synthetic power is always positive. + # The battery_state already tells us the direction; magnitude is + # all that matters for energy bookkeeping. + mag = abs(power_w) + if self._battery_state == "charging" and mag > 0: + energy_kwh = (mag / 1000.0) * delta_hours * self._charge_efficiency self._soe_kwh += energy_kwh - elif self._battery_state == "discharging" and power_w > 0: + elif self._battery_state == "discharging" and mag > 0: # Discharge: power delivered = stored energy * discharge_efficiency # So stored energy consumed = power / efficiency - energy_kwh = (power_w / 1000.0) * delta_hours / self._discharge_efficiency + energy_kwh = (mag / 1000.0) * delta_hours / self._discharge_efficiency self._soe_kwh -= energy_kwh # Clamp to bounds — use backup reserve for normal discharge, diff --git a/src/span_panel_simulator/engine.py b/src/span_panel_simulator/engine.py index fc2c5d4..13c48b2 100644 --- a/src/span_panel_simulator/engine.py +++ b/src/span_panel_simulator/engine.py @@ -1636,8 +1636,18 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: # bounds are reached. signed_battery_after = 0.0 if cloned_bsee is not None: - cloned_bsee.update(ts, raw_batt_a) - signed_battery_after = -cloned_bsee.battery_power_w + battery_user_modified = ( + battery_circuit is not None and battery_circuit.template.get("user_modified") + ) + if battery_user_modified: + # User changed the battery config — let BSEE apply the + # new schedule with SOE tracking. + cloned_bsee.update(ts, raw_batt_a) + signed_battery_after = -cloned_bsee.battery_power_w + else: + # Battery unchanged — recorder data is authoritative. + # Skip BSEE so After matches Before exactly. + signed_battery_after = -raw_batt_a grid_before = site_b + signed_battery_before grid_after = site_a + signed_battery_after From ff0f8d7e5e19fff03f08d376c4ce7342f815e795 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:57:19 -0700 Subject: [PATCH 23/30] =?UTF-8?q?fix:=20SYN=E2=86=92REC=20toggle=20works?= =?UTF-8?q?=20for=20template-cloned=20configs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit restore_recorder required a panel_source.recorder_map entry to find the recorder_entity — template clones have no panel_source so the restore always failed silently. Fall back to the template's own recorder_entity so clicking SYN correctly clears user_modified and re-enables recorder replay for any circuit. --- src/span_panel_simulator/dashboard/config_store.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/span_panel_simulator/dashboard/config_store.py b/src/span_panel_simulator/dashboard/config_store.py index 5ad68ee..fec0c5b 100644 --- a/src/span_panel_simulator/dashboard/config_store.py +++ b/src/span_panel_simulator/dashboard/config_store.py @@ -215,6 +215,10 @@ def restore_recorder(self, entity_id: str) -> bool: recorder_map = self.get_recorder_map() rec_entity = recorder_map.get(template_name) + # Fall back to the template's own recorder_entity — covers + # template-cloned configs that have no panel_source/recorder_map. + if not rec_entity: + rec_entity = templates[template_name].get("recorder_entity") if not rec_entity: return False From 43f9abba0450fd473acb7c547393370b4ad4631b Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:03:00 -0700 Subject: [PATCH 24/30] fix: BSEE always enforces schedule in After pass, zeros idle hours MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the skip-BSEE-when-not-user_modified approach — the After chart must always apply the BSEE schedule so users see the effect of their battery configuration. BSEE now zeros battery power during idle hours so the schedule is authoritative for all three states (charge, discharge, idle), not just when hitting SOE bounds. --- src/span_panel_simulator/bsee.py | 4 ++++ src/span_panel_simulator/engine.py | 14 ++------------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/span_panel_simulator/bsee.py b/src/span_panel_simulator/bsee.py index c409e31..d330d11 100644 --- a/src/span_panel_simulator/bsee.py +++ b/src/span_panel_simulator/bsee.py @@ -85,6 +85,10 @@ def update(self, current_time: float, battery_power_w: float) -> None: """ self._battery_state = self._resolve_battery_state(current_time) + # Enforce schedule: battery does nothing during idle hours + if self._battery_state == "idle": + battery_power_w = 0.0 + # Enforce SOE bounds — stop discharge at reserve, stop charge at max effective_min_pct = _SOE_HARD_MIN_PCT if self._forced_offline else self._backup_reserve_pct if (self._battery_state == "discharging" and self.soe_percentage <= effective_min_pct) or ( diff --git a/src/span_panel_simulator/engine.py b/src/span_panel_simulator/engine.py index 13c48b2..fc2c5d4 100644 --- a/src/span_panel_simulator/engine.py +++ b/src/span_panel_simulator/engine.py @@ -1636,18 +1636,8 @@ async def compute_modeling_data(self, horizon_hours: int) -> dict[str, Any]: # bounds are reached. signed_battery_after = 0.0 if cloned_bsee is not None: - battery_user_modified = ( - battery_circuit is not None and battery_circuit.template.get("user_modified") - ) - if battery_user_modified: - # User changed the battery config — let BSEE apply the - # new schedule with SOE tracking. - cloned_bsee.update(ts, raw_batt_a) - signed_battery_after = -cloned_bsee.battery_power_w - else: - # Battery unchanged — recorder data is authoritative. - # Skip BSEE so After matches Before exactly. - signed_battery_after = -raw_batt_a + cloned_bsee.update(ts, raw_batt_a) + signed_battery_after = -cloned_bsee.battery_power_w grid_before = site_b + signed_battery_before grid_after = site_a + signed_battery_after From cf00ca65f779ea498cad97183cb8028b517297d5 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:11:15 -0700 Subject: [PATCH 25/30] fix: delete companion history DB when a clone config is removed --- src/span_panel_simulator/dashboard/routes.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/span_panel_simulator/dashboard/routes.py b/src/span_panel_simulator/dashboard/routes.py index e351d67..2aaccff 100644 --- a/src/span_panel_simulator/dashboard/routes.py +++ b/src/span_panel_simulator/dashboard/routes.py @@ -1402,6 +1402,13 @@ async def handle_delete_config(request: web.Request) -> web.Response: await _purge_recorder_for_config(ctx, config_path) config_path.unlink() + + # Remove companion history DB if present + history_db = config_path.with_name(config_path.stem + "_history.db") + if history_db.exists(): + history_db.unlink() + _LOGGER.info("Deleted history DB %s", history_db.name) + _LOGGER.info("Deleted config %s", filename) # If we just deleted the active editor file, fall back to viewing From 8b475d5679c5eec9cd695780b3543f1e9982ef10 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:36:08 -0700 Subject: [PATCH 26/30] fix: row buttons auto-switch active config so entity list stays in sync Clone, Start, Stop, and Restart handlers now load the acted-on config into the editor and set config_filter before redirecting. Previously, cloning a template left the editor on the old config, so the entity list and modeling view showed the wrong panel's circuits. --- src/span_panel_simulator/dashboard/routes.py | 44 ++++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/span_panel_simulator/dashboard/routes.py b/src/span_panel_simulator/dashboard/routes.py index 2aaccff..e6fbdef 100644 --- a/src/span_panel_simulator/dashboard/routes.py +++ b/src/span_panel_simulator/dashboard/routes.py @@ -1110,10 +1110,12 @@ async def handle_clone(request: web.Request) -> web.Response: exc_info=True, ) - return web.Response( - text=f'
Cloned to {filename}
', - content_type="text/html", - ) + # Auto-switch the editor to the newly cloned config so the entity + # list, runtime controls, and modeling view all reflect the clone. + _store(request).load_from_file(output_path) + ctx.config_filter = filename + + return web.Response(status=200, headers={"HX-Redirect": "./"}) async def handle_save_reload(request: web.Request) -> web.Response: @@ -1242,12 +1244,12 @@ async def handle_start_panel(request: web.Request) -> web.Response: filename, err = await _read_panel_filename(request) if err is not None: return err - _ctx(request).start_panel(filename) - return web.Response( - text=f'
Starting {filename}…
', - content_type="text/html", - headers={"HX-Trigger": "refreshPanels"}, - ) + ctx = _ctx(request) + ctx.start_panel(filename) + # Auto-switch the editor to this panel so entity list stays in sync. + _store(request).load_from_file(ctx.config_dir / filename) + ctx.config_filter = filename + return web.Response(status=200, headers={"HX-Redirect": "./"}) async def handle_stop_panel(request: web.Request) -> web.Response: @@ -1255,12 +1257,11 @@ async def handle_stop_panel(request: web.Request) -> web.Response: filename, err = await _read_panel_filename(request) if err is not None: return err - _ctx(request).stop_panel(filename) - return web.Response( - text=f'
Stopping {filename}…
', - content_type="text/html", - headers={"HX-Trigger": "refreshPanels"}, - ) + ctx = _ctx(request) + ctx.stop_panel(filename) + _store(request).load_from_file(ctx.config_dir / filename) + ctx.config_filter = filename + return web.Response(status=200, headers={"HX-Redirect": "./"}) async def handle_restart_panel(request: web.Request) -> web.Response: @@ -1268,12 +1269,11 @@ async def handle_restart_panel(request: web.Request) -> web.Response: filename, err = await _read_panel_filename(request) if err is not None: return err - _ctx(request).restart_panel(filename) - return web.Response( - text=f'
Restarting {filename}…
', - content_type="text/html", - headers={"HX-Trigger": "refreshPanels"}, - ) + ctx = _ctx(request) + ctx.restart_panel(filename) + _store(request).load_from_file(ctx.config_dir / filename) + ctx.config_filter = filename + return web.Response(status=200, headers={"HX-Redirect": "./"}) async def _purge_recorder_for_config( From 905f2e010228e6bf7e360e4834c73a3ed51e75bb Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:40:49 -0700 Subject: [PATCH 27/30] add CI for linting --- .github/workflows/lint.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/lint.yaml diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000..036369f --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,28 @@ +name: Lint & Type Check + +on: + pull_request: + branches: [main] + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install pre-commit + run: pip install pre-commit + + - name: Cache pre-commit hooks + uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} + + - name: Run pre-commit on all files + run: pre-commit run --all-files From 300aff4f9a44cfc3b2e647f1be2072684f3d9ef9 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:43:23 -0700 Subject: [PATCH 28/30] update changelog --- span_panel_simulator/CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/span_panel_simulator/CHANGELOG.md b/span_panel_simulator/CHANGELOG.md index 20e539a..87c0818 100644 --- a/span_panel_simulator/CHANGELOG.md +++ b/span_panel_simulator/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 1.0.6 — 2026-03-26 + +### Features + +- Synthetic history: cloned panels generate a companion SQLite database so modeling works without Home Assistant +- Standalone CLI for synthetic history generation + +### Fixes + +- Row buttons in dashboard now auto-switch the active config so the entity list stays in sync +- Delete companion history database when a cloned config is removed +- Battery schedule (BSEE) always enforces discharge/idle hours in the After modeling pass +- SYN→REC toggle now works for template-cloned configs +- After modeling pass skips BSEE when battery is unchanged; fix SOE sign handling +- Before/After chart battery sign conventions are now consistent +- Before chart only includes BESS when battery was present in the original recorder baseline +- Fall through to SQLite when HA returns no recorder data +- Derive recorder entity mappings for configs without HA; generate history on simple clone + ## 1.0.5 — 2026-03-23 ### Features From 8caad522ec777ff14a3b78698537442616f24ec9 Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:47:35 -0700 Subject: [PATCH 29/30] =?UTF-8?q?fix:=20address=20copilot=20review=20?= =?UTF-8?q?=E2=80=94=20derived=20count,=20XSS=20sink,=20deterministic=20se?= =?UTF-8?q?ed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix log line that always counted len(circuits_to_generate) instead of actual derived mappings — use an integer counter instead of bool guard. - Replace innerHTML XSS sink with textContent when rendering fetch errors in the modeling view. - Replace Python's salted hash() with hashlib.sha256 for the weather factor seed so synthetic history is deterministic across processes. Precompute the seed per-circuit in _generate_rows to avoid recomputing SHA-256 on every timestamp. --- .../templates/partials/modeling_view.html | 7 ++++++- src/span_panel_simulator/history_generator.py | 18 +++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html b/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html index 259f792..edc1b49 100644 --- a/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html +++ b/src/span_panel_simulator/dashboard/templates/partials/modeling_view.html @@ -175,7 +175,12 @@

Modeling —

.catch(function(err) { spinner.style.display = 'none'; console.error('Modeling error:', err); - chartsWrap.innerHTML = '

Error loading data: ' + err.message + '

'; + var errP = document.createElement('p'); + errP.className = 'text-muted'; + errP.style.cssText = 'text-align:center;padding:2rem'; + errP.textContent = 'Error loading data: ' + err.message; + chartsWrap.innerHTML = ''; + chartsWrap.appendChild(errP); chartsWrap.style.display = ''; }); } diff --git a/src/span_panel_simulator/history_generator.py b/src/span_panel_simulator/history_generator.py index 3879539..88227d3 100644 --- a/src/span_panel_simulator/history_generator.py +++ b/src/span_panel_simulator/history_generator.py @@ -151,7 +151,7 @@ async def generate( tmpl_to_circuits.setdefault(tname, []).append(cid) circuits_to_generate: list[tuple[str, str, dict[str, object]]] = [] - derived_entities = False + derived_count = 0 for tmpl_name, tmpl in templates.items(): if not isinstance(tmpl, dict): continue @@ -170,9 +170,9 @@ async def generate( derived_entity = f"sensor.{clean_serial}_{cid}_power" tmpl["recorder_entity"] = derived_entity circuits_to_generate.append((tmpl_name, derived_entity, tmpl)) - derived_entities = True + derived_count += 1 - if derived_entities: + if derived_count: # Write updated config with recorder_entity mappings config_path.write_text( yaml.dump(raw, default_flow_style=False, sort_keys=False), @@ -180,7 +180,7 @@ async def generate( ) _LOGGER.info( "Derived %d recorder_entity mappings for %s", - sum(1 for _ in circuits_to_generate if derived_entities), + derived_count, config_path.name, ) @@ -354,6 +354,10 @@ def _generate_rows( # Mean of monthly factors for normalisation mean_mf = sum(monthly_factors.values()) / len(monthly_factors) if monthly_factors else 1.0 + # Precompute deterministic seed from serial for weather factor + serial_bytes = str(serial).encode("utf-8") + serial_seed = int.from_bytes(hashlib.sha256(serial_bytes).digest()[:8], "big") + batch: list[tuple[object, ...]] = [] ts = start_ts while ts < end_ts: @@ -365,7 +369,7 @@ def _generate_rows( lat=lat, lon=lon, tz=tz, - serial=serial, + serial_seed=serial_seed, hour_factors=hour_factors, mean_hf=mean_hf, tod_enabled=tod_enabled, @@ -423,7 +427,7 @@ def _compute_power_at( lat: float, lon: float, tz: ZoneInfo, - serial: str, + serial_seed: int, hour_factors: dict[int, float], mean_hf: float, tod_enabled: bool, @@ -478,7 +482,7 @@ def _compute_power_at( if mode == "producer": scale = abs(nameplate) if nameplate is not None and nameplate > 0 else abs(base) solar = solar_production_factor(ts, lat, lon) - weather = daily_weather_factor(ts, seed=hash(serial), monthly_factors=weather_monthly) + weather = daily_weather_factor(ts, seed=serial_seed, monthly_factors=weather_monthly) return scale * solar * weather # Time-of-day for consumers From 3e3cad4347c4965b6ad5748b6ce156ac1684192e Mon Sep 17 00:00:00 2001 From: cayossarian <23534755+cayossarian@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:50:51 -0700 Subject: [PATCH 30/30] update changelog --- span_panel_simulator/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/span_panel_simulator/CHANGELOG.md b/span_panel_simulator/CHANGELOG.md index 87c0818..06ec88d 100644 --- a/span_panel_simulator/CHANGELOG.md +++ b/span_panel_simulator/CHANGELOG.md @@ -18,6 +18,7 @@ - Before chart only includes BESS when battery was present in the original recorder baseline - Fall through to SQLite when HA returns no recorder data - Derive recorder entity mappings for configs without HA; generate history on simple clone +- Synthetic history now produces identical results across runs for the same config (deterministic seed) ## 1.0.5 — 2026-03-23