From a0cc124a6cb252fade3348995816dbe497b0209e Mon Sep 17 00:00:00 2001 From: MarcoMC Date: Sat, 12 Sep 2026 18:03:32 +0200 Subject: [PATCH 01/10] feat(reads): add complete scoped snapshots for 0.3.0 Add structured record and relationship completeness, metadata-derived schema aliases, consistent scoped reloads, and safe model construction. Cover malformed layouts and supported Python versions, and enable the release branch CI checks. --- .github/workflows/ci.yaml | 4 +- README.md | 39 +- src/moneywiz_api/__init__.py | 23 +- src/moneywiz_api/database_accessor.py | 381 +++++++++++++++--- src/moneywiz_api/managers/account_manager.py | 4 + src/moneywiz_api/managers/record_manager.py | 131 ++++-- .../managers/transaction_manager.py | 33 +- src/moneywiz_api/model/account.py | 26 +- src/moneywiz_api/model/investment_holding.py | 14 +- src/moneywiz_api/model/transaction.py | 16 +- src/moneywiz_api/moneywiz_api.py | 117 +++++- src/moneywiz_api/read_result.py | 217 ++++++++++ src/moneywiz_api/schema_profile.py | 53 +-- src/moneywiz_api/utils.py | 10 +- .../test_read_completeness_integration.py | 10 + tests/unit/test_account_diagnostics.py | 130 ++++++ tests/unit/test_api_snapshot_generation.py | 83 ++++ tests/unit/test_database_accessor.py | 100 +++++ tests/unit/test_profiled_investment_values.py | 28 +- tests/unit/test_read_completeness.py | 198 +++++++++ tests/unit/test_relationship_completeness.py | 148 +++++++ tests/unit/test_schema_profiles.py | 35 +- tests/unit/test_utils.py | 26 ++ 23 files changed, 1657 insertions(+), 169 deletions(-) create mode 100644 src/moneywiz_api/read_result.py create mode 100644 tests/integration/test_read_completeness_integration.py create mode 100644 tests/unit/test_account_diagnostics.py create mode 100644 tests/unit/test_api_snapshot_generation.py create mode 100644 tests/unit/test_database_accessor.py create mode 100644 tests/unit/test_read_completeness.py create mode 100644 tests/unit/test_relationship_completeness.py create mode 100644 tests/unit/test_utils.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5f49171..49e549c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,9 +4,9 @@ permissions: contents: read on: push: - branches: ["main"] + branches: ["main", "release/0.3.0"] pull_request: - branches: ["main"] + branches: ["main", "release/0.3.0"] jobs: build: diff --git a/README.md b/README.md index 7874bcf..31d5b36 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,12 @@ Support the original author on A Python API to access MoneyWiz Sqlite database. +## Table of Contents + +- [Get Started](#get-started) +- [Bounded reads](#bounded-reads) +- [Contribution](#contribution) + ## Get Started ```bash @@ -15,7 +21,6 @@ pip install moneywiz-api ``` ```python - from moneywiz_api import MoneywizApi moneywizApi = MoneywizApi("") @@ -38,7 +43,6 @@ moneywizApi = MoneywizApi("") record = accessor.get_record(record_id) print(record) - ``` It also offers a interactive shell `moneywiz-cli`. @@ -58,6 +62,37 @@ Integration tests are opt-in and never read a CLI default database path. Point MONEYWIZ_TEST_DB_PATH=/absolute/path/to/test.sqlite uv run pytest tests ``` +## Bounded reads + +The default remains an eager load of every manager. Use an explicit manager list +to isolate a read from unrelated malformed records: + +```python +from pathlib import Path + +from moneywiz_api import MoneywizApi + +with MoneywizApi( + Path("/absolute/path/to/moneywiz.sqlite"), + managers=("accounts", "transactions"), +) as api: + completeness = api.completeness().as_dict() + snapshot = api.snapshot().as_dict() +``` + +Supported manager names are `accounts`, `payees`, `categories`, `transactions`, +`investment_holdings`, and `tags`. `completeness` contains ordered source and +parsed IDs plus identity-only diagnostics for every skipped row. Transaction +completeness also reports category, refund, and tag relationship storage as +`present`, `absent`, or `unknown`; unknown or malformed storage makes the read +partial. `snapshot` contains JSON-safe records, completeness, and the selected +schema profile. + +Database paths must identify existing files. Connections are opened read-only, +and each multi-manager load uses one consistent SQLite read transaction. A +later scoped load atomically refreshes the union of requested and already loaded +managers so one snapshot never mixes database generations. + ## Contribution This project is in very early stage, all contributions are welcomed! diff --git a/src/moneywiz_api/__init__.py b/src/moneywiz_api/__init__.py index 45e4455..28b35d9 100644 --- a/src/moneywiz_api/__init__.py +++ b/src/moneywiz_api/__init__.py @@ -1,3 +1,24 @@ +from moneywiz_api.database_accessor import DatabasePathError, DatabaseSchemaError from moneywiz_api.moneywiz_api import MoneywizApi +from moneywiz_api.read_result import ( + ApiCompleteness, + LoadErrorKind, + ManagerLoadReport, + RelationshipLoadReport, + RelationshipStorage, + ReadSnapshot, + SkippedRecord, +) -__all__ = ["MoneywizApi"] +__all__ = [ + "ApiCompleteness", + "DatabasePathError", + "DatabaseSchemaError", + "LoadErrorKind", + "ManagerLoadReport", + "MoneywizApi", + "ReadSnapshot", + "RelationshipLoadReport", + "RelationshipStorage", + "SkippedRecord", +] diff --git a/src/moneywiz_api/database_accessor.py b/src/moneywiz_api/database_accessor.py index 0125ae2..e3ab502 100644 --- a/src/moneywiz_api/database_accessor.py +++ b/src/moneywiz_api/database_accessor.py @@ -1,5 +1,6 @@ import sqlite3 from collections import defaultdict +from contextlib import contextmanager from pathlib import Path from typing import Dict, List, Any, Callable, Tuple from decimal import Decimal @@ -11,13 +12,39 @@ InvestmentSellTransaction, ) from moneywiz_api.model.raw_data_handler import RawDataHandler as RDH +from moneywiz_api.read_result import ( + LoadErrorKind, + RelationshipLoadReport, + RelationshipStorage, + SkippedRecord, +) from moneywiz_api.schema_profile import SchemaProfile, detect_schema_profile from moneywiz_api.types import ENT_ID, ID, GID +class DatabasePathError(ValueError): + """Raised when an explicit database path cannot be opened safely.""" + + +class DatabaseSchemaError(ValueError): + """Raised when a readable SQLite file is not a supported MoneyWiz store.""" + + class DatabaseAccessor: def __init__(self, db_path: Path): - self._con = sqlite3.connect(db_path, uri=True) + self._db_path = Path(db_path).expanduser().resolve() + if not self._db_path.is_file(): + raise DatabasePathError("database path must identify an existing file") + + try: + self._con = sqlite3.connect( + f"{self._db_path.as_uri()}?mode=ro", + uri=True, + ) + except sqlite3.Error as exc: + raise DatabasePathError( + "database file could not be opened read-only" + ) from exc def dict_factory(cursor, row): record = {} @@ -25,25 +52,58 @@ def dict_factory(cursor, row): record[col[0]] = row[idx] return record - self._con.row_factory = dict_factory - self._schema_profile = detect_schema_profile(self._con) - - self._ent_to_typename: Dict[ENT_ID, str] = self._load_primarykey() + try: + self._con.row_factory = dict_factory + if not self._table_exists("Z_PRIMARYKEY") or not self._table_exists( + "ZSYNCOBJECT" + ): + raise DatabaseSchemaError( + "database is missing required MoneyWiz schema tables" + ) + required_metadata_columns = {"Z_ENT", "Z_NAME", "Z_SUPER"} + if not required_metadata_columns.issubset( + self._table_columns("Z_PRIMARYKEY") + ): + raise DatabaseSchemaError( + "Z_PRIMARYKEY is missing required entity metadata columns" + ) + self._schema_profile = detect_schema_profile(self._con) + self._ent_to_typename, self._ent_to_super = self._load_primarykey() + except Exception as exc: + self._con.close() + if isinstance(exc, DatabaseSchemaError): + raise + raise DatabaseSchemaError("database schema could not be read") from exc self._typename_to_ent: Dict[str, ENT_ID] = { v: k for k, v in self._ent_to_typename.items() } - def _load_primarykey(self) -> Dict[int, str]: + def _load_primarykey(self) -> tuple[Dict[int, str], Dict[int, int]]: cur = self._con.cursor() res = cur.execute( """ - SELECT * FROM "Z_PRIMARYKEY" ORDER BY "Z_ENT" LIMIT 1000 OFFSET 0; + SELECT Z_ENT, Z_NAME, Z_SUPER + FROM "Z_PRIMARYKEY" + ORDER BY Z_ENT + LIMIT 1000 OFFSET 0; """ ) ent_to_typename: Dict[int, str] = {} + ent_to_super: Dict[int, int] = {} for row in res.fetchall(): - ent_to_typename[row["Z_ENT"]] = row["Z_NAME"] - return ent_to_typename + ent_id = row["Z_ENT"] + typename = row["Z_NAME"] + super_id = row["Z_SUPER"] + if ( + not isinstance(ent_id, int) + or not isinstance(typename, str) + or not typename + or not isinstance(super_id, int) + ): + raise DatabaseSchemaError("Z_PRIMARYKEY contains invalid metadata") + ent_to_typename[ent_id] = typename + ent_to_super[ent_id] = super_id + return ent_to_typename, ent_to_super def __repr__(self): return "\n".join( @@ -61,6 +121,25 @@ def typename_for(self, ent_id: ENT_ID) -> str: def ent_for(self, typename: str) -> ENT_ID: return self._typename_to_ent.get(typename) + def descendant_typenames(self, roots: tuple[str, ...]) -> list[str]: + """Return physical entity names descended from the requested roots.""" + root_ids = { + ent_id + for root in roots + if (ent_id := self._typename_to_ent.get(root)) is not None + } + descendants: list[str] = [] + for ent_id, typename in self._ent_to_typename.items(): + current = ent_id + visited: set[int] = set() + while current and current not in visited: + if current in root_ids: + descendants.append(typename) + break + visited.add(current) + current = self._ent_to_super.get(current, 0) + return descendants + def _table_exists(self, table_name: str) -> bool: row = self._con.execute( "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", @@ -68,17 +147,76 @@ def _table_exists(self, table_name: str) -> bool: ).fetchone() return row is not None + def _table_columns(self, table_name: str) -> set[str]: + return { + str(row["name"]) + for row in self._con.execute( + f'PRAGMA table_info("{table_name}")' + ).fetchall() + } + + def _relationship_storage( + self, + *, + entity_names: tuple[str, ...], + table_name: str, + required_columns: tuple[str, ...], + ) -> RelationshipStorage: + if any(self.ent_for(name) is None for name in entity_names): + return RelationshipStorage.ABSENT + if not self._table_exists(table_name): + return RelationshipStorage.UNKNOWN + if not set(required_columns).issubset(self._table_columns(table_name)): + return RelationshipStorage.UNKNOWN + return RelationshipStorage.PRESENT + + @staticmethod + def _relationship_error(exc: Exception) -> LoadErrorKind: + if isinstance(exc, KeyError): + return LoadErrorKind.MISSING_FIELD + if isinstance(exc, AssertionError): + return LoadErrorKind.VALIDATION + if isinstance(exc, ValueError): + return LoadErrorKind.INVALID_VALUE + return LoadErrorKind.CONSTRUCTION + def query_objects(self, typenames: List[str]) -> List[Any]: + ent_ids = [self.ent_for(name) for name in typenames] + ent_ids = [ent_id for ent_id in ent_ids if ent_id is not None] + if not ent_ids: + return [] cur = self._con.cursor() res = cur.execute( """ SELECT * FROM ZSYNCOBJECT WHERE Z_ENT in (%s) """ - % (",".join("?" * len(typenames))), - [self.ent_for(x) for x in typenames], + % (",".join("?" * len(ent_ids))), + ent_ids, ) return res.fetchall() + def close(self) -> None: + """Close the read-only SQLite connection.""" + self._con.close() + + @contextmanager + def read_transaction(self): + """Keep a selected multi-manager load on one SQLite snapshot.""" + owns_transaction = not self._con.in_transaction + if owns_transaction: + self._con.execute("BEGIN") + try: + yield + finally: + if owns_transaction and self._con.in_transaction: + self._con.rollback() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.close() + def _construct_record(self, row, constructor: Callable): investment_constructors = { InvestmentBuyTransaction, @@ -86,8 +224,13 @@ def _construct_record(self, row, constructor: Callable): InvestmentHolding, } if constructor in investment_constructors: - return constructor(row, schema_profile=self.schema_profile) - return constructor(row) + record = constructor(row, schema_profile=self.schema_profile) + else: + record = constructor(row) + validate = getattr(record, "validate", None) + if validate is not None: + validate() + return record def get_record(self, pk_id: ID, constructor: Callable = Record): cur = self._con.cursor() @@ -113,51 +256,191 @@ def get_record_by_gid(self, gid: GID, constructor: Callable = Record): return self._construct_record(res.fetchone(), constructor) - def get_category_assignment(self) -> Dict[ID, List[Tuple[ID, Decimal]]]: + def read_category_assignments( + self, + ) -> tuple[Dict[ID, List[Tuple[ID, Decimal]]], RelationshipLoadReport]: transaction_map: Dict[ID, List[Tuple[ID, Decimal]]] = defaultdict(list) - if not self._table_exists("ZCATEGORYASSIGMENT"): - return transaction_map - cur = self._con.cursor() - res = cur.execute( - """ - SELECT ZCATEGORY, ZTRANSACTION, ZAMOUNT FROM ZCATEGORYASSIGMENT WHERE ZTRANSACTION IS NOT NULL - - """ + table_name = "ZCATEGORYASSIGMENT" + columns = ("Z_PK", "ZCATEGORY", "ZTRANSACTION", "ZAMOUNT") + storage = self._relationship_storage( + entity_names=("CategoryAssigment",), + table_name=table_name, + required_columns=columns, ) - for row in res.fetchall(): - transaction_map[row["ZTRANSACTION"]].append( - (row["ZCATEGORY"], RDH.get_decimal(row, "ZAMOUNT")) + if storage != RelationshipStorage.PRESENT: + return transaction_map, RelationshipLoadReport( + storage=storage, + storage_name=table_name, ) - return transaction_map - def get_refund_maps(self) -> Dict[ID, ID]: + source_ids: list[ID | None] = [] + parsed_ids: list[ID] = [] + skipped: list[SkippedRecord] = [] + rows = self._con.execute( + f'SELECT {", ".join(columns)} FROM "{table_name}" ' + "WHERE ZTRANSACTION IS NOT NULL" + ).fetchall() + for row in rows: + source_id = row.get("Z_PK") + source_ids.append(source_id) + try: + assert source_id is not None + category_id = row["ZCATEGORY"] + transaction_id = row["ZTRANSACTION"] + assert category_id is not None + assert transaction_id is not None + amount = RDH.get_decimal(row, "ZAMOUNT") + transaction_map[transaction_id].append((category_id, amount)) + except Exception as exc: + skipped.append( + SkippedRecord( + record_id=source_id, + entity="CategoryAssigment", + error=self._relationship_error(exc), + exception_type=type(exc).__name__, + ) + ) + continue + parsed_ids.append(source_id) + return transaction_map, RelationshipLoadReport( + storage=storage, + storage_name=table_name, + source_ids=tuple(source_ids), + parsed_ids=tuple(parsed_ids), + skipped=tuple(skipped), + ) + + def get_category_assignment(self) -> Dict[ID, List[Tuple[ID, Decimal]]]: + """Return category assignments without completeness metadata.""" + assignments, _ = self.read_category_assignments() + return assignments + + def read_refund_maps( + self, + ) -> tuple[Dict[ID, ID], RelationshipLoadReport]: refund_to_withdraw: Dict[ID, ID] = {} - if not self._table_exists("ZWITHDRAWREFUNDTRANSACTIONLINK"): - return refund_to_withdraw - cur = self._con.cursor() - res = cur.execute( - """ - SELECT ZREFUNDTRANSACTION, ZWITHDRAWTRANSACTION FROM ZWITHDRAWREFUNDTRANSACTIONLINK - - """ + table_name = "ZWITHDRAWREFUNDTRANSACTIONLINK" + columns = ("Z_PK", "ZREFUNDTRANSACTION", "ZWITHDRAWTRANSACTION") + storage = self._relationship_storage( + entity_names=("WithdrawRefundTransactionLink",), + table_name=table_name, + required_columns=columns, ) - for row in res.fetchall(): - refund_to_withdraw[row["ZREFUNDTRANSACTION"]] = row["ZWITHDRAWTRANSACTION"] - return refund_to_withdraw + if storage != RelationshipStorage.PRESENT: + return refund_to_withdraw, RelationshipLoadReport( + storage=storage, + storage_name=table_name, + ) - def get_tags_map(self) -> Dict[ID, List[ID]]: + source_ids: list[ID | None] = [] + parsed_ids: list[ID] = [] + skipped: list[SkippedRecord] = [] + rows = self._con.execute( + f'SELECT {", ".join(columns)} FROM "{table_name}"' + ).fetchall() + for row in rows: + source_id = row.get("Z_PK") + source_ids.append(source_id) + try: + assert source_id is not None + refund_id = row["ZREFUNDTRANSACTION"] + withdraw_id = row["ZWITHDRAWTRANSACTION"] + assert refund_id is not None + assert withdraw_id is not None + if refund_id in refund_to_withdraw: + raise ValueError() + refund_to_withdraw[refund_id] = withdraw_id + except Exception as exc: + skipped.append( + SkippedRecord( + record_id=source_id, + entity="WithdrawRefundTransactionLink", + error=self._relationship_error(exc), + exception_type=type(exc).__name__, + ) + ) + continue + parsed_ids.append(source_id) + return refund_to_withdraw, RelationshipLoadReport( + storage=storage, + storage_name=table_name, + source_ids=tuple(source_ids), + parsed_ids=tuple(parsed_ids), + skipped=tuple(skipped), + ) + + def get_refund_maps(self) -> Dict[ID, ID]: + """Return refund links without completeness metadata.""" + refund_maps, _ = self.read_refund_maps() + return refund_maps + + def read_tags_map( + self, + ) -> tuple[Dict[ID, List[ID]], RelationshipLoadReport]: transactions_to_tags: Dict[ID, List[ID]] = defaultdict(list) - if not self._table_exists("Z_36TAGS"): - return transactions_to_tags - cur = self._con.cursor() - res = cur.execute( - """ - SELECT Z_36TRANSACTIONS, Z_35TAGS FROM Z_36TAGS - - """ + transaction_ent = self.ent_for("Transaction") + tag_ent = self.ent_for("Tag") + if transaction_ent is None or tag_ent is None: + return transactions_to_tags, RelationshipLoadReport( + storage=RelationshipStorage.ABSENT, + ) + + table_name = f"Z_{transaction_ent}TAGS" + transaction_column = f"Z_{transaction_ent}TRANSACTIONS" + tag_column = f"Z_{tag_ent}TAGS" + columns = (transaction_column, tag_column) + storage = self._relationship_storage( + entity_names=("Transaction", "Tag"), + table_name=table_name, + required_columns=columns, ) - for row in res.fetchall(): - transactions_to_tags[row["Z_36TRANSACTIONS"]].append(row["Z_35TAGS"]) + if storage != RelationshipStorage.PRESENT: + return transactions_to_tags, RelationshipLoadReport( + storage=storage, + storage_name=table_name, + ) + + source_ids: list[str] = [] + parsed_ids: list[str] = [] + skipped: list[SkippedRecord] = [] + rows = self._con.execute( + f'SELECT {", ".join(columns)} FROM "{table_name}"' + ).fetchall() + for position, row in enumerate(rows): + transaction_id = row.get(transaction_column) + tag_id = row.get(tag_column) + source_id = ( + f"{transaction_id}:{tag_id}" + if transaction_id is not None and tag_id is not None + else f"row:{position}" + ) + source_ids.append(source_id) + try: + assert transaction_id is not None + assert tag_id is not None + transactions_to_tags[transaction_id].append(tag_id) + except Exception as exc: + skipped.append( + SkippedRecord( + record_id=source_id, + entity="TransactionTag", + error=self._relationship_error(exc), + exception_type=type(exc).__name__, + ) + ) + continue + parsed_ids.append(source_id) + return transactions_to_tags, RelationshipLoadReport( + storage=storage, + storage_name=table_name, + source_ids=tuple(source_ids), + parsed_ids=tuple(parsed_ids), + skipped=tuple(skipped), + ) + + def get_tags_map(self) -> Dict[ID, List[ID]]: + """Return transaction tags without completeness metadata.""" + transactions_to_tags, _ = self.read_tags_map() return transactions_to_tags def get_users(self) -> Dict[ID, str]: diff --git a/src/moneywiz_api/managers/account_manager.py b/src/moneywiz_api/managers/account_manager.py index 5a5ed48..2c53984 100644 --- a/src/moneywiz_api/managers/account_manager.py +++ b/src/moneywiz_api/managers/account_manager.py @@ -31,6 +31,10 @@ def ents(self) -> Dict[str, Callable]: "ForexAccount": ForexAccount, } + @property + def entity_roots(self) -> tuple[str, ...]: + return ("Account",) + def records(self) -> Dict[ID, Account]: return dict(sorted(super().records().items(), key=lambda x: x[1].display_order)) diff --git a/src/moneywiz_api/managers/record_manager.py b/src/moneywiz_api/managers/record_manager.py index f158c43..0caf80b 100644 --- a/src/moneywiz_api/managers/record_manager.py +++ b/src/moneywiz_api/managers/record_manager.py @@ -1,9 +1,14 @@ from abc import ABC, abstractmethod import logging -from typing import Dict, Generic, TypeVar, Callable +from typing import Callable, Dict, Generic, TypeVar from moneywiz_api.database_accessor import DatabaseAccessor from moneywiz_api.model.record import Record +from moneywiz_api.read_result import ( + LoadErrorKind, + ManagerLoadReport, + SkippedRecord, +) from moneywiz_api.types import ID, GID @@ -11,39 +16,99 @@ logger = logging.getLogger(__name__) +class DuplicateRecordIdError(ValueError): + """Raised when two source rows use the same primary key.""" + + +class DuplicateRecordGidError(ValueError): + """Raised when two source rows use the same global identifier.""" + + class RecordManager(ABC, Generic[T]): def __init__(self): self._records: Dict[ID, T] = {} self._gid_to_id: Dict[GID, ID] = {} - self._load_errors: list[tuple[ID, str, str]] = [] + self._load_report = ManagerLoadReport() @property @abstractmethod def ents(self) -> Dict[str, Callable]: raise NotImplementedError() - def load(self, db_accessor: DatabaseAccessor) -> None: - records = db_accessor.query_objects(self.ents.keys()) + @property + def entity_roots(self) -> tuple[str, ...]: + """Return schema roots whose unknown descendants must be reported.""" + return () + + def load(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: + self._records = {} + self._gid_to_id = {} + self._load_report = ManagerLoadReport() + + typenames = list(self.ents) + if self.entity_roots: + discovered = db_accessor.descendant_typenames(self.entity_roots) + typenames = list(dict.fromkeys([*typenames, *discovered])) + records = db_accessor.query_objects(typenames) + + source_ids: list[ID | None] = [] + parsed_ids: list[ID] = [] + skipped: list[SkippedRecord] = [] for record in records: - typename = db_accessor.typename_for(record["Z_ENT"]) - if typename in self.ents: - try: - obj = self.construct_record( - self.ents[typename], record, db_accessor - ) - except (AssertionError, KeyError, ValueError) as exc: - record_id = record.get("Z_PK") - detail = type(exc).__name__ - self._load_errors.append((record_id, typename, detail)) - logger.debug( - "Skipping unreadable %s record %s: %s", - typename, - record_id, - detail, - ) - continue + record_id = record.get("Z_PK") + source_ids.append(record_id) + typename = None + try: + typename = db_accessor.typename_for(record["Z_ENT"]) + if typename not in self.ents: + raise UnknownEntityError() + obj = self.construct_record(self.ents[typename], record, db_accessor) + validate = getattr(obj, "validate", None) + if validate is not None: + validate() self.add(obj) + except Exception as exc: + error = self._error_kind(exc) + skipped.append( + SkippedRecord( + record_id=record_id, + entity=typename, + error=error, + exception_type=type(exc).__name__, + ) + ) + logger.debug( + "Skipping unreadable %s record %s: %s", + typename, + record_id, + error.value, + ) + continue + parsed_ids.append(obj.id) + + self._load_report = ManagerLoadReport( + source_ids=tuple(source_ids), + parsed_ids=tuple(parsed_ids), + skipped=tuple(skipped), + ) + return self._load_report + + @staticmethod + def _error_kind(exc: Exception) -> LoadErrorKind: + if isinstance(exc, UnknownEntityError): + return LoadErrorKind.UNKNOWN_ENTITY + if isinstance(exc, DuplicateRecordIdError): + return LoadErrorKind.DUPLICATE_ID + if isinstance(exc, DuplicateRecordGidError): + return LoadErrorKind.DUPLICATE_GID + if isinstance(exc, KeyError): + return LoadErrorKind.MISSING_FIELD + if isinstance(exc, AssertionError): + return LoadErrorKind.VALIDATION + if isinstance(exc, ValueError): + return LoadErrorKind.INVALID_VALUE + return LoadErrorKind.CONSTRUCTION def construct_record( self, constructor: Callable, record, db_accessor: DatabaseAccessor @@ -52,12 +117,12 @@ def construct_record( return constructor(record) def add(self, record: T) -> None: - self._records[record.id] = record + if record.id in self._records: + raise DuplicateRecordIdError() if record.gid in self._gid_to_id: - raise RuntimeError( - f"Duplicate gid for {record}, existing record Id {self._gid_to_id[record.gid]}" - ) + raise DuplicateRecordGidError() + self._records[record.id] = record self._gid_to_id[record.gid] = record.id def get(self, record_id: ID) -> T | None: @@ -70,9 +135,21 @@ def records(self) -> Dict[ID, T]: return self._records @property - def load_errors(self) -> list[tuple[ID, str, str]]: + def load_errors(self) -> list[tuple[ID | None, str | None, str]]: """Return records skipped during best-effort read parsing.""" - return list(self._load_errors) + return [ + (item.record_id, item.entity, item.exception_type) + for item in self._load_report.skipped + ] + + @property + def load_report(self) -> ManagerLoadReport: + """Return structured completeness evidence from the latest load.""" + return self._load_report def __repr__(self): return "\n".join(f"{key}: {value}" for key, value in self.records().items()) + + +class UnknownEntityError(ValueError): + """Raised when a source row belongs to an unsupported subtype.""" diff --git a/src/moneywiz_api/managers/transaction_manager.py b/src/moneywiz_api/managers/transaction_manager.py index 03f6f36..03baf08 100644 --- a/src/moneywiz_api/managers/transaction_manager.py +++ b/src/moneywiz_api/managers/transaction_manager.py @@ -1,4 +1,5 @@ from datetime import datetime +from dataclasses import replace from typing import Dict, Callable, List, Tuple from decimal import Decimal @@ -17,6 +18,7 @@ WithdrawTransaction, ) from moneywiz_api.managers.record_manager import RecordManager +from moneywiz_api.read_result import ManagerLoadReport from moneywiz_api.types import ID @@ -25,7 +27,7 @@ def __init__(self): super().__init__() self.category_assignment: Dict[ID, List[Tuple[ID, Decimal]]] = {} self.refund_maps: Dict[ID, ID] = {} - self.tags_map: Dict[ID, ID] = {} + self.tags_map: Dict[ID, List[ID]] = {} @property def ents(self) -> Dict[str, Callable]: @@ -42,6 +44,10 @@ def ents(self) -> Dict[str, Callable]: "WithdrawTransaction": WithdrawTransaction, } + @property + def entity_roots(self) -> tuple[str, ...]: + return ("Transaction",) + def construct_record( self, constructor: Callable, record, db_accessor: DatabaseAccessor ): @@ -53,13 +59,26 @@ def construct_record( return constructor(record, schema_profile=db_accessor.schema_profile) return super().construct_record(constructor, record, db_accessor) - def load(self, db_accessor: DatabaseAccessor) -> None: - super().load(db_accessor) - self.category_assignment: Dict[ID, List[Tuple[ID, Decimal]]] = ( - db_accessor.get_category_assignment() + def load(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: + self.category_assignment = {} + self.refund_maps = {} + self.tags_map = {} + report = super().load(db_accessor) + category_assignment, category_report = db_accessor.read_category_assignments() + refund_maps, refund_report = db_accessor.read_refund_maps() + tags_map, tags_report = db_accessor.read_tags_map() + self.category_assignment = category_assignment + self.refund_maps = refund_maps + self.tags_map = tags_map + self._load_report = replace( + report, + relationships={ + "category_assignments": category_report, + "refund_links": refund_report, + "transaction_tags": tags_report, + }, ) - self.refund_maps: Dict[ID, ID] = db_accessor.get_refund_maps() - self.tags_map: Dict[ID, ID] = db_accessor.get_tags_map() + return self._load_report def category_for_transaction( self, transaction_id: ID diff --git a/src/moneywiz_api/model/account.py b/src/moneywiz_api/model/account.py index d46d97b..7bf0f4c 100644 --- a/src/moneywiz_api/model/account.py +++ b/src/moneywiz_api/model/account.py @@ -1,6 +1,7 @@ from abc import ABC from dataclasses import dataclass, field from decimal import Decimal +from typing import Optional from moneywiz_api.types import ID from moneywiz_api.model.raw_data_handler import RawDataHandler as RDH @@ -19,7 +20,7 @@ class Account(Record, ABC): name: str currency: str opening_balance: Decimal # might be a tiny number - info: str + info: Optional[str] user: ID def __init__(self, row): @@ -30,20 +31,18 @@ def __init__(self, row): self.name = row["ZNAME"] self.currency = row["ZCURRENCYNAME"] self.opening_balance = RDH.get_decimal(row, "ZOPENINGBALANCE") - self.info = row["ZINFO"] + self.info = row.get("ZINFO") self.user = row["ZUSER"] - # Fixes - - # Validate - assert self.display_order is not None, self.as_dict() - assert self.group_id is not None, self.as_dict() - assert self.name is not None, self.as_dict() - assert self.currency is not None, self.as_dict() - assert self.opening_balance is not None, self.as_dict() - assert self.info is not None, self.as_dict() - assert self.user is not None, self.as_dict() + def validate(self) -> None: + assert self.display_order is not None + assert self.group_id is not None + assert self.name is not None + assert self.currency is not None + assert self.opening_balance is not None + # info is nullable in valid MoneyWiz stores. + assert self.user is not None @dataclass @@ -88,7 +87,8 @@ def __init__(self, row): super().__init__(row) self.statement_day = row["ZSTATEMENTENDDAY"] - # Validate + def validate(self) -> None: + super().validate() assert self.statement_day is not None diff --git a/src/moneywiz_api/model/investment_holding.py b/src/moneywiz_api/model/investment_holding.py index 3b1dd8a..ee65a33 100644 --- a/src/moneywiz_api/model/investment_holding.py +++ b/src/moneywiz_api/model/investment_holding.py @@ -18,7 +18,7 @@ class InvestmentHolding(Record): opening_number_of_shares: Optional[Decimal] number_of_shares: Decimal - # price_per_share: Decimal + price_per_share: Optional[Decimal] symbol: str holding_type: Optional[str] description: str @@ -56,7 +56,12 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): "ZNUMBEROFSHARES", "ZNUMBEROFSHARES1", ) - # self.price_per_share = row["ZPRICEPERSHARE"] + self.price_per_share = RDH.get_profile_nullable_decimal( + row, + schema_profile.holding_price_per_share_column if schema_profile else None, + "ZPRICEPERSHARE", + "ZPRICEPERSHARE1", + ) self.symbol = row["ZSYMBOL"] self.holding_type = row["ZHOLDINGTYPE"] self.description = row["ZDESC"] @@ -69,16 +74,13 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): row, "ZCOSTBASISOFMISSINGOBSHARES" ) - # Fixes - self.number_of_shares = self.number_of_shares or Decimal(0) - # Validate self.validate() def validate(self): assert self.account is not None, self.as_dict() assert self.number_of_shares is not None, self.as_dict() - # assert self.price_per_share is not None + # price_per_share can be None when no current quote is stored. assert self.symbol is not None, self.as_dict() assert self.description is not None, self.as_dict() diff --git a/src/moneywiz_api/model/transaction.py b/src/moneywiz_api/model/transaction.py index 1b4342b..2bf3771 100644 --- a/src/moneywiz_api/model/transaction.py +++ b/src/moneywiz_api/model/transaction.py @@ -38,10 +38,10 @@ def __init__(self, row): # Fixes # Validate - assert self.reconciled is not None, self.as_dict() - assert self.amount is not None, self.as_dict() - assert self.description is not None, self.as_dict() - assert self.datetime is not None, self.as_dict() + assert self.reconciled is not None + assert self.amount is not None + assert self.description is not None + assert self.datetime is not None # self.notes can be None @@ -190,7 +190,9 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): ) self.price_per_share = RDH.get_profile_decimal( row, - schema_profile.price_per_share_column if schema_profile else None, + schema_profile.transaction_price_per_share_column + if schema_profile + else None, "ZPRICEPERSHARE1", "ZPRICEPERSHARE", ) @@ -257,7 +259,9 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): ) self.price_per_share = RDH.get_profile_decimal( row, - schema_profile.price_per_share_column if schema_profile else None, + schema_profile.transaction_price_per_share_column + if schema_profile + else None, "ZPRICEPERSHARE1", "ZPRICEPERSHARE", ) diff --git a/src/moneywiz_api/moneywiz_api.py b/src/moneywiz_api/moneywiz_api.py index 3195eb0..23aef16 100644 --- a/src/moneywiz_api/moneywiz_api.py +++ b/src/moneywiz_api/moneywiz_api.py @@ -1,5 +1,8 @@ +"""Public MoneyWiz read API.""" + +from dataclasses import asdict from pathlib import Path -import logging +from typing import Iterable from moneywiz_api.database_accessor import DatabaseAccessor from moneywiz_api.managers.account_manager import AccountManager @@ -10,12 +13,26 @@ from moneywiz_api.managers.payee_manager import PayeeManager from moneywiz_api.managers.transaction_manager import TransactionManager from moneywiz_api.managers.tag_manager import TagManager - -logger = logging.getLogger(__name__) +from moneywiz_api.read_result import ApiCompleteness, ReadSnapshot, json_safe class MoneywizApi: - def __init__(self, db_file: Path): + """Read a MoneyWiz database with explicit completeness evidence.""" + + MANAGER_NAMES = ( + "accounts", + "payees", + "categories", + "transactions", + "investment_holdings", + "tags", + ) + + def __init__( + self, + db_file: Path, + managers: Iterable[str] | None = None, + ): self.accessor = DatabaseAccessor(db_file) self.account_manager = AccountManager() self.payee_manager = PayeeManager() @@ -23,13 +40,89 @@ def __init__(self, db_file: Path): self.transaction_manager = TransactionManager() self.investment_holding_manager = InvestmentHoldingManager() self.tag_manager = TagManager() + self._managers = { + "accounts": self.account_manager, + "payees": self.payee_manager, + "categories": self.category_manager, + "transactions": self.transaction_manager, + "investment_holdings": self.investment_holding_manager, + "tags": self.tag_manager, + } + self._loaded_managers: set[str] = set() + + try: + self.load(managers) + except Exception: + self.close() + raise + + def _manager_names(self, managers: Iterable[str] | None) -> tuple[str, ...]: + if managers is None: + return self.MANAGER_NAMES + if isinstance(managers, str): + raise TypeError("managers must be an iterable of manager names") + names = tuple(dict.fromkeys(managers)) + unknown = [name for name in names if name not in self._managers] + if unknown: + raise ValueError(f"unknown manager names: {', '.join(unknown)}") + return names + + def load(self, managers: Iterable[str] | None = None) -> ApiCompleteness: + """Atomically reload requested managers and every retained manager.""" + requested = self._manager_names(managers) + names = tuple( + name + for name in self.MANAGER_NAMES + if name in self._loaded_managers or name in requested + ) + self._loaded_managers.difference_update(names) + with self.accessor.read_transaction(): + for name in names: + self._managers[name].load(self.accessor) + self._loaded_managers.update(names) + return self.completeness(names) + + def completeness(self, managers: Iterable[str] | None = None) -> ApiCompleteness: + """Return completeness evidence without reloading the database.""" + names = ( + tuple(name for name in self.MANAGER_NAMES if name in self._loaded_managers) + if managers is None + else self._manager_names(managers) + ) + unloaded = [name for name in names if name not in self._loaded_managers] + if unloaded: + raise ValueError(f"manager not loaded: {', '.join(unloaded)}") + return ApiCompleteness( + managers={name: self._managers[name].load_report for name in names} + ) + + def snapshot(self, managers: Iterable[str] | None = None) -> ReadSnapshot: + """Return loaded records and diagnostics in a JSON-safe structure.""" + names = ( + tuple(name for name in self.MANAGER_NAMES if name in self._loaded_managers) + if managers is None + else self._manager_names(managers) + ) + completeness = self.completeness(names) + records = { + name: tuple( + json_safe(record.as_dict()) + for record in self._managers[name].records().values() + ) + for name in names + } + return ReadSnapshot( + records=records, + completeness=completeness, + schema_profile=json_safe(asdict(self.accessor.schema_profile)), + ) + + def close(self) -> None: + """Close the underlying read-only database connection.""" + self.accessor.close() - self.load() + def __enter__(self): + return self - def load(self): - self.account_manager.load(self.accessor) - self.payee_manager.load(self.accessor) - self.category_manager.load(self.accessor) - self.transaction_manager.load(self.accessor) - self.investment_holding_manager.load(self.accessor) - self.tag_manager.load(self.accessor) + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.close() diff --git a/src/moneywiz_api/read_result.py b/src/moneywiz_api/read_result.py new file mode 100644 index 0000000..75431a7 --- /dev/null +++ b/src/moneywiz_api/read_result.py @@ -0,0 +1,217 @@ +"""Structured results for bounded MoneyWiz reads.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from datetime import date, datetime +from decimal import Decimal +from enum import Enum +from pathlib import Path +from typing import Any, Mapping + +from moneywiz_api.types import ID + + +class LoadErrorKind(str, Enum): + """Bounded classifications for records omitted from a read.""" + + MISSING_FIELD = "missing_field" + INVALID_VALUE = "invalid_value" + VALIDATION = "validation" + DUPLICATE_ID = "duplicate_id" + DUPLICATE_GID = "duplicate_gid" + UNKNOWN_ENTITY = "unknown_entity" + CONSTRUCTION = "construction" + + +class RelationshipStorage(str, Enum): + """Whether relationship storage is available and understood.""" + + PRESENT = "present" + ABSENT = "absent" + UNKNOWN = "unknown" + + +@dataclass(frozen=True) +class SkippedRecord: + """Identity-only diagnostic for one source row that was not parsed.""" + + record_id: ID | str | None + entity: str | None + error: LoadErrorKind + exception_type: str + + def as_dict(self) -> dict[str, Any]: + return { + "record_id": self.record_id, + "entity": self.entity, + "error": self.error.value, + "exception_type": self.exception_type, + } + + +@dataclass(frozen=True) +class RelationshipLoadReport: + """Completeness evidence for one relationship storage layout.""" + + storage: RelationshipStorage + storage_name: str | None = None + source_ids: tuple[ID | str | None, ...] = () + parsed_ids: tuple[ID | str, ...] = () + skipped: tuple[SkippedRecord, ...] = () + + @property + def source_count(self) -> int: + return len(self.source_ids) + + @property + def parsed_count(self) -> int: + return len(self.parsed_ids) + + @property + def complete(self) -> bool: + return ( + self.storage != RelationshipStorage.UNKNOWN + and self.source_count == self.parsed_count + and not self.skipped + ) + + @property + def status(self) -> str: + if self.storage == RelationshipStorage.ABSENT: + return "absent" + if self.complete: + return "complete" + if self.parsed_count: + return "partial" + return "error" + + def as_dict(self) -> dict[str, Any]: + return { + "status": self.status, + "complete": self.complete, + "storage": self.storage.value, + "storage_name": self.storage_name, + "source_count": self.source_count, + "parsed_count": self.parsed_count, + "source_ids": list(self.source_ids), + "parsed_ids": list(self.parsed_ids), + "skipped": [item.as_dict() for item in self.skipped], + } + + +@dataclass(frozen=True) +class ManagerLoadReport: + """Completeness evidence for one manager load.""" + + source_ids: tuple[ID | None, ...] = () + parsed_ids: tuple[ID, ...] = () + skipped: tuple[SkippedRecord, ...] = () + relationships: Mapping[str, RelationshipLoadReport] = field(default_factory=dict) + + @property + def source_count(self) -> int: + return len(self.source_ids) + + @property + def parsed_count(self) -> int: + return len(self.parsed_ids) + + @property + def complete(self) -> bool: + return ( + self.source_count == self.parsed_count + and not self.skipped + and all(report.complete for report in self.relationships.values()) + ) + + @property + def status(self) -> str: + if self.complete: + return "complete" + if self.parsed_count or any( + report.parsed_count for report in self.relationships.values() + ): + return "partial" + return "error" + + def as_dict(self) -> dict[str, Any]: + return { + "status": self.status, + "complete": self.complete, + "source_count": self.source_count, + "parsed_count": self.parsed_count, + "source_ids": list(self.source_ids), + "parsed_ids": list(self.parsed_ids), + "skipped": [item.as_dict() for item in self.skipped], + "relationships": { + name: report.as_dict() for name, report in self.relationships.items() + }, + } + + +@dataclass(frozen=True) +class ApiCompleteness: + """Aggregate completeness for the managers included in a read.""" + + managers: Mapping[str, ManagerLoadReport] + + @property + def complete(self) -> bool: + return bool(self.managers) and all( + report.complete for report in self.managers.values() + ) + + @property + def status(self) -> str: + if self.complete: + return "complete" + if self.managers and all( + report.status == "error" for report in self.managers.values() + ): + return "error" + return "partial" + + def as_dict(self) -> dict[str, Any]: + return { + "status": self.status, + "complete": self.complete, + "managers": { + name: report.as_dict() for name, report in self.managers.items() + }, + } + + +@dataclass(frozen=True) +class ReadSnapshot: + """JSON-safe records and completeness metadata from one API instance.""" + + records: Mapping[str, tuple[dict[str, Any], ...]] + completeness: ApiCompleteness + schema_profile: Mapping[str, Any] + + def as_dict(self) -> dict[str, Any]: + return { + "schema_profile": dict(self.schema_profile), + "completeness": self.completeness.as_dict(), + "records": {name: list(records) for name, records in self.records.items()}, + } + + +def json_safe(value: Any) -> Any: + """Convert public model values into deterministic JSON-compatible values.""" + if isinstance(value, Decimal): + return str(value) + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, Path): + return str(value) + if isinstance(value, Enum): + return value.value + if hasattr(value, "__dataclass_fields__"): + return json_safe(asdict(value)) + if isinstance(value, Mapping): + return {str(key): json_safe(item) for key, item in value.items()} + if isinstance(value, (tuple, list)): + return [json_safe(item) for item in value] + return value diff --git a/src/moneywiz_api/schema_profile.py b/src/moneywiz_api/schema_profile.py index fcfb40c..c27145f 100644 --- a/src/moneywiz_api/schema_profile.py +++ b/src/moneywiz_api/schema_profile.py @@ -13,7 +13,8 @@ class SchemaProfile: profile_id: str holding_number_of_shares_column: str | None transaction_number_of_shares_column: str | None - price_per_share_column: str | None + holding_price_per_share_column: str | None + transaction_price_per_share_column: str | None @property def is_known(self) -> bool: @@ -30,37 +31,42 @@ def detect_schema_profile(connection: sqlite3.Connection) -> SchemaProfile: has_suffixed_price = "ZPRICEPERSHARE1" in columns has_unsuffixed_price = "ZPRICEPERSHARE" in columns - if ( - has_suffixed_shares - and has_unsuffixed_shares - and has_suffixed_price - and has_unsuffixed_price + if not (has_suffixed_shares or has_unsuffixed_shares) or not ( + has_suffixed_price or has_unsuffixed_price ): profile_id = "unknown" holding_number_of_shares_column = None transaction_number_of_shares_column = None - price_per_share_column = None - elif ( - has_suffixed_shares - and has_unsuffixed_shares - and (has_suffixed_price or has_unsuffixed_price) + holding_price_per_share_column = None + transaction_price_per_share_column = None + elif (has_suffixed_shares and has_unsuffixed_shares) or ( + has_suffixed_price and has_unsuffixed_price ): profile_id = "mixed-investment-columns" - holding_number_of_shares_column = "ZNUMBEROFSHARES" - transaction_number_of_shares_column = "ZNUMBEROFSHARES1" - price_per_share_column = ( + holding_number_of_shares_column = ( + "ZNUMBEROFSHARES" if has_unsuffixed_shares else "ZNUMBEROFSHARES1" + ) + transaction_number_of_shares_column = ( + "ZNUMBEROFSHARES1" if has_suffixed_shares else "ZNUMBEROFSHARES" + ) + holding_price_per_share_column = ( + "ZPRICEPERSHARE" if has_unsuffixed_price else "ZPRICEPERSHARE1" + ) + transaction_price_per_share_column = ( "ZPRICEPERSHARE1" if has_suffixed_price else "ZPRICEPERSHARE" ) elif has_suffixed_shares and has_suffixed_price: profile_id = "suffixed-investment-columns" holding_number_of_shares_column = "ZNUMBEROFSHARES1" transaction_number_of_shares_column = "ZNUMBEROFSHARES1" - price_per_share_column = "ZPRICEPERSHARE1" + holding_price_per_share_column = "ZPRICEPERSHARE1" + transaction_price_per_share_column = "ZPRICEPERSHARE1" elif has_unsuffixed_shares and has_unsuffixed_price: profile_id = "unsuffixed-investment-columns" holding_number_of_shares_column = "ZNUMBEROFSHARES" transaction_number_of_shares_column = "ZNUMBEROFSHARES" - price_per_share_column = "ZPRICEPERSHARE" + holding_price_per_share_column = "ZPRICEPERSHARE" + transaction_price_per_share_column = "ZPRICEPERSHARE" elif (has_suffixed_shares or has_unsuffixed_shares) and ( has_suffixed_price or has_unsuffixed_price ): @@ -71,18 +77,19 @@ def detect_schema_profile(connection: sqlite3.Connection) -> SchemaProfile: transaction_number_of_shares_column = ( "ZNUMBEROFSHARES1" if has_suffixed_shares else "ZNUMBEROFSHARES" ) - price_per_share_column = ( + holding_price_per_share_column = ( + "ZPRICEPERSHARE" if has_unsuffixed_price else "ZPRICEPERSHARE1" + ) + transaction_price_per_share_column = ( "ZPRICEPERSHARE1" if has_suffixed_price else "ZPRICEPERSHARE" ) - else: - profile_id = "unknown" - holding_number_of_shares_column = None - transaction_number_of_shares_column = None - price_per_share_column = None + else: # pragma: no cover - exhaustive boolean cases above + raise AssertionError("unreachable investment schema profile") return SchemaProfile( profile_id=profile_id, holding_number_of_shares_column=holding_number_of_shares_column, transaction_number_of_shares_column=transaction_number_of_shares_column, - price_per_share_column=price_per_share_column, + holding_price_per_share_column=holding_price_per_share_column, + transaction_price_per_share_column=transaction_price_per_share_column, ) diff --git a/src/moneywiz_api/utils.py b/src/moneywiz_api/utils.py index 1d1d8ed..294e2eb 100644 --- a/src/moneywiz_api/utils.py +++ b/src/moneywiz_api/utils.py @@ -1,10 +1,11 @@ -from datetime import datetime +from datetime import datetime, timezone -_CUTOFF = datetime(2001, 1, 1, 0, 0, 0).timestamp() +_APPLE_EPOCH_TIMESTAMP = datetime(2001, 1, 1, tzinfo=timezone.utc).timestamp() def get_datetime(date: float) -> datetime: - return datetime.fromtimestamp(date + _CUTOFF) + """Convert an absolute Core Data timestamp to a naive local datetime.""" + return datetime.fromtimestamp(date + _APPLE_EPOCH_TIMESTAMP) def get_date_iso(date: float) -> str: @@ -12,4 +13,5 @@ def get_date_iso(date: float) -> str: def get_date(dt: datetime) -> float: - return dt.timestamp() - _CUTOFF + """Convert an aware instant or naive local datetime to Core Data seconds.""" + return dt.timestamp() - _APPLE_EPOCH_TIMESTAMP diff --git a/tests/integration/test_read_completeness_integration.py b/tests/integration/test_read_completeness_integration.py new file mode 100644 index 0000000..4ca1e8c --- /dev/null +++ b/tests/integration/test_read_completeness_integration.py @@ -0,0 +1,10 @@ +from conftest import moneywizApi + + +def test_loaded_managers_expose_accounted_source_rows() -> None: + completeness = moneywizApi.completeness() + + for report in completeness.managers.values(): + assert report.source_count == report.parsed_count + len(report.skipped) + assert len(report.source_ids) == report.source_count + assert len(report.parsed_ids) == report.parsed_count diff --git a/tests/unit/test_account_diagnostics.py b/tests/unit/test_account_diagnostics.py new file mode 100644 index 0000000..2aa17ae --- /dev/null +++ b/tests/unit/test_account_diagnostics.py @@ -0,0 +1,130 @@ +import pytest + +from moneywiz_api.database_accessor import DatabaseAccessor +from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.model.account import CreditCardAccount + + +def account_row(**overrides): + row = { + "Z_ENT": 13, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": "account-1", + "Z_PK": 1, + "ZDISPLAYORDER": 1, + "ZGROUPID": 1, + "ZNAME": "Card", + "ZCURRENCYNAME": "EUR", + "ZOPENINGBALANCE": 0.0, + "ZINFO": None, + "ZUSER": 1, + "ZSTATEMENTENDDAY": 15, + } + row.update(overrides) + return row + + +class AccountAccessor: + def __init__(self, rows): + self.rows = rows + + def descendant_typenames(self, _roots): + return ["CreditCardAccount"] + + def query_objects(self, _typenames): + return self.rows + + def typename_for(self, _ent_id): + return "CreditCardAccount" + + +def test_nullable_info_does_not_mask_complete_credit_card() -> None: + manager = AccountManager() + + report = manager.load(AccountAccessor([account_row()])) + + assert report.complete + assert manager.get(1).info is None + assert manager.get(1).statement_day == 15 + + +def test_missing_optional_info_column_is_nullable() -> None: + manager = AccountManager() + row = account_row() + del row["ZINFO"] + + report = manager.load(AccountAccessor([row])) + + assert report.complete + assert manager.get(1).info is None + + +def test_required_account_identity_remains_required() -> None: + manager = AccountManager() + + report = manager.load(AccountAccessor([account_row(ZNAME=None)])) + + assert not report.complete + assert report.skipped[0].error.value == "validation" + assert report.skipped[0].exception_type == "AssertionError" + + +class StaticCursor: + def __init__(self, row): + self.row = row + + def execute(self, _query, _parameters): + return self + + def fetchone(self): + return self.row + + +class StaticConnection: + def __init__(self, row): + self.row = row + + def cursor(self): + return StaticCursor(self.row) + + +def accessor_for(row): + accessor = DatabaseAccessor.__new__(DatabaseAccessor) + accessor._con = StaticConnection(row) + return accessor + + +def test_public_accessor_validates_nullable_info_after_construction() -> None: + account = accessor_for(account_row()).get_record(1, CreditCardAccount) + account_by_gid = accessor_for(account_row()).get_record_by_gid( + "account-1", CreditCardAccount + ) + + assert account.info is None + assert account_by_gid.statement_day == 15 + + +@pytest.mark.parametrize( + "row", + [ + account_row(ZNAME=None), + account_row(ZSTATEMENTENDDAY=None), + ], +) +@pytest.mark.parametrize( + ("method_name", "lookup"), + [("get_record", 1), ("get_record_by_gid", "account-1")], +) +def test_public_accessor_rejects_invalid_account_metadata( + row, method_name, lookup +) -> None: + with pytest.raises(AssertionError): + getattr(accessor_for(row), method_name)(lookup, CreditCardAccount) + + +def test_public_accessor_missing_subclass_field_is_not_masked() -> None: + row = account_row() + del row["ZSTATEMENTENDDAY"] + + with pytest.raises(KeyError, match="ZSTATEMENTENDDAY"): + accessor_for(row).get_record_by_gid("account-1", CreditCardAccount) diff --git a/tests/unit/test_api_snapshot_generation.py b/tests/unit/test_api_snapshot_generation.py new file mode 100644 index 0000000..c778d9e --- /dev/null +++ b/tests/unit/test_api_snapshot_generation.py @@ -0,0 +1,83 @@ +import sqlite3 + +import pytest + +from moneywiz_api import MoneywizApi + + +def create_read_schema(path) -> None: + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", + [ + (8, "SyncObject", 0, 0), + (9, "Account", 8, 0), + (10, "CashAccount", 9, 0), + (36, "Transaction", 8, 0), + (37, "DepositTransaction", 36, 0), + ], + ) + connection.execute( + "CREATE TABLE ZSYNCOBJECT (" + "Z_PK INTEGER PRIMARY KEY, Z_ENT INTEGER, ZOBJECTCREATIONDATE FLOAT, " + "ZGID TEXT, ZDISPLAYORDER INTEGER, ZGROUPID INTEGER, ZNAME TEXT, " + "ZCURRENCYNAME TEXT, ZOPENINGBALANCE FLOAT, ZINFO TEXT, ZUSER INTEGER, " + "ZRECONCILED INTEGER, ZAMOUNT1 FLOAT, ZDESC2 TEXT, ZDATE1 FLOAT, " + "ZNOTES1 TEXT, ZACCOUNT2 INTEGER, ZPAYEE2 INTEGER, " + "ZORIGINALCURRENCY TEXT, ZORIGINALAMOUNT FLOAT, " + "ZORIGINALEXCHANGERATE FLOAT)" + ) + connection.execute( + "INSERT INTO ZSYNCOBJECT " + "(Z_PK, Z_ENT, ZOBJECTCREATIONDATE, ZGID, ZDISPLAYORDER, ZGROUPID, " + "ZNAME, ZCURRENCYNAME, ZOPENINGBALANCE, ZINFO, ZUSER) " + "VALUES (1, 10, 0, 'account-1', 1, 1, 'Before', 'EUR', 0, NULL, 1)" + ) + connection.commit() + connection.close() + + +def mutate_between_loads(path) -> None: + connection = sqlite3.connect(path) + connection.execute("UPDATE ZSYNCOBJECT SET ZNAME = 'After' WHERE Z_PK = 1") + connection.execute( + "INSERT INTO ZSYNCOBJECT " + "(Z_PK, Z_ENT, ZOBJECTCREATIONDATE, ZGID, ZRECONCILED, ZAMOUNT1, " + "ZDESC2, ZDATE1, ZNOTES1, ZACCOUNT2, ZPAYEE2, ZORIGINALCURRENCY, " + "ZORIGINALAMOUNT, ZORIGINALEXCHANGERATE) " + "VALUES (2, 37, 0, 'transaction-2', 0, 5, 'Income', 0, NULL, 1, " + "NULL, 'EUR', 5, 1)" + ) + connection.commit() + connection.close() + + +@pytest.mark.parametrize( + ("initial_managers", "expected_managers"), + [ + (("accounts",), ("accounts", "transactions")), + (None, MoneywizApi.MANAGER_NAMES), + ], +) +def test_scoped_reload_refreshes_loaded_union_atomically( + tmp_path, initial_managers, expected_managers +) -> None: + path = tmp_path / "generation.sqlite" + create_read_schema(path) + + with MoneywizApi(path, managers=initial_managers) as api: + assert api.account_manager.get(1).name == "Before" + mutate_between_loads(path) + + completeness = api.load(("transactions",)) + snapshot = api.snapshot().as_dict() + + assert tuple(completeness.managers) == expected_managers + assert api.account_manager.get(1).name == "After" + assert api.transaction_manager.get(2).description == "Income" + assert tuple(snapshot["completeness"]["managers"]) == expected_managers + assert snapshot["records"]["accounts"][0]["name"] == "After" diff --git a/tests/unit/test_database_accessor.py b/tests/unit/test_database_accessor.py new file mode 100644 index 0000000..ac4b53c --- /dev/null +++ b/tests/unit/test_database_accessor.py @@ -0,0 +1,100 @@ +import sqlite3 + +import pytest + +from moneywiz_api.database_accessor import ( + DatabaseAccessor, + DatabasePathError, + DatabaseSchemaError, +) + + +def create_schema(path) -> None: + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.execute("CREATE TABLE ZSYNCOBJECT (Z_PK INTEGER, Z_ENT INTEGER)") + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", + [ + (8, "SyncObject", 0, 0), + (9, "Account", 8, 0), + (10, "CashAccount", 9, 0), + (11, "FutureAccount", 9, 0), + (12, "OnlineAccount", 8, 0), + ], + ) + connection.commit() + connection.close() + + +def test_accessor_requires_an_existing_database(tmp_path) -> None: + with pytest.raises(DatabasePathError): + DatabaseAccessor(tmp_path / "missing.sqlite") + + +def test_accessor_rejects_non_moneywiz_sqlite(tmp_path) -> None: + path = tmp_path / "other.sqlite" + sqlite3.connect(path).close() + + with pytest.raises(DatabaseSchemaError): + DatabaseAccessor(path) + + +def test_accessor_is_read_only_and_discovers_true_descendants(tmp_path) -> None: + path = tmp_path / "moneywiz.sqlite" + create_schema(path) + + with DatabaseAccessor(path) as accessor: + assert accessor.descendant_typenames(("Account",)) == [ + "Account", + "CashAccount", + "FutureAccount", + ] + with pytest.raises(sqlite3.OperationalError, match="readonly"): + accessor._con.execute("CREATE TABLE mutation (id INTEGER)") + + +class TrackingConnection(sqlite3.Connection): + was_closed = False + + def close(self): + self.was_closed = True + super().close() + + +@pytest.mark.parametrize("super_definition", ["", ", Z_SUPER INTEGER"]) +def test_metadata_failures_close_connection_and_raise_schema_error( + tmp_path, monkeypatch, super_definition +) -> None: + path = tmp_path / "malformed.sqlite" + original_connect = sqlite3.connect + setup_connection = original_connect(path) + setup_connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + f"(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT{super_definition})" + ) + setup_connection.execute("CREATE TABLE ZSYNCOBJECT (Z_PK INTEGER, Z_ENT INTEGER)") + if super_definition: + setup_connection.execute( + "INSERT INTO Z_PRIMARYKEY (Z_ENT, Z_NAME, Z_SUPER) " + "VALUES (8, 'SyncObject', NULL)" + ) + setup_connection.commit() + setup_connection.close() + opened = [] + + def tracked_connect(*args, **kwargs): + connection = original_connect(*args, factory=TrackingConnection, **kwargs) + opened.append(connection) + return connection + + monkeypatch.setattr(sqlite3, "connect", tracked_connect) + + with pytest.raises(DatabaseSchemaError): + DatabaseAccessor(path) + + assert len(opened) == 1 + assert opened[0].was_closed diff --git a/tests/unit/test_profiled_investment_values.py b/tests/unit/test_profiled_investment_values.py index 9a7c37f..f778c35 100644 --- a/tests/unit/test_profiled_investment_values.py +++ b/tests/unit/test_profiled_investment_values.py @@ -11,21 +11,24 @@ InvestmentSellTransaction, ) from moneywiz_api.schema_profile import SchemaProfile +from moneywiz_api.read_result import RelationshipLoadReport, RelationshipStorage UNSUFFIXED_PROFILE = SchemaProfile( profile_id="unsuffixed-investment-columns", holding_number_of_shares_column="ZNUMBEROFSHARES", transaction_number_of_shares_column="ZNUMBEROFSHARES", - price_per_share_column="ZPRICEPERSHARE", + holding_price_per_share_column="ZPRICEPERSHARE", + transaction_price_per_share_column="ZPRICEPERSHARE", ) MIXED_PROFILE = SchemaProfile( profile_id="mixed-investment-columns", holding_number_of_shares_column="ZNUMBEROFSHARES", transaction_number_of_shares_column="ZNUMBEROFSHARES1", - price_per_share_column="ZPRICEPERSHARE1", + holding_price_per_share_column="ZPRICEPERSHARE", + transaction_price_per_share_column="ZPRICEPERSHARE1", ) -UNKNOWN_PROFILE = SchemaProfile("unknown", None, None, None) +UNKNOWN_PROFILE = SchemaProfile("unknown", None, None, None, None) def investment_transaction_row(ent: int, amount: float) -> dict: @@ -59,6 +62,8 @@ def investment_holding_row() -> dict: "ZOPENNINGNUMBEROFSHARES": None, "ZNUMBEROFSHARES": 2.0, "ZNUMBEROFSHARES1": 9.0, + "ZPRICEPERSHARE": 10.0, + "ZPRICEPERSHARE1": 1.0, "ZSYMBOL": "ACME", "ZHOLDINGTYPE": None, "ZDESC": "Acme Corp.", @@ -86,6 +91,7 @@ def test_holding_uses_selected_profile_alias() -> None: holding = InvestmentHolding(investment_holding_row(), UNSUFFIXED_PROFILE) assert holding.number_of_shares == Decimal("2.0") + assert holding.price_per_share == Decimal("10.0") @pytest.mark.parametrize( @@ -106,6 +112,7 @@ def test_mixed_profile_uses_consumer_specific_share_aliases( assert transaction.number_of_shares == Decimal("9.0") assert transaction.price_per_share == Decimal("1.0") assert holding.number_of_shares == Decimal("2.0") + assert holding.price_per_share == Decimal("10.0") class ProfileAccessor: @@ -133,17 +140,20 @@ def __init__(self, typename: str, row: dict): def query_objects(self, _typenames): return [self.row] + def descendant_typenames(self, _roots): + return [self.typename] + def typename_for(self, _ent_id): return self.typename - def get_category_assignment(self): - return {} + def read_category_assignments(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) - def get_refund_maps(self): - return {} + def read_refund_maps(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) - def get_tags_map(self): - return {} + def read_tags_map(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) def test_managers_load_profiled_investment_records() -> None: diff --git a/tests/unit/test_read_completeness.py b/tests/unit/test_read_completeness.py new file mode 100644 index 0000000..7c3d675 --- /dev/null +++ b/tests/unit/test_read_completeness.py @@ -0,0 +1,198 @@ +from contextlib import contextmanager +from decimal import Decimal +import json + +import pytest + +from moneywiz_api.managers.record_manager import RecordManager +from moneywiz_api.model.record import Record +from moneywiz_api.read_result import ( + LoadErrorKind, + RelationshipLoadReport, + RelationshipStorage, +) +from moneywiz_api.schema_profile import SchemaProfile + + +class ExampleRecord(Record): + pass + + +class ExampleManager(RecordManager): + @property + def ents(self): + return {"ExampleRecord": ExampleRecord} + + @property + def entity_roots(self): + return ("ExampleRecord",) + + +def record_row(record_id=1, gid="record-1", ent=1): + return { + "Z_ENT": ent, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": gid, + "Z_PK": record_id, + } + + +class RecordAccessor: + def __init__(self, rows, typenames=None): + self.rows = rows + self.typenames = typenames or {1: "ExampleRecord"} + + def descendant_typenames(self, _roots): + return list(self.typenames.values()) + + def query_objects(self, _typenames): + return self.rows + + def typename_for(self, ent_id): + return self.typenames.get(ent_id) + + +def test_report_counts_duplicate_and_unknown_rows_without_partial_mutation() -> None: + manager = ExampleManager() + rows = [ + record_row(), + record_row(gid="duplicate-id"), + record_row(record_id=2), + record_row(record_id=3, gid="record-3", ent=2), + ] + + report = manager.load(RecordAccessor(rows, {1: "ExampleRecord", 2: "FutureRecord"})) + + assert report.source_ids == (1, 1, 2, 3) + assert report.parsed_ids == (1,) + assert [item.error for item in report.skipped] == [ + LoadErrorKind.DUPLICATE_ID, + LoadErrorKind.DUPLICATE_GID, + LoadErrorKind.UNKNOWN_ENTITY, + ] + assert list(manager.records()) == [1] + + +def test_each_load_resets_records_and_diagnostics() -> None: + manager = ExampleManager() + manager.load(RecordAccessor([record_row(), record_row(gid="duplicate-id")])) + + report = manager.load(RecordAccessor([record_row(record_id=4, gid="record-4")])) + + assert report.complete + assert report.source_ids == (4,) + assert report.parsed_ids == (4,) + assert list(manager.records()) == [4] + assert manager.load_errors == [] + + +class FakeAccessor: + def __init__(self, _db_file): + self.schema_profile = SchemaProfile( + "unsuffixed-investment-columns", + "ZNUMBEROFSHARES", + "ZNUMBEROFSHARES", + "ZPRICEPERSHARE", + "ZPRICEPERSHARE", + ) + self.queries = [] + self.transaction_entries = 0 + self.closed = False + + @contextmanager + def read_transaction(self): + self.transaction_entries += 1 + yield + + def descendant_typenames(self, _roots): + return [] + + def query_objects(self, typenames): + self.queries.append(tuple(typenames)) + return [] + + def typename_for(self, _ent_id): + return None + + def read_category_assignments(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def read_refund_maps(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def read_tags_map(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def close(self): + self.closed = True + + +def test_api_supports_scoped_loads_and_json_safe_snapshot(monkeypatch) -> None: + import moneywiz_api.moneywiz_api as api_module + + monkeypatch.setattr(api_module, "DatabaseAccessor", FakeAccessor) + + with api_module.MoneywizApi( + "unused.sqlite", managers=("accounts", "transactions") + ) as api: + report = api.completeness().as_dict() + snapshot = api.snapshot().as_dict() + + assert list(report["managers"]) == ["accounts", "transactions"] + assert report["complete"] + assert snapshot["records"] == {"accounts": [], "transactions": []} + assert snapshot["schema_profile"]["profile_id"] == ( + "unsuffixed-investment-columns" + ) + assert api.accessor.transaction_entries == 1 + json.dumps(snapshot) + + assert api.accessor.closed + + +def test_api_preserves_default_eager_loading(monkeypatch) -> None: + import moneywiz_api.moneywiz_api as api_module + + monkeypatch.setattr(api_module, "DatabaseAccessor", FakeAccessor) + + with api_module.MoneywizApi("unused.sqlite") as api: + assert list(api.completeness().managers) == list(api.MANAGER_NAMES) + assert len(api.accessor.queries) == len(api.MANAGER_NAMES) + assert api.accessor.transaction_entries == 1 + + +def test_api_rejects_unloaded_or_unknown_manager(monkeypatch) -> None: + import moneywiz_api.moneywiz_api as api_module + + monkeypatch.setattr(api_module, "DatabaseAccessor", FakeAccessor) + api = api_module.MoneywizApi("unused.sqlite", managers=()) + + with pytest.raises(ValueError, match="manager not loaded"): + api.completeness(("accounts",)) + with pytest.raises(ValueError, match="unknown manager"): + api.load(("missing",)) + + api.close() + + +def test_json_safe_snapshot_formats_decimal(monkeypatch) -> None: + import moneywiz_api.moneywiz_api as api_module + + monkeypatch.setattr(api_module, "DatabaseAccessor", FakeAccessor) + api = api_module.MoneywizApi("unused.sqlite", managers=()) + api.account_manager.add( + type( + "SnapshotRecord", + (), + { + "id": 1, + "gid": "snapshot-record", + "display_order": 1, + "as_dict": lambda self: {"amount": Decimal("1.20")}, + }, + )() + ) + api._loaded_managers.add("accounts") + + assert api.snapshot().as_dict()["records"]["accounts"][0]["amount"] == "1.20" + api.close() diff --git a/tests/unit/test_relationship_completeness.py b/tests/unit/test_relationship_completeness.py new file mode 100644 index 0000000..f50b000 --- /dev/null +++ b/tests/unit/test_relationship_completeness.py @@ -0,0 +1,148 @@ +import sqlite3 + +import pytest + +from moneywiz_api.database_accessor import DatabaseAccessor +from moneywiz_api.managers.transaction_manager import TransactionManager +from moneywiz_api.read_result import RelationshipStorage + + +def create_relationship_schema(path, *, include_tables=True) -> None: + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.execute("CREATE TABLE ZSYNCOBJECT (Z_PK INTEGER, Z_ENT INTEGER)") + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", + [ + (2, "CategoryAssigment", 0, 0), + (8, "SyncObject", 0, 0), + (36, "Tag", 8, 0), + (37, "Transaction", 8, 0), + (50, "WithdrawRefundTransactionLink", 0, 0), + ], + ) + if include_tables: + connection.execute( + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER PRIMARY KEY, ZCATEGORY INTEGER, ZAMOUNT FLOAT, " + "ZTRANSACTION INTEGER)" + ) + connection.executemany( + "INSERT INTO ZCATEGORYASSIGMENT VALUES (?, ?, ?, ?)", + [(1, 10, 5.0, 100), (2, 11, None, 101), (3, 12, 2.0, None)], + ) + connection.execute( + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER PRIMARY KEY, ZREFUNDTRANSACTION INTEGER, " + "ZWITHDRAWTRANSACTION INTEGER)" + ) + connection.executemany( + "INSERT INTO ZWITHDRAWREFUNDTRANSACTIONLINK VALUES (?, ?, ?)", + [(4, 102, 103), (5, 104, None)], + ) + connection.execute( + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_37TAGS VALUES (?, ?)", + [(100, 200), (101, None)], + ) + connection.commit() + connection.close() + + +def test_shifted_relationship_layouts_report_valid_and_skipped_rows(tmp_path) -> None: + path = tmp_path / "relationships.sqlite" + create_relationship_schema(path) + + with DatabaseAccessor(path) as accessor: + categories, category_report = accessor.read_category_assignments() + refunds, refund_report = accessor.read_refund_maps() + tags, tag_report = accessor.read_tags_map() + + assert categories == {100: [(10, 5)]} + assert category_report.source_ids == (1, 2) + assert category_report.parsed_ids == (1,) + assert category_report.status == "partial" + assert refunds == {102: 103} + assert refund_report.source_ids == (4, 5) + assert refund_report.parsed_ids == (4,) + assert refund_report.status == "partial" + assert tags == {100: [200]} + assert tag_report.storage_name == "Z_37TAGS" + assert tag_report.source_ids == ("100:200", "row:1") + assert tag_report.parsed_ids == ("100:200",) + assert tag_report.status == "partial" + + manager_report = TransactionManager().load(accessor) + + assert manager_report.status == "partial" + assert not manager_report.complete + assert set(manager_report.relationships) == { + "category_assignments", + "refund_links", + "transaction_tags", + } + assert ( + manager_report.as_dict()["relationships"]["transaction_tags"]["storage_name"] + == "Z_37TAGS" + ) + + +@pytest.mark.parametrize( + "reader_name", + ["read_category_assignments", "read_refund_maps", "read_tags_map"], +) +def test_known_entity_with_missing_storage_is_unknown(tmp_path, reader_name) -> None: + path = tmp_path / "missing-storage.sqlite" + create_relationship_schema(path, include_tables=False) + + with DatabaseAccessor(path) as accessor: + data, report = getattr(accessor, reader_name)() + + assert data == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert report.status == "error" + assert not report.complete + + +def test_missing_optional_relationship_entities_are_absent(tmp_path) -> None: + path = tmp_path / "absent-relationships.sqlite" + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.execute("CREATE TABLE ZSYNCOBJECT (Z_PK INTEGER, Z_ENT INTEGER)") + connection.commit() + connection.close() + + with DatabaseAccessor(path) as accessor: + reports = [ + accessor.read_category_assignments()[1], + accessor.read_refund_maps()[1], + accessor.read_tags_map()[1], + ] + manager_report = TransactionManager().load(accessor) + + assert all(report.storage == RelationshipStorage.ABSENT for report in reports) + assert all(report.complete for report in reports) + assert manager_report.complete + + +def test_unknown_relationship_storage_propagates_to_manager(tmp_path) -> None: + path = tmp_path / "unknown-relationships.sqlite" + create_relationship_schema(path, include_tables=False) + + with DatabaseAccessor(path) as accessor: + manager_report = TransactionManager().load(accessor) + + assert not manager_report.complete + assert manager_report.status == "error" + assert all( + report.storage == RelationshipStorage.UNKNOWN + for report in manager_report.relationships.values() + ) diff --git a/tests/unit/test_schema_profiles.py b/tests/unit/test_schema_profiles.py index 3579411..27f4892 100644 --- a/tests/unit/test_schema_profiles.py +++ b/tests/unit/test_schema_profiles.py @@ -19,18 +19,32 @@ def test_detects_suffixed_fixture_profile() -> None: assert profile.profile_id == "suffixed-investment-columns" assert profile.holding_number_of_shares_column == "ZNUMBEROFSHARES1" assert profile.transaction_number_of_shares_column == "ZNUMBEROFSHARES1" - assert profile.price_per_share_column == "ZPRICEPERSHARE1" + assert profile.holding_price_per_share_column == "ZPRICEPERSHARE1" + assert profile.transaction_price_per_share_column == "ZPRICEPERSHARE1" def test_detects_unsuffixed_live_profile() -> None: - connection = make_connection("ZNUMBEROFSHARES", "ZPRICEPERSHARE", "ZPRICEPERSHARE1") + connection = make_connection("ZNUMBEROFSHARES", "ZPRICEPERSHARE") profile = detect_schema_profile(connection) assert profile.profile_id == "unsuffixed-investment-columns" assert profile.holding_number_of_shares_column == "ZNUMBEROFSHARES" assert profile.transaction_number_of_shares_column == "ZNUMBEROFSHARES" - assert profile.price_per_share_column == "ZPRICEPERSHARE" + assert profile.holding_price_per_share_column == "ZPRICEPERSHARE" + assert profile.transaction_price_per_share_column == "ZPRICEPERSHARE" + + +def test_detects_consumer_specific_mixed_price_columns() -> None: + connection = make_connection("ZNUMBEROFSHARES", "ZPRICEPERSHARE", "ZPRICEPERSHARE1") + + profile = detect_schema_profile(connection) + + assert profile.profile_id == "mixed-investment-columns" + assert profile.holding_number_of_shares_column == "ZNUMBEROFSHARES" + assert profile.transaction_number_of_shares_column == "ZNUMBEROFSHARES" + assert profile.holding_price_per_share_column == "ZPRICEPERSHARE" + assert profile.transaction_price_per_share_column == "ZPRICEPERSHARE1" def test_detects_mixed_investment_columns() -> None: @@ -41,7 +55,8 @@ def test_detects_mixed_investment_columns() -> None: assert profile.profile_id == "mixed-investment-columns" assert profile.holding_number_of_shares_column == "ZNUMBEROFSHARES" assert profile.transaction_number_of_shares_column == "ZNUMBEROFSHARES" - assert profile.price_per_share_column == "ZPRICEPERSHARE1" + assert profile.holding_price_per_share_column == "ZPRICEPERSHARE1" + assert profile.transaction_price_per_share_column == "ZPRICEPERSHARE1" def test_detects_consumer_specific_mixed_share_columns() -> None: @@ -54,10 +69,11 @@ def test_detects_consumer_specific_mixed_share_columns() -> None: assert profile.profile_id == "mixed-investment-columns" assert profile.holding_number_of_shares_column == "ZNUMBEROFSHARES" assert profile.transaction_number_of_shares_column == "ZNUMBEROFSHARES1" - assert profile.price_per_share_column == "ZPRICEPERSHARE1" + assert profile.holding_price_per_share_column == "ZPRICEPERSHARE1" + assert profile.transaction_price_per_share_column == "ZPRICEPERSHARE1" -def test_rejects_ambiguous_investment_column_profile() -> None: +def test_detects_consumer_specific_mixed_share_and_price_columns() -> None: connection = make_connection( "ZNUMBEROFSHARES", "ZNUMBEROFSHARES1", @@ -67,8 +83,11 @@ def test_rejects_ambiguous_investment_column_profile() -> None: profile = detect_schema_profile(connection) - assert profile.profile_id == "unknown" - assert not profile.is_known + assert profile.profile_id == "mixed-investment-columns" + assert profile.holding_number_of_shares_column == "ZNUMBEROFSHARES" + assert profile.transaction_number_of_shares_column == "ZNUMBEROFSHARES1" + assert profile.holding_price_per_share_column == "ZPRICEPERSHARE" + assert profile.transaction_price_per_share_column == "ZPRICEPERSHARE1" def test_decimal_alias_reads_both_profiles() -> None: diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 0000000..5b1b56d --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,26 @@ +import os +import time +from datetime import datetime, timezone + +import pytest + +from moneywiz_api.utils import get_date, get_datetime + + +@pytest.mark.skipif(not hasattr(time, "tzset"), reason="tzset is Unix-only") +def test_core_data_epoch_uses_utc_and_returns_naive_local_time(monkeypatch) -> None: + original_tz = os.environ.get("TZ") + monkeypatch.setenv("TZ", "America/New_York") + time.tzset() + try: + assert get_datetime(0) == datetime(2000, 12, 31, 19, 0) + summer = datetime(2024, 7, 1, 12, 0, tzinfo=timezone.utc) + core_data_value = get_date(summer) + assert get_datetime(core_data_value) == datetime(2024, 7, 1, 8, 0) + assert get_date(get_datetime(core_data_value)) == core_data_value + finally: + if original_tz is None: + monkeypatch.delenv("TZ", raising=False) + else: + monkeypatch.setenv("TZ", original_tz) + time.tzset() From b2ecd54d4e3d016d44b722a52c16e2de9a2bd61e Mon Sep 17 00:00:00 2001 From: MarcoMC Date: Sun, 13 Sep 2026 01:41:44 +0200 Subject: [PATCH 02/10] Enforce relationship completeness and optimization-safe validation Share exact tag-table shape classification across discovery and admission, reject metadata/storage contradictions, and preserve validation under optimized Python. Add synthetic shape matrices and optimization parity regressions. --- src/moneywiz_api/database_accessor.py | 141 +++++- src/moneywiz_api/model/account.py | 21 +- src/moneywiz_api/model/category.py | 7 +- src/moneywiz_api/model/investment_holding.py | 19 +- src/moneywiz_api/model/payee.py | 5 +- src/moneywiz_api/model/raw_data_handler.py | 13 +- src/moneywiz_api/model/record.py | 11 +- src/moneywiz_api/model/tag.py | 5 +- src/moneywiz_api/model/transaction.py | 364 ++++++++++---- src/moneywiz_api/validation.py | 4 + tests/unit/optimized_validation_probe.py | 434 +++++++++++++++++ tests/unit/test_optimized_validation.py | 130 +++++ tests/unit/test_relationship_completeness.py | 477 +++++++++++++++++++ 13 files changed, 1474 insertions(+), 157 deletions(-) create mode 100644 src/moneywiz_api/validation.py create mode 100644 tests/unit/optimized_validation_probe.py create mode 100644 tests/unit/test_optimized_validation.py diff --git a/src/moneywiz_api/database_accessor.py b/src/moneywiz_api/database_accessor.py index e3ab502..f7452be 100644 --- a/src/moneywiz_api/database_accessor.py +++ b/src/moneywiz_api/database_accessor.py @@ -1,9 +1,11 @@ import sqlite3 +import re from collections import defaultdict from contextlib import contextmanager from pathlib import Path from typing import Dict, List, Any, Callable, Tuple from decimal import Decimal +from enum import Enum from moneywiz_api.model.record import Record from moneywiz_api.model.investment_holding import InvestmentHolding @@ -20,6 +22,20 @@ ) from moneywiz_api.schema_profile import SchemaProfile, detect_schema_profile from moneywiz_api.types import ENT_ID, ID, GID +from moneywiz_api.validation import require_valid + + +TRANSACTION_TAG_TABLE_RE = re.compile(r"Z_\d+TAGS") +SUPPORTED_UNRELATED_TAG_TABLES = { + "Z_23TAGS": frozenset({"Z_23INFOCARDS5", "Z_35TAGS1"}), + "Z_31TAGS": frozenset({"Z_31SCHEDULEDTRANSACTIONS1", "Z_35TAGS2"}), +} + + +class _TagTableShape(Enum): + DIRECT = "direct" + UNRELATED = "unrelated" + AMBIGUOUS = "ambiguous" class DatabasePathError(ValueError): @@ -162,14 +178,78 @@ def _relationship_storage( table_name: str, required_columns: tuple[str, ...], ) -> RelationshipStorage: - if any(self.ent_for(name) is None for name in entity_names): + metadata_present = all(self.ent_for(name) is not None for name in entity_names) + table_present = self._table_exists(table_name) + if not metadata_present and not table_present: return RelationshipStorage.ABSENT - if not self._table_exists(table_name): + if not metadata_present or not table_present: return RelationshipStorage.UNKNOWN if not set(required_columns).issubset(self._table_columns(table_name)): return RelationshipStorage.UNKNOWN return RelationshipStorage.PRESENT + def _classify_tag_table(self, table_name: str) -> _TagTableShape: + """Classify an exact evidenced shape; a matching name alone is insufficient.""" + columns = frozenset(self._table_columns(table_name)) + if columns == SUPPORTED_UNRELATED_TAG_TABLES.get(table_name): + return _TagTableShape.UNRELATED + transaction_ent = self.ent_for("Transaction") + tag_ent = self.ent_for("Tag") + if ( + transaction_ent is not None + and tag_ent is not None + and table_name == f"Z_{transaction_ent}TAGS" + and columns == {f"Z_{transaction_ent}TRANSACTIONS", f"Z_{tag_ent}TAGS"} + ): + return _TagTableShape.DIRECT + return _TagTableShape.AMBIGUOUS + + def _transaction_tag_candidates(self) -> tuple[str, ...]: + """Return bounded tables that could store direct transaction-tag links.""" + rows = self._con.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name" + ).fetchall() + candidates: list[str] = [] + for row in rows: + table_name = row["name"] + if not isinstance( + table_name, str + ) or not TRANSACTION_TAG_TABLE_RE.fullmatch(table_name): + continue + if self._classify_tag_table(table_name) == _TagTableShape.UNRELATED: + continue + candidates.append(table_name) + return tuple(candidates) + + def _transaction_tag_storage( + self, + ) -> tuple[RelationshipStorage, str | None, str | None, str | None]: + transaction_ent = self.ent_for("Transaction") + tag_ent = self.ent_for("Tag") + candidates = self._transaction_tag_candidates() + metadata_count = sum(ent is not None for ent in (transaction_ent, tag_ent)) + if metadata_count == 0: + storage = ( + RelationshipStorage.UNKNOWN + if candidates + else RelationshipStorage.ABSENT + ) + return storage, candidates[0] if len(candidates) == 1 else None, None, None + if metadata_count == 1: + return RelationshipStorage.UNKNOWN, None, None, None + + table_name = f"Z_{transaction_ent}TAGS" + transaction_column = f"Z_{transaction_ent}TRANSACTIONS" + tag_column = f"Z_{tag_ent}TAGS" + if ( + candidates == (table_name,) + and self._classify_tag_table(table_name) == _TagTableShape.DIRECT + ): + storage = RelationshipStorage.PRESENT + else: + storage = RelationshipStorage.UNKNOWN + return storage, table_name, transaction_column, tag_column + @staticmethod def _relationship_error(exc: Exception) -> LoadErrorKind: if isinstance(exc, KeyError): @@ -284,11 +364,19 @@ def read_category_assignments( source_id = row.get("Z_PK") source_ids.append(source_id) try: - assert source_id is not None + require_valid( + source_id is not None, "category assignment ID is required" + ) category_id = row["ZCATEGORY"] transaction_id = row["ZTRANSACTION"] - assert category_id is not None - assert transaction_id is not None + require_valid( + category_id is not None, + "category assignment category endpoint is required", + ) + require_valid( + transaction_id is not None, + "category assignment transaction endpoint is required", + ) amount = RDH.get_decimal(row, "ZAMOUNT") transaction_map[transaction_id].append((category_id, amount)) except Exception as exc: @@ -342,11 +430,15 @@ def read_refund_maps( source_id = row.get("Z_PK") source_ids.append(source_id) try: - assert source_id is not None + require_valid(source_id is not None, "refund link ID is required") refund_id = row["ZREFUNDTRANSACTION"] withdraw_id = row["ZWITHDRAWTRANSACTION"] - assert refund_id is not None - assert withdraw_id is not None + require_valid( + refund_id is not None, "refund transaction endpoint is required" + ) + require_valid( + withdraw_id is not None, "withdraw transaction endpoint is required" + ) if refund_id in refund_to_withdraw: raise ValueError() refund_to_withdraw[refund_id] = withdraw_id @@ -378,27 +470,21 @@ def read_tags_map( self, ) -> tuple[Dict[ID, List[ID]], RelationshipLoadReport]: transactions_to_tags: Dict[ID, List[ID]] = defaultdict(list) - transaction_ent = self.ent_for("Transaction") - tag_ent = self.ent_for("Tag") - if transaction_ent is None or tag_ent is None: - return transactions_to_tags, RelationshipLoadReport( - storage=RelationshipStorage.ABSENT, - ) - - table_name = f"Z_{transaction_ent}TAGS" - transaction_column = f"Z_{transaction_ent}TRANSACTIONS" - tag_column = f"Z_{tag_ent}TAGS" - columns = (transaction_column, tag_column) - storage = self._relationship_storage( - entity_names=("Transaction", "Tag"), - table_name=table_name, - required_columns=columns, + storage, table_name, transaction_column, tag_column = ( + self._transaction_tag_storage() ) if storage != RelationshipStorage.PRESENT: return transactions_to_tags, RelationshipLoadReport( storage=storage, storage_name=table_name, ) + require_valid(table_name is not None, "transaction-tag table is required") + require_valid( + transaction_column is not None, + "transaction-tag transaction column is required", + ) + require_valid(tag_column is not None, "transaction-tag tag column is required") + columns = (transaction_column, tag_column) source_ids: list[str] = [] parsed_ids: list[str] = [] @@ -416,8 +502,13 @@ def read_tags_map( ) source_ids.append(source_id) try: - assert transaction_id is not None - assert tag_id is not None + require_valid( + transaction_id is not None, + "transaction-tag transaction endpoint is required", + ) + require_valid( + tag_id is not None, "transaction-tag tag endpoint is required" + ) transactions_to_tags[transaction_id].append(tag_id) except Exception as exc: skipped.append( diff --git a/src/moneywiz_api/model/account.py b/src/moneywiz_api/model/account.py index 7bf0f4c..aa42695 100644 --- a/src/moneywiz_api/model/account.py +++ b/src/moneywiz_api/model/account.py @@ -6,6 +6,7 @@ from moneywiz_api.types import ID from moneywiz_api.model.raw_data_handler import RawDataHandler as RDH from moneywiz_api.model.record import Record +from moneywiz_api.validation import require_valid @dataclass @@ -36,13 +37,17 @@ def __init__(self, row): self.user = row["ZUSER"] def validate(self) -> None: - assert self.display_order is not None - assert self.group_id is not None - assert self.name is not None - assert self.currency is not None - assert self.opening_balance is not None + require_valid( + self.display_order is not None, "account display order is required" + ) + require_valid(self.group_id is not None, "account group identity is required") + require_valid(self.name is not None, "account name is required") + require_valid(self.currency is not None, "account currency is required") + require_valid( + self.opening_balance is not None, "account opening balance is required" + ) # info is nullable in valid MoneyWiz stores. - assert self.user is not None + require_valid(self.user is not None, "account owner is required") @dataclass @@ -89,7 +94,9 @@ def __init__(self, row): def validate(self) -> None: super().validate() - assert self.statement_day is not None + require_valid( + self.statement_day is not None, "account statement day is required" + ) @dataclass diff --git a/src/moneywiz_api/model/category.py b/src/moneywiz_api/model/category.py index 10d9405..745b5f0 100644 --- a/src/moneywiz_api/model/category.py +++ b/src/moneywiz_api/model/category.py @@ -3,6 +3,7 @@ from moneywiz_api.model.record import Record from moneywiz_api.types import CategoryType, ID +from moneywiz_api.validation import require_valid @dataclass @@ -26,9 +27,9 @@ def __init__(self, row): # Fixes # Validate - assert self.name is not None, self.as_dict() - assert self.type is not None, self.as_dict() - assert self.user is not None, self.as_dict() + require_valid(self.name is not None, "category name is required") + require_valid(self.type is not None, "category type is required") + require_valid(self.user is not None, "category owner is required") @staticmethod def _convert_type(type_: Optional[int]) -> CategoryType: diff --git a/src/moneywiz_api/model/investment_holding.py b/src/moneywiz_api/model/investment_holding.py index ee65a33..e9d7aab 100644 --- a/src/moneywiz_api/model/investment_holding.py +++ b/src/moneywiz_api/model/investment_holding.py @@ -6,6 +6,7 @@ from moneywiz_api.model.record import Record from moneywiz_api.schema_profile import SchemaProfile from moneywiz_api.types import ID +from moneywiz_api.validation import require_valid @dataclass @@ -78,14 +79,20 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): self.validate() def validate(self): - assert self.account is not None, self.as_dict() - assert self.number_of_shares is not None, self.as_dict() + require_valid(self.account is not None, "holding account is required") + require_valid(self.number_of_shares is not None, "holding quantity is required") # price_per_share can be None when no current quote is stored. - assert self.symbol is not None, self.as_dict() - assert self.description is not None, self.as_dict() + require_valid(self.symbol is not None, "holding symbol is required") + require_valid(self.description is not None, "holding description is required") - assert self._investment_object_type is not None, self.as_dict() - assert self._cost_basis_of_missing_ob_shares is not None, self.as_dict() + require_valid( + self._investment_object_type is not None, + "holding investment object type is required", + ) + require_valid( + self._cost_basis_of_missing_ob_shares is not None, + "holding missing-share cost basis is required", + ) def as_dict(self) -> Dict[str, Any]: original = super().as_dict() diff --git a/src/moneywiz_api/model/payee.py b/src/moneywiz_api/model/payee.py index 836e303..b251c2a 100644 --- a/src/moneywiz_api/model/payee.py +++ b/src/moneywiz_api/model/payee.py @@ -2,6 +2,7 @@ from moneywiz_api.model.record import Record from moneywiz_api.types import ID +from moneywiz_api.validation import require_valid @dataclass @@ -21,5 +22,5 @@ def __init__(self, row): # Fixes # Validate - assert self.name is not None, self.as_dict() - assert self.user is not None, self.as_dict() + require_valid(self.name is not None, "payee name is required") + require_valid(self.user is not None, "payee owner is required") diff --git a/src/moneywiz_api/model/raw_data_handler.py b/src/moneywiz_api/model/raw_data_handler.py index 5ad237b..0d98d15 100644 --- a/src/moneywiz_api/model/raw_data_handler.py +++ b/src/moneywiz_api/model/raw_data_handler.py @@ -3,15 +3,16 @@ from decimal import Decimal from moneywiz_api.utils import get_datetime +from moneywiz_api.validation import require_valid class RawDataHandler: @staticmethod def get_datetime(row: Dict[str, Any], key: str) -> datetime: raw_value = row[key] - assert isinstance(raw_value, float) or isinstance(raw_value, int), ( - f"row['{key}'] = {row[key]}, is not a float or int, where row is: " - + str(RawDataHandler.filter_row(row)) + require_valid( + isinstance(raw_value, float) or isinstance(raw_value, int), + "datetime field must be numeric", ) return get_datetime(raw_value) @@ -26,9 +27,9 @@ def get_nullable_decimal(row: Dict[str, Any], key: str) -> Optional[Decimal]: @staticmethod def get_decimal(row: Dict[str, Any], key: str) -> Decimal: raw_value = row[key] - assert isinstance(raw_value, float) or isinstance(raw_value, int), ( - f"row['{key}'] = {row[key]}, is not a float or int, where row is: " - + str(RawDataHandler.filter_row(row)) + require_valid( + isinstance(raw_value, float) or isinstance(raw_value, int), + "decimal field must be numeric", ) return Decimal(str(raw_value)) diff --git a/src/moneywiz_api/model/record.py b/src/moneywiz_api/model/record.py index d35b99b..a5483af 100644 --- a/src/moneywiz_api/model/record.py +++ b/src/moneywiz_api/model/record.py @@ -4,6 +4,7 @@ from moneywiz_api.types import ID, ENT_ID from moneywiz_api.model.raw_data_handler import RawDataHandler as RDH +from moneywiz_api.validation import require_valid @dataclass @@ -24,11 +25,11 @@ def __init__(self, row): # Fixes # Validate - assert self._raw - assert self._ent - assert self._created_at - assert self.gid - assert self.id + require_valid(self._raw, "record source must not be empty") + require_valid(self._ent, "record entity must be set") + require_valid(self._created_at, "record creation date must be set") + require_valid(self.gid, "record global identity must be set") + require_valid(self.id, "record identity must be set") def ent(self) -> ENT_ID: return self._ent diff --git a/src/moneywiz_api/model/tag.py b/src/moneywiz_api/model/tag.py index 11b61d5..a5305a3 100644 --- a/src/moneywiz_api/model/tag.py +++ b/src/moneywiz_api/model/tag.py @@ -2,6 +2,7 @@ from moneywiz_api.model.record import Record from moneywiz_api.types import ID +from moneywiz_api.validation import require_valid @dataclass @@ -21,5 +22,5 @@ def __init__(self, row): # Fixes # Validate - assert self.name is not None, self.as_dict() - assert self.user is not None, self.as_dict() + require_valid(self.name is not None, "tag name is required") + require_valid(self.user is not None, "tag owner is required") diff --git a/src/moneywiz_api/model/transaction.py b/src/moneywiz_api/model/transaction.py index 2bf3771..651a8ac 100644 --- a/src/moneywiz_api/model/transaction.py +++ b/src/moneywiz_api/model/transaction.py @@ -10,6 +10,7 @@ from moneywiz_api.model.record import Record from moneywiz_api.schema_profile import SchemaProfile from moneywiz_api.types import ID +from moneywiz_api.validation import require_valid ABS_TOLERANCE = 0.001 @@ -38,10 +39,12 @@ def __init__(self, row): # Fixes # Validate - assert self.reconciled is not None - assert self.amount is not None - assert self.description is not None - assert self.datetime is not None + require_valid(self.reconciled is not None, "transaction state is required") + require_valid(self.amount is not None, "transaction amount is required") + require_valid( + self.description is not None, "transaction description is required" + ) + require_valid(self.datetime is not None, "transaction date is required") # self.notes can be None @@ -79,17 +82,29 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None, self.as_dict() - assert self.amount is not None, self.as_dict() + require_valid(self.account is not None, "deposit account is required") + require_valid(self.amount is not None, "deposit amount is required") # self.payee can be None - assert self.original_currency is not None, self.as_dict() - assert self.original_amount is not None, self.as_dict() + require_valid( + self.original_currency is not None, "deposit original currency is required" + ) + require_valid( + self.original_amount is not None, "deposit original amount is required" + ) - assert self.amount * self.original_amount > 0, self.as_dict() # Same sign + require_valid( + self.amount * self.original_amount > 0, + "deposit amounts must have the same sign", + ) if self.original_exchange_rate is not None: - assert self.amount == pytest.approx( - self.original_amount * self.original_exchange_rate, abs=ABS_TOLERANCE - ), self.as_dict() + require_valid( + self.amount + == pytest.approx( + self.original_amount * self.original_exchange_rate, + abs=ABS_TOLERANCE, + ), + "deposit exchange rate must match the amounts", + ) @dataclass @@ -134,15 +149,30 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None - assert self.from_investment_holding is not None - assert self.from_symbol - assert self.to_investment_holding is not None - assert self.to_symbol - assert self.from_number_of_shares <= 0 - assert self.to_number_of_shares >= 0 - assert self.original_fee is not None - assert self.original_fee_currency in [self.from_symbol, self.to_symbol] + require_valid(self.account is not None, "exchange account is required") + require_valid( + self.from_investment_holding is not None, + "exchange source holding is required", + ) + require_valid(self.from_symbol, "exchange source symbol is required") + require_valid( + self.to_investment_holding is not None, + "exchange destination holding is required", + ) + require_valid(self.to_symbol, "exchange destination symbol is required") + require_valid( + self.from_number_of_shares <= 0, + "exchange source quantity must not be positive", + ) + require_valid( + self.to_number_of_shares >= 0, + "exchange destination quantity must not be negative", + ) + require_valid(self.original_fee is not None, "exchange fee is required") + require_valid( + self.original_fee_currency in [self.from_symbol, self.to_symbol], + "exchange fee currency must match one holding", + ) @dataclass @@ -204,24 +234,41 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): self.validate() def validate(self): - assert self.account is not None - assert self.amount is not None - assert self.amount <= 0 - assert self.fee is not None - assert self.fee >= 0 + require_valid(self.account is not None, "investment buy account is required") + require_valid(self.amount is not None, "investment buy amount is required") + require_valid(self.amount <= 0, "investment buy amount must not be positive") + require_valid(self.fee is not None, "investment buy fee is required") + require_valid(self.fee >= 0, "investment buy fee must not be negative") # Either tiny (close to 0) or positive - assert ( + require_valid( abs(self.fee) == pytest.approx(0, abs=ABS_TOLERANCE) - or self.fee > ABS_TOLERANCE + or self.fee > ABS_TOLERANCE, + "investment buy fee must be tiny or positive", + ) + require_valid( + self.investment_holding is not None, + "investment buy holding is required", + ) + require_valid( + self.number_of_shares is not None, + "investment buy quantity is required", + ) + require_valid( + self.number_of_shares > 0, "investment buy quantity must be positive" + ) + require_valid( + self.price_per_share is not None, + "investment buy share price is required", + ) + require_valid( + self.price_per_share >= 0, + "investment buy share price must not be negative", + ) + require_valid( + -(self.number_of_shares * self.price_per_share + self.fee) + == pytest.approx(self.amount, abs=ABS_TOLERANCE), + "investment buy total must match amount", ) - assert self.investment_holding is not None - assert self.number_of_shares is not None - assert self.number_of_shares > 0 - assert self.price_per_share is not None - assert self.price_per_share >= 0 - assert -( - self.number_of_shares * self.price_per_share + self.fee - ) == pytest.approx(self.amount, abs=ABS_TOLERANCE) @dataclass @@ -273,25 +320,42 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): self.validate() def validate(self): - assert self.account is not None - assert self.amount is not None + require_valid(self.account is not None, "investment sell account is required") + require_valid(self.amount is not None, "investment sell amount is required") - assert self.fee is not None - assert self.fee >= 0 + require_valid(self.fee is not None, "investment sell fee is required") + require_valid(self.fee >= 0, "investment sell fee must not be negative") # Either tiny (close to 0) or positive - assert ( + require_valid( abs(self.fee) == pytest.approx(0, abs=ABS_TOLERANCE) - or self.fee > ABS_TOLERANCE + or self.fee > ABS_TOLERANCE, + "investment sell fee must be tiny or positive", ) - assert self.investment_holding is not None - assert self.number_of_shares is not None - assert self.number_of_shares > 0 - assert self.price_per_share is not None - assert self.price_per_share >= 0 - assert ( + require_valid( + self.investment_holding is not None, + "investment sell holding is required", + ) + require_valid( + self.number_of_shares is not None, + "investment sell quantity is required", + ) + require_valid( + self.number_of_shares > 0, "investment sell quantity must be positive" + ) + require_valid( + self.price_per_share is not None, + "investment sell share price is required", + ) + require_valid( + self.price_per_share >= 0, + "investment sell share price must not be negative", + ) + require_valid( self.number_of_shares * self.price_per_share - self.fee - ) == pytest.approx(self.amount, abs=ABS_TOLERANCE) + == pytest.approx(self.amount, abs=ABS_TOLERANCE), + "investment sell total must match amount", + ) @dataclass @@ -317,10 +381,11 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None - assert ( + require_valid(self.account is not None, "reconcile account is required") + require_valid( self.reconcile_amount is not None - or self.reconcile_number_of_shares is not None + or self.reconcile_number_of_shares is not None, + "reconcile amount or quantity is required", ) @@ -359,17 +424,28 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None - assert self.amount is not None - assert self.amount > 0 + require_valid(self.account is not None, "refund account is required") + require_valid(self.amount is not None, "refund amount is required") + require_valid(self.amount > 0, "refund amount must be positive") - assert self.original_currency is not None - assert self.original_amount is not None - assert self.original_amount > 0 + require_valid( + self.original_currency is not None, "refund original currency is required" + ) + require_valid( + self.original_amount is not None, "refund original amount is required" + ) + require_valid( + self.original_amount > 0, "refund original amount must be positive" + ) if self.original_exchange_rate is not None: - assert self.amount == pytest.approx( - self.original_amount * self.original_exchange_rate, abs=ABS_TOLERANCE + require_valid( + self.amount + == pytest.approx( + self.original_amount * self.original_exchange_rate, + abs=ABS_TOLERANCE, + ), + "refund exchange rate must match the amounts", ) @@ -437,28 +513,62 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None - assert self.amount is not None - assert self.amount > 0 - assert self.sender_account is not None - assert self.sender_transaction is not None - assert self.original_amount is not None - assert self.original_amount > 0 - assert self.original_currency is not None - assert self.sender_amount is not None - assert self.sender_amount <= 0 - assert self.sender_currency is not None + require_valid(self.account is not None, "transfer deposit account is required") + require_valid(self.amount is not None, "transfer deposit amount is required") + require_valid(self.amount > 0, "transfer deposit amount must be positive") + require_valid( + self.sender_account is not None, + "transfer deposit sender account is required", + ) + require_valid( + self.sender_transaction is not None, + "transfer deposit sender transaction is required", + ) + require_valid( + self.original_amount is not None, + "transfer deposit original amount is required", + ) + require_valid( + self.original_amount > 0, + "transfer deposit original amount must be positive", + ) + require_valid( + self.original_currency is not None, + "transfer deposit original currency is required", + ) + require_valid( + self.sender_amount is not None, + "transfer deposit sender amount is required", + ) + require_valid( + self.sender_amount <= 0, + "transfer deposit sender amount must not be positive", + ) + require_valid( + self.sender_currency is not None, + "transfer deposit sender currency is required", + ) if self.original_fee is not None and self.original_fee != 0: - assert self.original_fee_currency is not None + require_valid( + self.original_fee_currency is not None, + "transfer deposit fee currency is required", + ) - assert self.original_exchange_rate is not None + require_valid( + self.original_exchange_rate is not None, + "transfer deposit exchange rate is required", + ) # assert self.amount == self.original_amount # original_amount could be different with amount ZCURRENCYEXCHANGERATE is playing up - assert self.original_amount == pytest.approx( - -self.sender_amount * self.original_exchange_rate - - (self.original_fee or 0), - abs=ABS_TOLERANCE, + require_valid( + self.original_amount + == pytest.approx( + -self.sender_amount * self.original_exchange_rate + - (self.original_fee or 0), + abs=ABS_TOLERANCE, + ), + "transfer deposit exchange rate must match the amounts", ) @@ -520,27 +630,66 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None - assert self.amount is not None - assert self.amount < 0 - assert self.recipient_account is not None - assert self.recipient_transaction is not None - assert self.original_amount is not None - assert self.original_amount < 0 - assert self.original_currency is not None - assert self.recipient_amount is not None - assert self.recipient_amount > 0 - assert self.recipient_currency is not None + require_valid( + self.account is not None, "transfer withdrawal account is required" + ) + require_valid(self.amount is not None, "transfer withdrawal amount is required") + require_valid(self.amount < 0, "transfer withdrawal amount must be negative") + require_valid( + self.recipient_account is not None, + "transfer withdrawal recipient account is required", + ) + require_valid( + self.recipient_transaction is not None, + "transfer withdrawal recipient transaction is required", + ) + require_valid( + self.original_amount is not None, + "transfer withdrawal original amount is required", + ) + require_valid( + self.original_amount < 0, + "transfer withdrawal original amount must be negative", + ) + require_valid( + self.original_currency is not None, + "transfer withdrawal original currency is required", + ) + require_valid( + self.recipient_amount is not None, + "transfer withdrawal recipient amount is required", + ) + require_valid( + self.recipient_amount > 0, + "transfer withdrawal recipient amount must be positive", + ) + require_valid( + self.recipient_currency is not None, + "transfer withdrawal recipient currency is required", + ) if self.original_fee is not None and self.original_fee != 0: - assert self.original_fee_currency is not None + require_valid( + self.original_fee_currency is not None, + "transfer withdrawal fee currency is required", + ) - assert self.original_exchange_rate is not None + require_valid( + self.original_exchange_rate is not None, + "transfer withdrawal exchange rate is required", + ) - assert self.amount == self.original_amount - assert self.amount == pytest.approx( - -self.recipient_amount / self.original_exchange_rate, - abs=ABS_TOLERANCE, + require_valid( + self.amount == self.original_amount, + "transfer withdrawal amount must match original amount", + ) + require_valid( + self.amount + == pytest.approx( + -self.recipient_amount / self.original_exchange_rate, + abs=ABS_TOLERANCE, + ), + "transfer withdrawal exchange rate must match the amounts", ) @@ -582,15 +731,28 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None - assert self.amount is not None + require_valid(self.account is not None, "withdrawal account is required") + require_valid(self.amount is not None, "withdrawal amount is required") # self.payee can be None - assert self.original_currency is not None - assert self.original_amount is not None + require_valid( + self.original_currency is not None, + "withdrawal original currency is required", + ) + require_valid( + self.original_amount is not None, "withdrawal original amount is required" + ) - assert self.amount * self.original_amount > 0 + require_valid( + self.amount * self.original_amount > 0, + "withdrawal amounts must have the same sign", + ) if self.original_exchange_rate is not None: - assert self.amount == pytest.approx( - self.original_amount * self.original_exchange_rate, abs=ABS_TOLERANCE + require_valid( + self.amount + == pytest.approx( + self.original_amount * self.original_exchange_rate, + abs=ABS_TOLERANCE, + ), + "withdrawal exchange rate must match the amounts", ) diff --git a/src/moneywiz_api/validation.py b/src/moneywiz_api/validation.py new file mode 100644 index 0000000..260aa64 --- /dev/null +++ b/src/moneywiz_api/validation.py @@ -0,0 +1,4 @@ +def require_valid(condition: object, message: str) -> None: + """Raise the historical validation exception even under optimized Python.""" + if not condition: + raise AssertionError(message) diff --git a/tests/unit/optimized_validation_probe.py b/tests/unit/optimized_validation_probe.py new file mode 100644 index 0000000..e9643ce --- /dev/null +++ b/tests/unit/optimized_validation_probe.py @@ -0,0 +1,434 @@ +"""Emit runtime-validation behavior for normal and optimized Python processes.""" + +from __future__ import annotations + +import json +import sqlite3 +import sys +import tempfile +from decimal import Decimal +from pathlib import Path + +from moneywiz_api.database_accessor import DatabaseAccessor +from moneywiz_api.managers.payee_manager import PayeeManager +from moneywiz_api.managers.transaction_manager import TransactionManager +from moneywiz_api.model.account import CreditCardAccount +from moneywiz_api.model.category import Category +from moneywiz_api.model.investment_holding import InvestmentHolding +from moneywiz_api.model.payee import Payee +from moneywiz_api.model.raw_data_handler import RawDataHandler +from moneywiz_api.model.record import Record +from moneywiz_api.model.tag import Tag +from moneywiz_api.model.transaction import ( + DepositTransaction, + InvestmentBuyTransaction, + TransferWithdrawTransaction, + WithdrawTransaction, +) +from moneywiz_api.schema_profile import SchemaProfile + + +PROFILE = SchemaProfile( + "unsuffixed-investment-columns", + "ZNUMBEROFSHARES", + "ZNUMBEROFSHARES", + "ZPRICEPERSHARE", + "ZPRICEPERSHARE", +) + + +def normalize(value): + if isinstance(value, Decimal): + return str(value) + if hasattr(value, "isoformat"): + return value.isoformat() + if isinstance(value, dict): + return {str(key): normalize(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [normalize(item) for item in value] + return value + + +def capture(operation): + try: + value = operation() + except Exception as exc: + return { + "status": "error", + "type": type(exc).__name__, + "message": str(exc), + } + return {"status": "ok", "value": normalize(value)} + + +def common_row(**overrides): + row = { + "Z_ENT": 1, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": "record-1", + "Z_PK": 1, + } + row.update(overrides) + return row + + +def account_row(**overrides): + row = { + **common_row(Z_ENT=13, ZGID="account-1"), + "ZDISPLAYORDER": 1, + "ZGROUPID": 1, + "ZNAME": "Card", + "ZCURRENCYNAME": "EUR", + "ZOPENINGBALANCE": 0.0, + "ZINFO": None, + "ZUSER": 1, + "ZSTATEMENTENDDAY": 15, + } + row.update(overrides) + return row + + +def transaction_row(**overrides): + row = { + **common_row(Z_ENT=37, ZGID="transaction-1"), + "ZRECONCILED": 0, + "ZAMOUNT1": 20.0, + "ZDESC2": "Transaction", + "ZDATE1": 0.0, + "ZNOTES1": None, + "ZACCOUNT2": 1, + "ZPAYEE2": None, + "ZORIGINALCURRENCY": "EUR", + "ZORIGINALAMOUNT": 10.0, + "ZORIGINALEXCHANGERATE": 2.0, + } + row.update(overrides) + return row + + +def holding_row(**overrides): + row = { + **common_row(Z_ENT=24, ZGID="holding-1"), + "ZINVESTMENTACCOUNT": 1, + "ZOPENNINGNUMBEROFSHARES": None, + "ZNUMBEROFSHARES": 2.0, + "ZPRICEPERSHARE": 10.0, + "ZSYMBOL": "ACME", + "ZHOLDINGTYPE": None, + "ZDESC": "Acme", + "ZISPRICEPERSHAREAVAILABLEONLINE": 0, + "ZINVESTMENTOBJECTTYPE": 0, + "ZCOSTBASISOFMISSINGOBSHARES": 0.0, + } + row.update(overrides) + return row + + +def buy_row(**overrides): + row = { + **transaction_row(Z_ENT=40, ZGID="buy-1", ZAMOUNT1=-21.0), + "ZFEE2": 1.0, + "ZINVESTMENTHOLDING": 2, + "ZNUMBEROFSHARES": 2.0, + "ZPRICEPERSHARE": 10.0, + } + row.update(overrides) + return row + + +def transfer_withdraw_row(**overrides): + row = { + **transaction_row(Z_ENT=46, ZGID="transfer-withdraw-1", ZAMOUNT1=-10.0), + "ZRECIPIENTACCOUNT1": 2, + "ZRECIPIENTTRANSACTION": 3, + "ZORIGINALAMOUNT": 0.0, + "ZORIGINALRECIPIENTAMOUNT": 10.0, + "ZORIGINALRECIPIENTCURRENCY": "USD", + "ZORIGINALFEE": None, + "ZORIGINALFEECURRENCY": None, + "ZORIGINALEXCHANGERATE": 0.0, + } + row.update(overrides) + return row + + +class StaticCursor: + def __init__(self, row): + self.row = row + + def execute(self, _query, _parameters): + return self + + def fetchone(self): + return self.row + + +class StaticConnection: + def __init__(self, row): + self.row = row + + def cursor(self): + return StaticCursor(self.row) + + +def accessor_for(row): + accessor = DatabaseAccessor.__new__(DatabaseAccessor) + accessor._con = StaticConnection(row) + return accessor + + +class PayeeAccessor: + def query_objects(self, _typenames): + return [ + { + **common_row(Z_ENT=28, ZGID="payee-valid", Z_PK=10), + "ZNAME5": "A", + "ZUSER7": 1, + }, + { + **common_row(Z_ENT=28, ZGID="payee-invalid", Z_PK=11), + "ZNAME5": "B", + "ZUSER7": None, + }, + ] + + def typename_for(self, _ent_id): + return "Payee" + + +def named_entities_probe(): + rows = { + "payee_valid": ( + Payee, + {**common_row(Z_ENT=28), "ZNAME5": "Payee", "ZUSER7": 1}, + ), + "payee_invalid": ( + Payee, + {**common_row(Z_ENT=28), "ZNAME5": "Payee", "ZUSER7": None}, + ), + "tag_valid": ( + Tag, + {**common_row(Z_ENT=35), "ZNAME6": "Tag", "ZUSER8": 1}, + ), + "tag_invalid": ( + Tag, + {**common_row(Z_ENT=35), "ZNAME6": "Tag", "ZUSER8": None}, + ), + "category_valid": ( + Category, + { + **common_row(Z_ENT=19), + "ZNAME2": "Category", + "ZPARENTCATEGORY": None, + "ZTYPE2": 1, + "ZUSER3": 1, + }, + ), + "category_invalid": ( + Category, + { + **common_row(Z_ENT=19), + "ZNAME2": "Category", + "ZPARENTCATEGORY": None, + "ZTYPE2": 1, + "ZUSER3": None, + }, + ), + } + return { + name: capture(lambda constructor=constructor, row=row: constructor(row).name) + for name, (constructor, row) in rows.items() + } + + +def relationship_probe(): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "relationships.sqlite" + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.execute("CREATE TABLE ZSYNCOBJECT (Z_PK INTEGER, Z_ENT INTEGER)") + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", + [ + (2, "CategoryAssigment", 0, 0), + (36, "Tag", 0, 0), + (37, "Transaction", 0, 0), + (50, "WithdrawRefundTransactionLink", 0, 0), + ], + ) + connection.execute( + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT)" + ) + connection.executemany( + "INSERT INTO ZCATEGORYASSIGMENT VALUES (?, ?, ?, ?)", + [ + (1, 10, 100, 5.0), + (2, None, 101, 2.0), + (3, 12, 102, "PRIVATE_PAYLOAD"), + (4, 13, None, 4.0), + ], + ) + connection.execute( + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, ZWITHDRAWTRANSACTION INTEGER)" + ) + connection.executemany( + "INSERT INTO ZWITHDRAWREFUNDTRANSACTIONLINK VALUES (?, ?, ?)", + [(5, 103, 104), (6, 105, None)], + ) + connection.execute( + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_37TAGS VALUES (?, ?)", [(100, 200), (101, None)] + ) + connection.commit() + connection.close() + + with DatabaseAccessor(path) as accessor: + categories, category_report = accessor.read_category_assignments() + refunds, refund_report = accessor.read_refund_maps() + tags, tag_report = accessor.read_tags_map() + aggregate_report = TransactionManager().load(accessor) + return normalize( + { + "categories": categories, + "category_report": category_report.as_dict(), + "refunds": refunds, + "refund_report": refund_report.as_dict(), + "tags": tags, + "tag_report": tag_report.as_dict(), + "aggregate": aggregate_report.as_dict(), + } + ) + + +def sentinel_probe(): + original_payee_as_dict = Payee.as_dict + original_deposit_as_dict = DepositTransaction.as_dict + + def reject_serialization(_self): + raise RuntimeError("PRIVATE_PAYLOAD_SERIALIZED") + + Payee.as_dict = reject_serialization + DepositTransaction.as_dict = reject_serialization + try: + payee = capture( + lambda: Payee( + { + **common_row(Z_ENT=28), + "ZNAME5": "Payee", + "ZUSER7": None, + } + ) + ) + deposit = capture(lambda: DepositTransaction(transaction_row(ZACCOUNT2=None))) + finally: + Payee.as_dict = original_payee_as_dict + DepositTransaction.as_dict = original_deposit_as_dict + return {"payee": payee, "deposit": deposit} + + +def main(): + manager = PayeeManager() + manager_report = manager.load(PayeeAccessor()) + valid_account = accessor_for(account_row()).get_record(1, CreditCardAccount) + valid_deposit = DepositTransaction(transaction_row()) + zero_rate_deposit = DepositTransaction( + transaction_row(ZAMOUNT1=10.0, ZORIGINALAMOUNT=10.0, ZORIGINALEXCHANGERATE=0.0) + ) + corrected_withdrawal = WithdrawTransaction( + transaction_row( + Z_ENT=47, + ZGID="withdraw-1", + ZAMOUNT1=-10.0, + ZORIGINALAMOUNT=10.0, + ZORIGINALEXCHANGERATE=0.0, + ) + ) + output = { + "optimize": sys.flags.optimize, + "cases": { + "decimal_valid": capture( + lambda: RawDataHandler.get_decimal({"value": 1.25}, "value") + ), + "decimal_string": capture( + lambda: RawDataHandler.get_decimal( + {"value": "PRIVATE_PAYLOAD"}, "value" + ) + ), + "date_valid": capture( + lambda: RawDataHandler.get_datetime({"value": 0.0}, "value") + ), + "date_string": capture( + lambda: RawDataHandler.get_datetime( + {"value": "PRIVATE_PAYLOAD"}, "value" + ) + ), + "record_valid": capture(lambda: Record(common_row()).gid), + "record_invalid_identity": capture(lambda: Record(common_row(ZGID=""))), + "account_nullable_info": capture( + lambda: { + "info": valid_account.info, + "statement_day": valid_account.statement_day, + } + ), + "account_invalid_name_public": capture( + lambda: accessor_for(account_row(ZNAME=None)).get_record_by_gid( + "account-1", CreditCardAccount + ) + ), + "named_entities": named_entities_probe(), + "manager": manager_report.as_dict(), + "holding_valid": capture( + lambda: InvestmentHolding(holding_row(), PROFILE).number_of_shares + ), + "holding_missing_quantity": capture( + lambda: InvestmentHolding(holding_row(ZNUMBEROFSHARES=None), PROFILE) + ), + "deposit_valid": capture( + lambda: { + "amount": valid_deposit.amount, + "rate": valid_deposit.original_exchange_rate, + } + ), + "deposit_invalid_sign": capture( + lambda: DepositTransaction(transaction_row(ZORIGINALAMOUNT=-10.0)) + ), + "deposit_invalid_fx": capture( + lambda: DepositTransaction(transaction_row(ZAMOUNT1=20.01)) + ), + "deposit_zero_rate": capture( + lambda: zero_rate_deposit.original_exchange_rate + ), + "investment_buy_valid_fee": capture( + lambda: { + "amount": InvestmentBuyTransaction(buy_row(), PROFILE).amount, + "fee": InvestmentBuyTransaction(buy_row(), PROFILE).fee, + } + ), + "investment_buy_invalid_quantity": capture( + lambda: InvestmentBuyTransaction(buy_row(ZNUMBEROFSHARES=0.0), PROFILE) + ), + "investment_buy_invalid_total": capture( + lambda: InvestmentBuyTransaction(buy_row(ZAMOUNT1=-20.0), PROFILE) + ), + "withdrawal_sign_fix": capture( + lambda: corrected_withdrawal.original_amount + ), + "transfer_zero_rate": capture( + lambda: TransferWithdrawTransaction(transfer_withdraw_row()) + ), + "relationships": relationship_probe(), + "sentinel": sentinel_probe(), + }, + } + print(json.dumps(normalize(output), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_optimized_validation.py b/tests/unit/test_optimized_validation.py new file mode 100644 index 0000000..95ae9f5 --- /dev/null +++ b/tests/unit/test_optimized_validation.py @@ -0,0 +1,130 @@ +import ast +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +PROBE = Path(__file__).with_name("optimized_validation_probe.py") +RUNTIME_SOURCE = Path(__file__).parents[2] / "src" / "moneywiz_api" + + +def run_probe(*options, optimize_environment=None): + environment = os.environ.copy() + environment.pop("PYTHONOPTIMIZE", None) + source_path = str(RUNTIME_SOURCE.parent) + existing_pythonpath = environment.get("PYTHONPATH") + environment["PYTHONPATH"] = ( + f"{source_path}{os.pathsep}{existing_pythonpath}" + if existing_pythonpath + else source_path + ) + if optimize_environment is not None: + environment["PYTHONOPTIMIZE"] = optimize_environment + completed = subprocess.run( + [sys.executable, *options, str(PROBE)], + check=True, + capture_output=True, + text=True, + env=environment, + ) + return json.loads(completed.stdout) + + +@pytest.fixture(scope="module") +def optimization_results(): + return [ + run_probe(), + run_probe("-O"), + run_probe("-OO"), + run_probe(optimize_environment="1"), + ] + + +def test_validation_results_are_optimization_invariant(optimization_results) -> None: + assert [result["optimize"] for result in optimization_results] == [0, 1, 2, 1] + baseline = optimization_results[0]["cases"] + assert all(result["cases"] == baseline for result in optimization_results[1:]) + + +def test_probe_covers_success_error_and_completeness_paths( + optimization_results, +) -> None: + cases = optimization_results[0]["cases"] + + assert cases["decimal_valid"] == {"status": "ok", "value": "1.25"} + assert cases["decimal_string"] == { + "status": "error", + "type": "AssertionError", + "message": "decimal field must be numeric", + } + assert cases["date_string"]["type"] == "AssertionError" + assert cases["record_invalid_identity"]["type"] == "AssertionError" + assert cases["account_nullable_info"]["status"] == "ok" + assert cases["account_invalid_name_public"]["type"] == "AssertionError" + + named_entities = cases["named_entities"] + assert all( + named_entities[name]["status"] == "ok" + for name in ("payee_valid", "tag_valid", "category_valid") + ) + assert all( + named_entities[name]["type"] == "AssertionError" + for name in ("payee_invalid", "tag_invalid", "category_invalid") + ) + assert cases["manager"]["parsed_count"] == 1 + assert cases["manager"]["skipped"][0]["error"] == "validation" + assert cases["holding_missing_quantity"]["type"] == "AssertionError" + assert cases["deposit_invalid_sign"]["type"] == "AssertionError" + assert cases["deposit_invalid_fx"]["type"] == "AssertionError" + assert cases["deposit_zero_rate"] == {"status": "ok", "value": None} + assert cases["investment_buy_invalid_quantity"]["type"] == "AssertionError" + assert cases["investment_buy_invalid_total"]["type"] == "AssertionError" + assert cases["transfer_zero_rate"]["type"] == "ValueError" + + relationships = cases["relationships"] + assert relationships["category_report"]["parsed_count"] == 1 + assert [item["error"] for item in relationships["category_report"]["skipped"]] == [ + "validation", + "validation", + ] + assert relationships["refund_report"]["parsed_count"] == 1 + assert relationships["tag_report"]["parsed_count"] == 1 + assert not relationships["aggregate"]["complete"] + + serialized = json.dumps(cases["sentinel"], sort_keys=True) + assert "PRIVATE_PAYLOAD_SERIALIZED" not in serialized + assert all( + outcome["type"] == "AssertionError" for outcome in cases["sentinel"].values() + ) + assert "PRIVATE_PAYLOAD" not in json.dumps(cases, sort_keys=True) + + +def test_runtime_sources_contain_no_active_assert_statements() -> None: + active_asserts = [] + dynamic_validation_messages = [] + for source_path in sorted(RUNTIME_SOURCE.rglob("*.py")): + tree = ast.parse(source_path.read_text(), filename=str(source_path)) + active_asserts.extend( + (source_path.relative_to(RUNTIME_SOURCE), node.lineno) + for node in ast.walk(tree) + if isinstance(node, ast.Assert) + ) + dynamic_validation_messages.extend( + (source_path.relative_to(RUNTIME_SOURCE), node.lineno) + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "require_valid" + and ( + len(node.args) < 2 + or not isinstance(node.args[1], ast.Constant) + or not isinstance(node.args[1].value, str) + ) + ) + + assert active_asserts == [] + assert dynamic_validation_messages == [] diff --git a/tests/unit/test_relationship_completeness.py b/tests/unit/test_relationship_completeness.py index f50b000..bad088b 100644 --- a/tests/unit/test_relationship_completeness.py +++ b/tests/unit/test_relationship_completeness.py @@ -7,6 +7,14 @@ from moneywiz_api.read_result import RelationshipStorage +SCHEDULED_TAG_TABLE = ( + "CREATE TABLE Z_31TAGS (Z_31SCHEDULEDTRANSACTIONS1 INTEGER, Z_35TAGS2 INTEGER)" +) +INFO_CARD_TAG_TABLE = ( + "CREATE TABLE Z_23TAGS (Z_23INFOCARDS5 INTEGER, Z_35TAGS1 INTEGER)" +) + + def create_relationship_schema(path, *, include_tables=True) -> None: connection = sqlite3.connect(path) connection.execute( @@ -54,6 +62,94 @@ def create_relationship_schema(path, *, include_tables=True) -> None: connection.close() +def create_custom_relationship_schema(path, metadata=(), tables=()) -> None: + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.execute("CREATE TABLE ZSYNCOBJECT (Z_PK INTEGER, Z_ENT INTEGER)") + connection.executemany("INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", metadata) + for statement in tables: + connection.execute(statement) + connection.commit() + connection.close() + + +def read_tags_and_aggregate(path): + with DatabaseAccessor(path) as accessor: + tags, report = accessor.read_tags_map() + manager_report = TransactionManager().load(accessor) + return tags, report, manager_report + + +@pytest.mark.parametrize("populated", [False, True], ids=["empty", "populated"]) +@pytest.mark.parametrize( + "metadata", + [ + (), + ((37, "Transaction", 0, 0),), + ((36, "Tag", 0, 0),), + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + ], + ids=["absent", "transaction-only", "tag-only", "full"], +) +@pytest.mark.parametrize( + ("table_name", "include_expected"), + [("Z_37TAGS", False), ("Z_99TAGS", False), ("Z_99TAGS", True)], + ids=["expected", "alternate", "alternate-beside-direct"], +) +@pytest.mark.parametrize( + "columns", + [ + ("Z_37TRANSACTIONS",), + ("Z_36TAGS",), + ("Z_37TRANSACTIONS", "Z_36TAGS", "Z_37SCHEDULEDTRANSACTIONS1"), + ("Z_37TRANSACTIONS", "Z_36TAGS", "Z_37INFOCARDS5"), + ("Z_37TRANSACTIONS", "Z_36TAGS", "Z_99TRANSACTIONS"), + ("Z_37TRANSACTIONS", "Z_36TAGS", "Z_99TAGS"), + ("Z_37TRANSACTIONS", "Z_36TAGS", "Z_PK"), + ], + ids=[ + "missing-tag", + "missing-transaction", + "extra-scheduled", + "extra-info-card", + "extra-transaction", + "extra-tag", + "extra-auxiliary", + ], +) +def test_ambiguous_direct_tag_shape_is_unknown( + tmp_path, populated, metadata, table_name, include_expected, columns +) -> None: + path = tmp_path / "ambiguous-direct-tags.sqlite" + definition = ", ".join(f"{column} INTEGER" for column in columns) + tables = [f"CREATE TABLE {table_name} ({definition})"] + if include_expected: + tables.append( + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" + ) + create_custom_relationship_schema(path, metadata=metadata, tables=tables) + if populated: + with sqlite3.connect(path) as connection: + connection.execute( + f"INSERT INTO {table_name} VALUES ({', '.join('?' for _ in columns)})", + tuple(100 * (index + 1) for index in range(len(columns))), + ) + + tags, report, manager_report = read_tags_and_aggregate(path) + + assert report.storage == RelationshipStorage.UNKNOWN + assert tags == {} + assert not report.complete + assert not manager_report.complete + assert ( + manager_report.relationships["transaction_tags"].storage + == RelationshipStorage.UNKNOWN + ) + + def test_shifted_relationship_layouts_report_valid_and_skipped_rows(tmp_path) -> None: path = tmp_path / "relationships.sqlite" create_relationship_schema(path) @@ -146,3 +242,384 @@ def test_unknown_relationship_storage_propagates_to_manager(tmp_path) -> None: report.storage == RelationshipStorage.UNKNOWN for report in manager_report.relationships.values() ) + + +@pytest.mark.parametrize( + ("reader_name", "table_statement"), + [ + ( + "read_category_assignments", + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)", + ), + ( + "read_refund_maps", + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, " + "ZWITHDRAWTRANSACTION INTEGER)", + ), + ( + "read_tags_map", + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + ), + ], +) +def test_physical_storage_without_metadata_is_unknown( + tmp_path, reader_name, table_statement +) -> None: + path = tmp_path / f"{reader_name}-physical-only.sqlite" + create_custom_relationship_schema(path, tables=(table_statement,)) + + with DatabaseAccessor(path) as accessor: + data, report = getattr(accessor, reader_name)() + + assert data == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert report.status == "error" + assert not report.complete + + +@pytest.mark.parametrize( + ("reader_name", "metadata", "table_statement"), + [ + ( + "read_category_assignments", + ((2, "CategoryAssigment", 0, 0),), + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)", + ), + ( + "read_refund_maps", + ((50, "WithdrawRefundTransactionLink", 0, 0),), + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, " + "ZWITHDRAWTRANSACTION INTEGER)", + ), + ( + "read_tags_map", + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + ), + ], +) +def test_valid_empty_relationship_storage_is_present( + tmp_path, reader_name, metadata, table_statement +) -> None: + path = tmp_path / f"{reader_name}-empty.sqlite" + create_custom_relationship_schema( + path, metadata=metadata, tables=(table_statement,) + ) + + with DatabaseAccessor(path) as accessor: + data, report = getattr(accessor, reader_name)() + + assert data == {} + assert report.storage == RelationshipStorage.PRESENT + assert report.complete + assert report.status == "complete" + + +@pytest.mark.parametrize( + ("reader_name", "metadata", "table_statement"), + [ + ( + "read_category_assignments", + ((2, "CategoryAssigment", 0, 0),), + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER)", + ), + ( + "read_refund_maps", + ((50, "WithdrawRefundTransactionLink", 0, 0),), + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER)", + ), + ( + "read_tags_map", + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER)", + ), + ], +) +def test_relationship_storage_with_missing_columns_is_unknown( + tmp_path, reader_name, metadata, table_statement +) -> None: + path = tmp_path / f"{reader_name}-missing-column.sqlite" + create_custom_relationship_schema( + path, metadata=metadata, tables=(table_statement,) + ) + + with DatabaseAccessor(path) as accessor: + data, report = getattr(accessor, reader_name)() + + assert data == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert not report.complete + + +@pytest.mark.parametrize( + "metadata", + [ + ((37, "Transaction", 0, 0),), + ((36, "Tag", 0, 0),), + ], +) +def test_partial_transaction_tag_metadata_is_unknown(tmp_path, metadata) -> None: + path = tmp_path / "partial-tag-metadata.sqlite" + create_custom_relationship_schema(path, metadata=metadata) + + with DatabaseAccessor(path) as accessor: + tags, report = accessor.read_tags_map() + + assert tags == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert not report.complete + + +def test_scheduled_tag_storage_is_not_a_transaction_tag_candidate(tmp_path) -> None: + path = tmp_path / "scheduled-tags.sqlite" + create_custom_relationship_schema(path, tables=(SCHEDULED_TAG_TABLE,)) + + with DatabaseAccessor(path) as accessor: + tags, absent_report = accessor.read_tags_map() + + assert tags == {} + assert absent_report.storage == RelationshipStorage.ABSENT + assert absent_report.complete + + path_with_metadata = tmp_path / "scheduled-and-direct-tags.sqlite" + direct_table = "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" + create_custom_relationship_schema( + path_with_metadata, + metadata=((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + tables=(SCHEDULED_TAG_TABLE, direct_table), + ) + + with DatabaseAccessor(path_with_metadata) as accessor: + _, present_report = accessor.read_tags_map() + + assert present_report.storage == RelationshipStorage.PRESENT + assert present_report.complete + + +@pytest.mark.parametrize( + ("alternate_table", "include_expected"), + [ + ( + "CREATE TABLE Z_36TAGS (Z_36TRANSACTIONS INTEGER, Z_35TAGS INTEGER)", + False, + ), + ( + "CREATE TABLE Z_99TAGS (Z_99TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + True, + ), + ], +) +def test_wrong_or_conflicting_transaction_tag_layout_is_unknown( + tmp_path, alternate_table, include_expected +) -> None: + path = tmp_path / "conflicting-tags.sqlite" + expected_table = ( + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" + ) + tables = ( + (expected_table, alternate_table) if include_expected else (alternate_table,) + ) + create_custom_relationship_schema( + path, + metadata=((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + tables=tables, + ) + + with DatabaseAccessor(path) as accessor: + tags, report = accessor.read_tags_map() + manager_report = TransactionManager().load(accessor) + + assert tags == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert not report.complete + assert not manager_report.complete + assert ( + manager_report.relationships["transaction_tags"].storage + == RelationshipStorage.UNKNOWN + ) + + +@pytest.mark.parametrize("populated", [False, True]) +@pytest.mark.parametrize( + "metadata", + [ + (), + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + ((37, "Transaction", 0, 0),), + ((36, "Tag", 0, 0),), + ], +) +def test_malformed_numbered_tag_table_is_always_unknown( + tmp_path, populated, metadata +) -> None: + path = tmp_path / "malformed-numbered-tags.sqlite" + create_custom_relationship_schema( + path, + metadata=metadata, + tables=("CREATE TABLE Z_37TAGS (Z_PK INTEGER)",), + ) + if populated: + connection = sqlite3.connect(path) + connection.execute("INSERT INTO Z_37TAGS VALUES (1)") + connection.commit() + connection.close() + + tags, report, manager_report = read_tags_and_aggregate(path) + + assert tags == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert not report.complete + assert not manager_report.complete + assert ( + manager_report.relationships["transaction_tags"].storage + == RelationshipStorage.UNKNOWN + ) + + +def test_valid_direct_tags_with_malformed_alternate_are_unknown(tmp_path) -> None: + path = tmp_path / "valid-and-malformed-tags.sqlite" + create_custom_relationship_schema( + path, + metadata=((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + tables=( + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + "CREATE TABLE Z_99TAGS (Z_PK INTEGER)", + ), + ) + + tags, report, manager_report = read_tags_and_aggregate(path) + + assert tags == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert not report.complete + assert not manager_report.complete + + +@pytest.mark.parametrize("populated", [False, True]) +@pytest.mark.parametrize( + "metadata", + [ + (), + ((37, "Transaction", 0, 0),), + ((36, "Tag", 0, 0),), + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + ], +) +@pytest.mark.parametrize("unrelated_table", [SCHEDULED_TAG_TABLE, INFO_CARD_TAG_TABLE]) +def test_complete_unrelated_tag_shape_establishes_absence( + tmp_path, unrelated_table, metadata, populated +) -> None: + path = tmp_path / "unrelated-tags.sqlite" + create_custom_relationship_schema( + path, metadata=metadata, tables=(unrelated_table,) + ) + if populated: + table_name = unrelated_table.split()[2] + with sqlite3.connect(path) as connection: + connection.execute(f"INSERT INTO {table_name} VALUES (100, 200)") + + tags, report, manager_report = read_tags_and_aggregate(path) + + assert tags == {} + expected_storage = ( + RelationshipStorage.UNKNOWN if metadata else RelationshipStorage.ABSENT + ) + assert report.storage == expected_storage + assert report.complete == (not metadata) + assert manager_report.complete == (not metadata) + assert manager_report.relationships["transaction_tags"].storage == expected_storage + + +@pytest.mark.parametrize("populated", [False, True]) +@pytest.mark.parametrize("include_unrelated", [False, True]) +@pytest.mark.parametrize(("transaction_ent", "tag_ent"), [(36, 35), (37, 36), (61, 58)]) +def test_unrelated_controls_do_not_conflict_with_valid_direct_tags( + tmp_path, populated, include_unrelated, transaction_ent, tag_ent +) -> None: + path = tmp_path / "unrelated-controls-and-direct-tags.sqlite" + table_name = f"Z_{transaction_ent}TAGS" + tables = [ + f"CREATE TABLE {table_name} (Z_{transaction_ent}TRANSACTIONS INTEGER, Z_{tag_ent}TAGS INTEGER)" + ] + if include_unrelated: + tables.extend((INFO_CARD_TAG_TABLE, SCHEDULED_TAG_TABLE)) + create_custom_relationship_schema( + path, + metadata=((tag_ent, "Tag", 0, 0), (transaction_ent, "Transaction", 0, 0)), + tables=tables, + ) + if populated: + with sqlite3.connect(path) as connection: + connection.execute(f"INSERT INTO {table_name} VALUES (100, 200)") + if include_unrelated: + connection.execute("INSERT INTO Z_23TAGS VALUES (101, 201)") + connection.execute("INSERT INTO Z_31TAGS VALUES (102, 202)") + + tags, report, manager_report = read_tags_and_aggregate(path) + + assert tags == ({100: [200]} if populated else {}) + assert report.storage == RelationshipStorage.PRESENT + assert report.storage_name == table_name + assert report.complete + assert manager_report.complete + assert ( + manager_report.relationships["transaction_tags"].storage + == RelationshipStorage.PRESENT + ) + + +@pytest.mark.parametrize("populated", [False, True]) +@pytest.mark.parametrize( + "metadata", + [ + (), + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + ((37, "Transaction", 0, 0),), + ((36, "Tag", 0, 0),), + ], +) +@pytest.mark.parametrize( + "ambiguous_table", + [ + "CREATE TABLE Z_31TAGS (Z_31SCHEDULEDTRANSACTIONS1 INTEGER)", + "CREATE TABLE Z_31TAGS " + "(Z_31SCHEDULEDTRANSACTIONS1 INTEGER, Z_35TAGS2 INTEGER, " + "Z_31TRANSACTIONS INTEGER)", + "CREATE TABLE Z_23TAGS (Z_23INFOCARDS5 INTEGER)", + "CREATE TABLE Z_23TAGS " + "(Z_23INFOCARDS5 INTEGER, Z_35TAGS1 INTEGER, " + "Z_23TRANSACTIONS INTEGER)", + ], +) +def test_incomplete_or_hybrid_unrelated_tag_shape_is_unknown( + tmp_path, metadata, ambiguous_table, populated +) -> None: + path = tmp_path / "ambiguous-unrelated-tags.sqlite" + create_custom_relationship_schema( + path, metadata=metadata, tables=(ambiguous_table,) + ) + if populated: + table_name = ambiguous_table.split()[2] + columns_count = ambiguous_table.count("INTEGER") + with sqlite3.connect(path) as connection: + connection.execute( + f"INSERT INTO {table_name} VALUES ({', '.join('?' for _ in range(columns_count))})", + (1,) * columns_count, + ) + + tags, report, manager_report = read_tags_and_aggregate(path) + + assert tags == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert not report.complete + assert not manager_report.complete + assert ( + manager_report.relationships["transaction_tags"].storage + == RelationshipStorage.UNKNOWN + ) From 45b7a9730de5352fc803f7c765879ee20e5efad8 Mon Sep 17 00:00:00 2001 From: MarcoMC Date: Sun, 13 Sep 2026 02:08:28 +0200 Subject: [PATCH 03/10] Preserve failed reload state and direct model validation Stage manager generations before publication, retain public manager identity on success and failure, validate account constructors after owned fields initialize, and classify unsupported category values precisely. Cover failure boundaries, retry, direct constructors and optimized diagnostics. --- README.md | 3 +- src/moneywiz_api/managers/record_manager.py | 6 + .../managers/transaction_manager.py | 7 + src/moneywiz_api/model/account.py | 2 + src/moneywiz_api/model/category.py | 2 +- src/moneywiz_api/moneywiz_api.py | 6 +- tests/unit/optimized_validation_probe.py | 111 ++++++- tests/unit/test_account_diagnostics.py | 95 +++++- tests/unit/test_api_atomic_reload.py | 276 ++++++++++++++++++ tests/unit/test_category_diagnostics.py | 116 ++++++++ tests/unit/test_optimized_validation.py | 24 ++ 11 files changed, 641 insertions(+), 7 deletions(-) create mode 100644 tests/unit/test_api_atomic_reload.py create mode 100644 tests/unit/test_category_diagnostics.py diff --git a/README.md b/README.md index 31d5b36..508af85 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,8 @@ schema profile. Database paths must identify existing files. Connections are opened read-only, and each multi-manager load uses one consistent SQLite read transaction. A later scoped load atomically refreshes the union of requested and already loaded -managers so one snapshot never mixes database generations. +managers so one snapshot never mixes database generations. If a reload fails, +the previously published records and completeness evidence remain available. ## Contribution diff --git a/src/moneywiz_api/managers/record_manager.py b/src/moneywiz_api/managers/record_manager.py index 0caf80b..4d3f742 100644 --- a/src/moneywiz_api/managers/record_manager.py +++ b/src/moneywiz_api/managers/record_manager.py @@ -94,6 +94,12 @@ def load(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: ) return self._load_report + def _adopt_loaded_state(self, staged: "RecordManager[T]") -> None: + """Publish a successfully staged load without replacing this manager.""" + self._records = staged._records + self._gid_to_id = staged._gid_to_id + self._load_report = staged._load_report + @staticmethod def _error_kind(exc: Exception) -> LoadErrorKind: if isinstance(exc, UnknownEntityError): diff --git a/src/moneywiz_api/managers/transaction_manager.py b/src/moneywiz_api/managers/transaction_manager.py index 03baf08..53eb02e 100644 --- a/src/moneywiz_api/managers/transaction_manager.py +++ b/src/moneywiz_api/managers/transaction_manager.py @@ -80,6 +80,13 @@ def load(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: ) return self._load_report + def _adopt_loaded_state(self, staged: "TransactionManager") -> None: + """Publish records and relationship state from one completed load.""" + super()._adopt_loaded_state(staged) + self.category_assignment = staged.category_assignment + self.refund_maps = staged.refund_maps + self.tags_map = staged.tags_map + def category_for_transaction( self, transaction_id: ID ) -> List[Tuple[ID, Decimal]] | None: diff --git a/src/moneywiz_api/model/account.py b/src/moneywiz_api/model/account.py index aa42695..a235589 100644 --- a/src/moneywiz_api/model/account.py +++ b/src/moneywiz_api/model/account.py @@ -35,6 +35,7 @@ def __init__(self, row): self.info = row.get("ZINFO") self.user = row["ZUSER"] + Account.validate(self) def validate(self) -> None: require_valid( @@ -91,6 +92,7 @@ class CreditCardAccount(Account): def __init__(self, row): super().__init__(row) self.statement_day = row["ZSTATEMENTENDDAY"] + CreditCardAccount.validate(self) def validate(self) -> None: super().validate() diff --git a/src/moneywiz_api/model/category.py b/src/moneywiz_api/model/category.py index 745b5f0..8f851d7 100644 --- a/src/moneywiz_api/model/category.py +++ b/src/moneywiz_api/model/category.py @@ -35,4 +35,4 @@ def __init__(self, row): def _convert_type(type_: Optional[int]) -> CategoryType: if type_ and type_ in [1, 2]: return "Expenses" if type_ == 1 else "Income" - raise RuntimeError(f"Invalid type {type_}") + raise ValueError("unsupported category type") diff --git a/src/moneywiz_api/moneywiz_api.py b/src/moneywiz_api/moneywiz_api.py index 23aef16..17e8095 100644 --- a/src/moneywiz_api/moneywiz_api.py +++ b/src/moneywiz_api/moneywiz_api.py @@ -75,10 +75,12 @@ def load(self, managers: Iterable[str] | None = None) -> ApiCompleteness: for name in self.MANAGER_NAMES if name in self._loaded_managers or name in requested ) - self._loaded_managers.difference_update(names) + staged = {name: type(self._managers[name])() for name in names} with self.accessor.read_transaction(): for name in names: - self._managers[name].load(self.accessor) + staged[name].load(self.accessor) + for name in names: + self._managers[name]._adopt_loaded_state(staged[name]) self._loaded_managers.update(names) return self.completeness(names) diff --git a/tests/unit/optimized_validation_probe.py b/tests/unit/optimized_validation_probe.py index e9643ce..253d234 100644 --- a/tests/unit/optimized_validation_probe.py +++ b/tests/unit/optimized_validation_probe.py @@ -10,9 +10,19 @@ from pathlib import Path from moneywiz_api.database_accessor import DatabaseAccessor +from moneywiz_api.managers.category_manager import CategoryManager from moneywiz_api.managers.payee_manager import PayeeManager from moneywiz_api.managers.transaction_manager import TransactionManager -from moneywiz_api.model.account import CreditCardAccount +from moneywiz_api.model.account import ( + Account, + BankChequeAccount, + BankSavingAccount, + CashAccount, + CreditCardAccount, + ForexAccount, + InvestmentAccount, + LoanAccount, +) from moneywiz_api.model.category import Category from moneywiz_api.model.investment_holding import InvestmentHolding from moneywiz_api.model.payee import Payee @@ -196,6 +206,97 @@ def typename_for(self, _ent_id): return "Payee" +class CategoryAccessor: + def __init__(self, rows): + self.rows = rows + + def query_objects(self, _typenames): + return self.rows + + def typename_for(self, _ent_id): + return "Category" + + +def direct_account_probe(): + constructors = ( + Account, + BankChequeAccount, + BankSavingAccount, + CashAccount, + CreditCardAccount, + LoanAccount, + InvestmentAccount, + ForexAccount, + ) + return { + constructor.__name__: { + "valid": capture( + lambda constructor=constructor: constructor(account_row()).info + ), + "invalid_name": capture( + lambda constructor=constructor: constructor(account_row(ZNAME=None)) + ), + "invalid_statement_day": ( + capture( + lambda constructor=constructor: constructor( + account_row(ZSTATEMENTENDDAY=None) + ) + ) + if issubclass(constructor, CreditCardAccount) + else None + ), + } + for constructor in constructors + } + + +def category_type_probe(): + def row(value=1, **overrides): + category = { + **common_row(Z_ENT=19), + "ZNAME2": "Category", + "ZPARENTCATEGORY": None, + "ZTYPE2": value, + "ZUSER3": 1, + } + category.update(overrides) + return category + + manager_reports = {} + for label, value in ( + ("null", None), + ("zero", 0), + ("integer", 3), + ("text", "PRIVATE_PAYLOAD"), + ): + report = CategoryManager().load(CategoryAccessor([row(value)])) + manager_reports[label] = report.as_dict() + missing = row() + del missing["ZTYPE2"] + missing_report = CategoryManager().load(CategoryAccessor([missing])) + owner_report = CategoryManager().load(CategoryAccessor([row(ZUSER3=None)])) + + def unexpected_constructor(_row): + raise RuntimeError("synthetic unexpected constructor failure") + + class UnexpectedCategoryManager(CategoryManager): + @property + def ents(self): + return {"Category": unexpected_constructor} + + unexpected_report = UnexpectedCategoryManager().load(CategoryAccessor([row()])) + return { + "supported": { + "1": capture(lambda: Category(row(1)).type), + "2": capture(lambda: Category(row(2)).type), + }, + "unsupported": manager_reports, + "missing": missing_report.as_dict(), + "invalid_owner": owner_report.as_dict(), + "unexpected": unexpected_report.as_dict(), + } + + def named_entities_probe(): rows = { "payee_valid": ( @@ -310,12 +411,14 @@ def relationship_probe(): def sentinel_probe(): original_payee_as_dict = Payee.as_dict original_deposit_as_dict = DepositTransaction.as_dict + original_account_as_dict = CashAccount.as_dict def reject_serialization(_self): raise RuntimeError("PRIVATE_PAYLOAD_SERIALIZED") Payee.as_dict = reject_serialization DepositTransaction.as_dict = reject_serialization + CashAccount.as_dict = reject_serialization try: payee = capture( lambda: Payee( @@ -327,10 +430,12 @@ def reject_serialization(_self): ) ) deposit = capture(lambda: DepositTransaction(transaction_row(ZACCOUNT2=None))) + account = capture(lambda: CashAccount(account_row(ZNAME=None))) finally: Payee.as_dict = original_payee_as_dict DepositTransaction.as_dict = original_deposit_as_dict - return {"payee": payee, "deposit": deposit} + CashAccount.as_dict = original_account_as_dict + return {"payee": payee, "deposit": deposit, "account": account} def main(): @@ -382,6 +487,8 @@ def main(): "account-1", CreditCardAccount ) ), + "direct_accounts": direct_account_probe(), + "category_types": category_type_probe(), "named_entities": named_entities_probe(), "manager": manager_report.as_dict(), "holding_valid": capture( diff --git a/tests/unit/test_account_diagnostics.py b/tests/unit/test_account_diagnostics.py index 2aa17ae..dca4639 100644 --- a/tests/unit/test_account_diagnostics.py +++ b/tests/unit/test_account_diagnostics.py @@ -2,7 +2,28 @@ from moneywiz_api.database_accessor import DatabaseAccessor from moneywiz_api.managers.account_manager import AccountManager -from moneywiz_api.model.account import CreditCardAccount +from moneywiz_api.model.account import ( + Account, + BankChequeAccount, + BankSavingAccount, + CashAccount, + CreditCardAccount, + ForexAccount, + InvestmentAccount, + LoanAccount, +) + + +ACCOUNT_CONSTRUCTORS = ( + Account, + BankChequeAccount, + BankSavingAccount, + CashAccount, + CreditCardAccount, + LoanAccount, + InvestmentAccount, + ForexAccount, +) def account_row(**overrides): @@ -69,6 +90,78 @@ def test_required_account_identity_remains_required() -> None: assert report.skipped[0].exception_type == "AssertionError" +@pytest.mark.parametrize("constructor", ACCOUNT_CONSTRUCTORS) +@pytest.mark.parametrize("info_present", [True, False]) +def test_direct_account_constructors_accept_nullable_or_absent_info( + constructor, info_present +) -> None: + row = account_row() + if not info_present: + del row["ZINFO"] + + account = constructor(row) + + assert account.info is None + assert account.name == "Card" + + +@pytest.mark.parametrize("constructor", ACCOUNT_CONSTRUCTORS) +def test_direct_account_constructors_reject_missing_required_name(constructor) -> None: + with pytest.raises(AssertionError, match="account name is required"): + constructor(account_row(ZNAME=None)) + + +@pytest.mark.parametrize("constructor", [CreditCardAccount, LoanAccount]) +def test_direct_credit_constructors_reject_null_statement_day(constructor) -> None: + with pytest.raises(AssertionError, match="account statement day is required"): + constructor(account_row(ZSTATEMENTENDDAY=None)) + + +@pytest.mark.parametrize("constructor", [CreditCardAccount, LoanAccount]) +def test_direct_credit_constructors_preserve_missing_statement_key(constructor) -> None: + row = account_row() + del row["ZSTATEMENTENDDAY"] + + with pytest.raises(KeyError, match="ZSTATEMENTENDDAY"): + constructor(row) + + +@pytest.mark.parametrize( + ("field", "value", "exception"), + [ + ("ZGID", "", AssertionError), + ("Z_PK", 0, AssertionError), + ("ZGID", None, AssertionError), + ], +) +def test_direct_account_preserves_record_identity_validation( + field, value, exception +) -> None: + with pytest.raises(exception): + CashAccount(account_row(**{field: value})) + + +def test_direct_account_preserves_missing_record_identity_field() -> None: + row = account_row() + del row["ZGID"] + + with pytest.raises(KeyError, match="ZGID"): + CashAccount(row) + + +def test_direct_account_validation_does_not_serialize_source(monkeypatch) -> None: + def reject_serialization(_self): + raise RuntimeError("PRIVATE_PAYLOAD_SERIALIZED") + + monkeypatch.setattr(CashAccount, "as_dict", reject_serialization) + row = account_row(ZNAME=None, ZINFO="PRIVATE_PAYLOAD") + + with pytest.raises(AssertionError, match="account name is required") as error: + CashAccount(row) + + assert "PRIVATE_PAYLOAD" not in str(error.value) + + class StaticCursor: def __init__(self, row): self.row = row diff --git a/tests/unit/test_api_atomic_reload.py b/tests/unit/test_api_atomic_reload.py new file mode 100644 index 0000000..550e532 --- /dev/null +++ b/tests/unit/test_api_atomic_reload.py @@ -0,0 +1,276 @@ +from contextlib import contextmanager +from decimal import Decimal + +import pytest + +from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.read_result import RelationshipLoadReport, RelationshipStorage +from moneywiz_api.schema_profile import SchemaProfile + + +def account_row(record_id, name): + return { + "Z_ENT": 10, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": f"account-{record_id}", + "Z_PK": record_id, + "ZDISPLAYORDER": record_id, + "ZGROUPID": 1, + "ZNAME": name, + "ZCURRENCYNAME": "EUR", + "ZOPENINGBALANCE": 0.0, + "ZINFO": None, + "ZUSER": 1, + } + + +def transaction_row(record_id, account_id): + return { + "Z_ENT": 37, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": f"transaction-{record_id}", + "Z_PK": record_id, + "ZRECONCILED": 0, + "ZAMOUNT1": 5.0, + "ZDESC2": f"Transaction {record_id}", + "ZDATE1": 0.0, + "ZNOTES1": None, + "ZACCOUNT2": account_id, + "ZPAYEE2": None, + "ZORIGINALCURRENCY": "EUR", + "ZORIGINALAMOUNT": 5.0, + "ZORIGINALEXCHANGERATE": 1.0, + } + + +class SyntheticAccessor: + def __init__(self): + self.schema_profile = SchemaProfile( + "unsuffixed-investment-columns", + "ZNUMBEROFSHARES", + "ZNUMBEROFSHARES", + "ZPRICEPERSHARE", + "ZPRICEPERSHARE", + ) + self.generation = "old" + self.fail_at = None + self.extra_invalid_account = False + self.in_transaction = False + self.transaction_entries = 0 + self.transaction_exits = 0 + + @contextmanager + def read_transaction(self): + if self.fail_at == "enter": + raise RuntimeError("synthetic transaction enter failure") + self.transaction_entries += 1 + self.in_transaction = True + try: + yield + finally: + self.in_transaction = False + self.transaction_exits += 1 + if self.fail_at == "exit": + raise RuntimeError("synthetic transaction exit failure") + + def descendant_typenames(self, _roots): + return [] + + def query_objects(self, typenames): + if self.fail_at == "transaction_query" and "DepositTransaction" in typenames: + raise RuntimeError("synthetic transaction query failure") + if "CashAccount" in typenames: + if self.generation == "old": + return [account_row(1, "Old")] + rows = [account_row(2, "New")] + if self.extra_invalid_account: + rows.append(account_row(3, None)) + return rows + if "DepositTransaction" in typenames: + record_id, account_id = (10, 1) if self.generation == "old" else (20, 2) + return [transaction_row(record_id, account_id)] + return [] + + def typename_for(self, ent_id): + return {10: "CashAccount", 37: "DepositTransaction"}.get(ent_id) + + def _relationship(self, name): + if self.fail_at == name: + raise RuntimeError(f"synthetic {name} failure") + transaction_id = 10 if self.generation == "old" else 20 + if name == "category": + value = {transaction_id: [(30, Decimal("5"))]} + elif name == "refund": + value = {transaction_id: 40} + else: + value = {transaction_id: [50]} + report = RelationshipLoadReport( + storage=RelationshipStorage.PRESENT, + storage_name=name, + source_ids=(f"{name}-{transaction_id}",), + parsed_ids=(f"{name}-{transaction_id}",), + ) + return value, report + + def read_category_assignments(self): + return self._relationship("category") + + def read_refund_maps(self): + return self._relationship("refund") + + def read_tags_map(self): + return self._relationship("tags") + + def close(self): + pass + + +def api_for(monkeypatch, accessor, managers): + import moneywiz_api.moneywiz_api as api_module + + monkeypatch.setattr(api_module, "DatabaseAccessor", lambda _path: accessor) + return api_module.MoneywizApi("unused.sqlite", managers=managers) + + +@pytest.mark.parametrize( + "failure", + ["transaction_query", "category", "refund", "tags"], +) +def test_failed_later_load_keeps_published_union_and_relationships( + monkeypatch, failure +) -> None: + accessor = SyntheticAccessor() + api = api_for(monkeypatch, accessor, ("accounts", "transactions")) + account_manager = api.account_manager + transaction_manager = api.transaction_manager + old_state = { + "account_records": account_manager._records, + "account_gid_index": account_manager._gid_to_id, + "account_report": account_manager.load_report, + "account": account_manager.get(1), + "transaction_records": transaction_manager._records, + "transaction_gid_index": transaction_manager._gid_to_id, + "transaction_report": transaction_manager.load_report, + "transaction": transaction_manager.get(10), + "categories": transaction_manager.category_assignment, + "refunds": transaction_manager.refund_maps, + "tags": transaction_manager.tags_map, + } + accessor.generation = "new" + accessor.fail_at = failure + + with pytest.raises(RuntimeError, match="synthetic"): + api.load(("transactions",)) + + assert api.account_manager is account_manager + assert api.transaction_manager is transaction_manager + assert api._loaded_managers == {"accounts", "transactions"} + assert account_manager._records is old_state["account_records"] + assert account_manager._gid_to_id is old_state["account_gid_index"] + assert account_manager.load_report is old_state["account_report"] + assert account_manager.get(1) is old_state["account"] + assert account_manager.get_by_gid("account-1") is old_state["account"] + assert transaction_manager._records is old_state["transaction_records"] + assert transaction_manager._gid_to_id is old_state["transaction_gid_index"] + assert transaction_manager.load_report is old_state["transaction_report"] + assert transaction_manager.get(10) is old_state["transaction"] + assert transaction_manager.get_by_gid("transaction-10") is old_state["transaction"] + assert transaction_manager.category_assignment is old_state["categories"] + assert transaction_manager.refund_maps is old_state["refunds"] + assert transaction_manager.tags_map is old_state["tags"] + assert api.completeness().managers["accounts"] is old_state["account_report"] + assert ( + api.completeness().managers["transactions"] is old_state["transaction_report"] + ) + assert not accessor.in_transaction + + +@pytest.mark.parametrize("failure", ["enter", "exit"]) +def test_transaction_context_failure_does_not_publish(monkeypatch, failure) -> None: + accessor = SyntheticAccessor() + api = api_for(monkeypatch, accessor, ("accounts",)) + records = api.account_manager._records + report = api.account_manager.load_report + account = api.account_manager.get(1) + accessor.generation = "new" + accessor.fail_at = failure + + with pytest.raises(RuntimeError, match="transaction .* failure"): + api.load(("accounts",)) + + assert api.account_manager._records is records + assert api.account_manager.load_report is report + assert api.account_manager.get(1) is account + assert api._loaded_managers == {"accounts"} + assert not accessor.in_transaction + + +def test_failed_new_selection_remains_unloaded_then_successful_retry_publishes( + monkeypatch, +) -> None: + accessor = SyntheticAccessor() + api = api_for(monkeypatch, accessor, ("accounts",)) + account_manager = api.account_manager + transaction_manager = api.transaction_manager + old_account = account_manager.get(1) + accessor.generation = "new" + accessor.fail_at = "transaction_query" + + with pytest.raises(RuntimeError, match="transaction query"): + api.load(("transactions",)) + + assert api._loaded_managers == {"accounts"} + assert account_manager.get(1) is old_account + assert transaction_manager.records() == {} + with pytest.raises(ValueError, match="manager not loaded: transactions"): + api.completeness(("transactions",)) + + accessor.fail_at = None + report = api.load(("transactions",)) + + assert api.account_manager is account_manager + assert api.transaction_manager is transaction_manager + assert api._loaded_managers == {"accounts", "transactions"} + assert account_manager.get(1) is None + assert account_manager.get(2).name == "New" + assert account_manager.get_by_gid("account-2") is account_manager.get(2) + assert transaction_manager.get(20).description == "Transaction 20" + assert transaction_manager.category_assignment == {20: [(30, Decimal("5"))]} + assert transaction_manager.refund_maps == {20: 40} + assert transaction_manager.tags_map == {20: [50]} + assert report.complete + + +def test_successful_publication_occurs_after_transaction_exit(monkeypatch) -> None: + accessor = SyntheticAccessor() + adoption_states = [] + original_adopt = AccountManager._adopt_loaded_state + + def observe_adoption(self, staged): + adoption_states.append(accessor.in_transaction) + original_adopt(self, staged) + + monkeypatch.setattr(AccountManager, "_adopt_loaded_state", observe_adoption) + api = api_for(monkeypatch, accessor, ("accounts",)) + accessor.generation = "new" + + api.load(("accounts",)) + + assert adoption_states == [False, False] + assert accessor.transaction_entries == accessor.transaction_exits == 2 + + +def test_partial_staged_generation_is_published(monkeypatch) -> None: + accessor = SyntheticAccessor() + api = api_for(monkeypatch, accessor, ("accounts",)) + accessor.generation = "new" + accessor.extra_invalid_account = True + + report = api.load(("accounts",)).managers["accounts"] + + assert report.source_ids == (2, 3) + assert report.parsed_ids == (2,) + assert report.skipped[0].record_id == 3 + assert report.skipped[0].error.value == "validation" + assert api.account_manager.get(2).name == "New" + assert api.account_manager.get(3) is None diff --git a/tests/unit/test_category_diagnostics.py b/tests/unit/test_category_diagnostics.py new file mode 100644 index 0000000..fd5f274 --- /dev/null +++ b/tests/unit/test_category_diagnostics.py @@ -0,0 +1,116 @@ +import json + +import pytest + +from moneywiz_api.managers.category_manager import CategoryManager +from moneywiz_api.model.category import Category +from moneywiz_api.moneywiz_api import MoneywizApi + + +def category_row(**overrides): + row = { + "Z_ENT": 19, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": "category-1", + "Z_PK": 1, + "ZNAME2": "Category", + "ZPARENTCATEGORY": None, + "ZTYPE2": 1, + "ZUSER3": 1, + } + row.update(overrides) + return row + + +class CategoryAccessor: + def __init__(self, rows): + self.rows = rows + + def query_objects(self, _typenames): + return self.rows + + def typename_for(self, _ent_id): + return "Category" + + +@pytest.mark.parametrize( + ("raw_type", "expected"), + [(1, "Expenses"), (2, "Income")], +) +def test_supported_category_types_are_unchanged(raw_type, expected) -> None: + assert Category(category_row(ZTYPE2=raw_type)).type == expected + + +@pytest.mark.parametrize("raw_type", [None, 0, 3, "PRIVATE_PAYLOAD"]) +def test_unsupported_category_type_is_invalid_value(raw_type, caplog) -> None: + manager = CategoryManager() + caplog.set_level("DEBUG") + + report = manager.load(CategoryAccessor([category_row(ZTYPE2=raw_type)])) + + assert report.source_ids == (1,) + assert report.parsed_ids == () + assert len(report.skipped) == 1 + assert report.skipped[0].record_id == 1 + assert report.skipped[0].error.value == "invalid_value" + assert report.skipped[0].exception_type == "ValueError" + serialized = json.dumps(report.as_dict(), sort_keys=True) + assert "PRIVATE_PAYLOAD" not in serialized + assert "PRIVATE_PAYLOAD" not in caplog.text + + +def test_unsupported_category_direct_error_has_static_message() -> None: + with pytest.raises(ValueError, match="^unsupported category type$") as error: + Category(category_row(ZTYPE2="PRIVATE_PAYLOAD")) + + assert "PRIVATE_PAYLOAD" not in str(error.value) + + +def test_missing_category_type_remains_missing_field() -> None: + row = category_row() + del row["ZTYPE2"] + + report = CategoryManager().load(CategoryAccessor([row])) + + assert report.skipped[0].error.value == "missing_field" + assert report.skipped[0].exception_type == "KeyError" + + +def test_invalid_category_owner_remains_validation() -> None: + report = CategoryManager().load(CategoryAccessor([category_row(ZUSER3=None)])) + + assert report.skipped[0].error.value == "validation" + assert report.skipped[0].exception_type == "AssertionError" + + +def test_unexpected_runtime_error_remains_construction() -> None: + def unexpected_constructor(_row): + raise RuntimeError("synthetic unexpected constructor failure") + + class UnexpectedCategoryManager(CategoryManager): + @property + def ents(self): + return {"Category": unexpected_constructor} + + report = UnexpectedCategoryManager().load(CategoryAccessor([category_row()])) + + assert report.skipped[0].error.value == "construction" + assert report.skipped[0].exception_type == "RuntimeError" + + +def test_category_diagnostic_is_exposed_by_api_completeness() -> None: + manager = CategoryManager() + manager.load(CategoryAccessor([category_row(ZTYPE2=3)])) + api = MoneywizApi.__new__(MoneywizApi) + api.category_manager = manager + api._managers = {"categories": manager} + api._loaded_managers = {"categories"} + + report = api.completeness(("categories",)).as_dict() + + category_report = report["managers"]["categories"] + assert report["status"] == "error" + assert not report["complete"] + assert category_report["source_count"] == 1 + assert category_report["parsed_count"] == 0 + assert category_report["skipped"][0]["error"] == "invalid_value" diff --git a/tests/unit/test_optimized_validation.py b/tests/unit/test_optimized_validation.py index 95ae9f5..907603e 100644 --- a/tests/unit/test_optimized_validation.py +++ b/tests/unit/test_optimized_validation.py @@ -65,6 +65,30 @@ def test_probe_covers_success_error_and_completeness_paths( assert cases["record_invalid_identity"]["type"] == "AssertionError" assert cases["account_nullable_info"]["status"] == "ok" assert cases["account_invalid_name_public"]["type"] == "AssertionError" + direct_accounts = cases["direct_accounts"] + assert all( + outcomes["valid"] == {"status": "ok", "value": None} + for outcomes in direct_accounts.values() + ) + assert all( + outcomes["invalid_name"]["type"] == "AssertionError" + for outcomes in direct_accounts.values() + ) + assert all( + direct_accounts[name]["invalid_statement_day"]["type"] == "AssertionError" + for name in ("CreditCardAccount", "LoanAccount") + ) + + category_types = cases["category_types"] + assert category_types["supported"]["1"]["value"] == "Expenses" + assert category_types["supported"]["2"]["value"] == "Income" + assert all( + report["skipped"][0]["error"] == "invalid_value" + for report in category_types["unsupported"].values() + ) + assert category_types["missing"]["skipped"][0]["error"] == "missing_field" + assert category_types["invalid_owner"]["skipped"][0]["error"] == "validation" + assert category_types["unexpected"]["skipped"][0]["error"] == "construction" named_entities = cases["named_entities"] assert all( From 737694e9c487827328bb9cdff92fedcc00997a3b Mon Sep 17 00:00:00 2001 From: MarcoMC Date: Sun, 13 Sep 2026 02:47:48 +0200 Subject: [PATCH 04/10] Bind cached schema and reads to coherent SQLite snapshots Capture complete entity metadata and physical schema in one initialization transaction. Verify schema before every cache-dependent public read, reject drift without altering published state, and require reopen. Cover metadata and DDL migrations, nested ownership, normal growth, and mutation-sensitive initialization races. --- README.md | 3 + src/moneywiz_api/database_accessor.py | 158 ++++- tests/unit/accessor_test_support.py | 37 ++ tests/unit/optimized_validation_probe.py | 24 +- tests/unit/test_account_diagnostics.py | 25 +- tests/unit/test_profiled_investment_values.py | 41 +- tests/unit/test_schema_cache_binding.py | 594 ++++++++++++++++++ 7 files changed, 786 insertions(+), 96 deletions(-) create mode 100644 tests/unit/accessor_test_support.py create mode 100644 tests/unit/test_schema_cache_binding.py diff --git a/README.md b/README.md index 508af85..97ef228 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,9 @@ and each multi-manager load uses one consistent SQLite read transaction. A later scoped load atomically refreshes the union of requested and already loaded managers so one snapshot never mixes database generations. If a reload fails, the previously published records and completeness evidence remain available. +Schema metadata and physical table definitions are bound when the accessor is +opened; if either changes, cache-dependent reads require closing and reopening +the API instead of combining current rows with stale schema information. ## Contribution diff --git a/src/moneywiz_api/database_accessor.py b/src/moneywiz_api/database_accessor.py index f7452be..f674025 100644 --- a/src/moneywiz_api/database_accessor.py +++ b/src/moneywiz_api/database_accessor.py @@ -70,6 +70,16 @@ def dict_factory(cursor, row): try: self._con.row_factory = dict_factory + self._initialize_schema_cache() + except Exception as exc: + self._con.close() + if isinstance(exc, DatabaseSchemaError): + raise + raise DatabaseSchemaError("database schema could not be read") from exc + + def _initialize_schema_cache(self) -> None: + """Bind all schema-dependent caches from one SQLite snapshot.""" + with self._raw_read_transaction(): if not self._table_exists("Z_PRIMARYKEY") or not self._table_exists( "ZSYNCOBJECT" ): @@ -83,29 +93,30 @@ def dict_factory(cursor, row): raise DatabaseSchemaError( "Z_PRIMARYKEY is missing required entity metadata columns" ) - self._schema_profile = detect_schema_profile(self._con) - self._ent_to_typename, self._ent_to_super = self._load_primarykey() - except Exception as exc: - self._con.close() - if isinstance(exc, DatabaseSchemaError): - raise - raise DatabaseSchemaError("database schema could not be read") from exc - self._typename_to_ent: Dict[str, ENT_ID] = { - v: k for k, v in self._ent_to_typename.items() - } + schema_identity = self._read_schema_identity() + schema_profile = detect_schema_profile(self._con) + ent_to_typename, ent_to_super, typename_to_ent = self._entity_maps( + schema_identity[1] + ) + + self._schema_identity = schema_identity + self._schema_profile = schema_profile + self._ent_to_typename = ent_to_typename + self._ent_to_super = ent_to_super + self._typename_to_ent = typename_to_ent - def _load_primarykey(self) -> tuple[Dict[int, str], Dict[int, int]]: + def _read_entity_metadata(self) -> tuple[tuple[int, str, int], ...]: cur = self._con.cursor() res = cur.execute( """ SELECT Z_ENT, Z_NAME, Z_SUPER FROM "Z_PRIMARYKEY" ORDER BY Z_ENT - LIMIT 1000 OFFSET 0; """ ) - ent_to_typename: Dict[int, str] = {} - ent_to_super: Dict[int, int] = {} + rows: list[tuple[int, str, int]] = [] + ent_ids: set[int] = set() + typenames: set[str] = set() for row in res.fetchall(): ent_id = row["Z_ENT"] typename = row["Z_NAME"] @@ -117,9 +128,45 @@ def _load_primarykey(self) -> tuple[Dict[int, str], Dict[int, int]]: or not isinstance(super_id, int) ): raise DatabaseSchemaError("Z_PRIMARYKEY contains invalid metadata") - ent_to_typename[ent_id] = typename - ent_to_super[ent_id] = super_id - return ent_to_typename, ent_to_super + if ent_id in ent_ids: + raise DatabaseSchemaError("Z_PRIMARYKEY contains duplicate entity IDs") + if typename in typenames: + raise DatabaseSchemaError( + "Z_PRIMARYKEY contains duplicate entity names" + ) + ent_ids.add(ent_id) + typenames.add(typename) + rows.append((ent_id, typename, super_id)) + return tuple(rows) + + @staticmethod + def _entity_maps( + rows: tuple[tuple[int, str, int], ...], + ) -> tuple[Dict[int, str], Dict[int, int], Dict[str, int]]: + ent_to_typename = {ent_id: typename for ent_id, typename, _ in rows} + ent_to_super = {ent_id: super_id for ent_id, _, super_id in rows} + typename_to_ent = {typename: ent_id for ent_id, typename, _ in rows} + return ent_to_typename, ent_to_super, typename_to_ent + + def _read_schema_identity( + self, + ) -> tuple[ + tuple[tuple[str, str, str, str | None], ...], + tuple[tuple[int, str, int], ...], + ]: + schema_rows = self._con.execute( + """ + SELECT type, name, tbl_name, sql + FROM sqlite_schema + WHERE name NOT LIKE 'sqlite_%' + ORDER BY type, name, tbl_name + """ + ).fetchall() + physical_schema = tuple( + (row["type"], row["name"], row["tbl_name"], row["sql"]) + for row in schema_rows + ) + return physical_schema, self._read_entity_metadata() def __repr__(self): return "\n".join( @@ -261,6 +308,11 @@ def _relationship_error(exc: Exception) -> LoadErrorKind: return LoadErrorKind.CONSTRUCTION def query_objects(self, typenames: List[str]) -> List[Any]: + """Query live rows against the verified cached entity mapping.""" + with self.read_transaction(): + return self._query_objects(typenames) + + def _query_objects(self, typenames: List[str]) -> List[Any]: ent_ids = [self.ent_for(name) for name in typenames] ent_ids = [ent_id for ent_id in ent_ids if ent_id is not None] if not ent_ids: @@ -280,8 +332,8 @@ def close(self) -> None: self._con.close() @contextmanager - def read_transaction(self): - """Keep a selected multi-manager load on one SQLite snapshot.""" + def _raw_read_transaction(self): + """Provide snapshot ownership without consulting schema caches.""" owns_transaction = not self._con.in_transaction if owns_transaction: self._con.execute("BEGIN") @@ -291,6 +343,33 @@ def read_transaction(self): if owns_transaction and self._con.in_transaction: self._con.rollback() + def _verify_schema_identity(self) -> None: + try: + baseline = self._schema_identity + except AttributeError as exc: + raise DatabaseSchemaError( + "database schema cache is not initialized; close and reopen the accessor" + ) from exc + try: + current = self._read_schema_identity() + except DatabaseSchemaError: + raise + except sqlite3.Error as exc: + raise DatabaseSchemaError( + "database schema could not be verified; close and reopen the accessor" + ) from exc + if current != baseline: + raise DatabaseSchemaError( + "database schema changed; close and reopen the accessor" + ) + + @contextmanager + def read_transaction(self): + """Keep cache-dependent reads on one verified SQLite snapshot.""" + with self._raw_read_transaction(): + self._verify_schema_identity() + yield + def __enter__(self): return self @@ -313,6 +392,10 @@ def _construct_record(self, row, constructor: Callable): return record def get_record(self, pk_id: ID, constructor: Callable = Record): + with self.read_transaction(): + return self._get_record(pk_id, constructor) + + def _get_record(self, pk_id: ID, constructor: Callable): cur = self._con.cursor() res = cur.execute( """ @@ -325,6 +408,10 @@ def get_record(self, pk_id: ID, constructor: Callable = Record): return self._construct_record(res.fetchone(), constructor) def get_record_by_gid(self, gid: GID, constructor: Callable = Record): + with self.read_transaction(): + return self._get_record_by_gid(gid, constructor) + + def _get_record_by_gid(self, gid: GID, constructor: Callable): cur = self._con.cursor() res = cur.execute( """ @@ -338,6 +425,12 @@ def get_record_by_gid(self, gid: GID, constructor: Callable = Record): def read_category_assignments( self, + ) -> tuple[Dict[ID, List[Tuple[ID, Decimal]]], RelationshipLoadReport]: + with self.read_transaction(): + return self._read_category_assignments() + + def _read_category_assignments( + self, ) -> tuple[Dict[ID, List[Tuple[ID, Decimal]]], RelationshipLoadReport]: transaction_map: Dict[ID, List[Tuple[ID, Decimal]]] = defaultdict(list) table_name = "ZCATEGORYASSIGMENT" @@ -400,11 +493,18 @@ def read_category_assignments( def get_category_assignment(self) -> Dict[ID, List[Tuple[ID, Decimal]]]: """Return category assignments without completeness metadata.""" - assignments, _ = self.read_category_assignments() - return assignments + with self.read_transaction(): + assignments, _ = self._read_category_assignments() + return assignments def read_refund_maps( self, + ) -> tuple[Dict[ID, ID], RelationshipLoadReport]: + with self.read_transaction(): + return self._read_refund_maps() + + def _read_refund_maps( + self, ) -> tuple[Dict[ID, ID], RelationshipLoadReport]: refund_to_withdraw: Dict[ID, ID] = {} table_name = "ZWITHDRAWREFUNDTRANSACTIONLINK" @@ -463,11 +563,18 @@ def read_refund_maps( def get_refund_maps(self) -> Dict[ID, ID]: """Return refund links without completeness metadata.""" - refund_maps, _ = self.read_refund_maps() - return refund_maps + with self.read_transaction(): + refund_maps, _ = self._read_refund_maps() + return refund_maps def read_tags_map( self, + ) -> tuple[Dict[ID, List[ID]], RelationshipLoadReport]: + with self.read_transaction(): + return self._read_tags_map() + + def _read_tags_map( + self, ) -> tuple[Dict[ID, List[ID]], RelationshipLoadReport]: transactions_to_tags: Dict[ID, List[ID]] = defaultdict(list) storage, table_name, transaction_column, tag_column = ( @@ -531,8 +638,9 @@ def read_tags_map( def get_tags_map(self) -> Dict[ID, List[ID]]: """Return transaction tags without completeness metadata.""" - transactions_to_tags, _ = self.read_tags_map() - return transactions_to_tags + with self.read_transaction(): + transactions_to_tags, _ = self._read_tags_map() + return transactions_to_tags def get_users(self) -> Dict[ID, str]: users_map: Dict[ID, str] = {} diff --git a/tests/unit/accessor_test_support.py b/tests/unit/accessor_test_support.py new file mode 100644 index 0000000..f68cd36 --- /dev/null +++ b/tests/unit/accessor_test_support.py @@ -0,0 +1,37 @@ +import sqlite3 + +from moneywiz_api.database_accessor import DatabaseAccessor + + +def initialized_memory_accessor(rows, entity_metadata): + """Build a guarded accessor after assembling its complete synthetic schema.""" + connection = sqlite3.connect(":memory:") + + def dict_factory(cursor, values): + return { + description[0]: values[index] + for index, description in enumerate(cursor.description) + } + + connection.row_factory = dict_factory + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, 0)", entity_metadata + ) + columns = tuple(dict.fromkeys(key for row in rows for key in row)) + definitions = ", ".join(f'"{column}"' for column in columns) + connection.execute(f"CREATE TABLE ZSYNCOBJECT ({definitions})") + placeholders = ", ".join("?" for _ in columns) + connection.executemany( + f"INSERT INTO ZSYNCOBJECT VALUES ({placeholders})", + [tuple(row.get(column) for column in columns) for row in rows], + ) + connection.commit() + + accessor = DatabaseAccessor.__new__(DatabaseAccessor) + accessor._con = connection + accessor._initialize_schema_cache() + return accessor diff --git a/tests/unit/optimized_validation_probe.py b/tests/unit/optimized_validation_probe.py index 253d234..728dbe1 100644 --- a/tests/unit/optimized_validation_probe.py +++ b/tests/unit/optimized_validation_probe.py @@ -36,6 +36,7 @@ WithdrawTransaction, ) from moneywiz_api.schema_profile import SchemaProfile +from accessor_test_support import initialized_memory_accessor PROFILE = SchemaProfile( @@ -162,29 +163,8 @@ def transfer_withdraw_row(**overrides): return row -class StaticCursor: - def __init__(self, row): - self.row = row - - def execute(self, _query, _parameters): - return self - - def fetchone(self): - return self.row - - -class StaticConnection: - def __init__(self, row): - self.row = row - - def cursor(self): - return StaticCursor(self.row) - - def accessor_for(row): - accessor = DatabaseAccessor.__new__(DatabaseAccessor) - accessor._con = StaticConnection(row) - return accessor + return initialized_memory_accessor([row], [(13, "CreditCardAccount", 0)]) class PayeeAccessor: diff --git a/tests/unit/test_account_diagnostics.py b/tests/unit/test_account_diagnostics.py index dca4639..340cfc1 100644 --- a/tests/unit/test_account_diagnostics.py +++ b/tests/unit/test_account_diagnostics.py @@ -1,6 +1,5 @@ import pytest -from moneywiz_api.database_accessor import DatabaseAccessor from moneywiz_api.managers.account_manager import AccountManager from moneywiz_api.model.account import ( Account, @@ -12,6 +11,7 @@ InvestmentAccount, LoanAccount, ) +from tests.unit.accessor_test_support import initialized_memory_accessor ACCOUNT_CONSTRUCTORS = ( @@ -162,29 +162,8 @@ def reject_serialization(_self): assert "PRIVATE_PAYLOAD" not in str(error.value) -class StaticCursor: - def __init__(self, row): - self.row = row - - def execute(self, _query, _parameters): - return self - - def fetchone(self): - return self.row - - -class StaticConnection: - def __init__(self, row): - self.row = row - - def cursor(self): - return StaticCursor(self.row) - - def accessor_for(row): - accessor = DatabaseAccessor.__new__(DatabaseAccessor) - accessor._con = StaticConnection(row) - return accessor + return initialized_memory_accessor([row], [(13, "CreditCardAccount", 0)]) def test_public_accessor_validates_nullable_info_after_construction() -> None: diff --git a/tests/unit/test_profiled_investment_values.py b/tests/unit/test_profiled_investment_values.py index f778c35..4a78d9b 100644 --- a/tests/unit/test_profiled_investment_values.py +++ b/tests/unit/test_profiled_investment_values.py @@ -4,7 +4,6 @@ from moneywiz_api.managers.investment_holding_manager import InvestmentHoldingManager from moneywiz_api.managers.transaction_manager import TransactionManager -from moneywiz_api.database_accessor import DatabaseAccessor from moneywiz_api.model.investment_holding import InvestmentHolding from moneywiz_api.model.transaction import ( InvestmentBuyTransaction, @@ -12,6 +11,7 @@ ) from moneywiz_api.schema_profile import SchemaProfile from moneywiz_api.read_result import RelationshipLoadReport, RelationshipStorage +from tests.unit.accessor_test_support import initialized_memory_accessor UNSUFFIXED_PROFILE = SchemaProfile( @@ -170,32 +170,21 @@ def test_managers_load_profiled_investment_records() -> None: assert holding_manager.get(24).number_of_shares == Decimal("2.0") -class StaticCursor: - def __init__(self, row: dict): - self.row = row - - def execute(self, _query, _parameters): - return self - - def fetchone(self): - return self.row - - -class StaticConnection: - def __init__(self, row: dict): - self.row = row - - def cursor(self): - return StaticCursor(self.row) - - def test_accessor_public_constructors_receive_schema_profile() -> None: - transaction_accessor = DatabaseAccessor.__new__(DatabaseAccessor) - transaction_accessor._con = StaticConnection(investment_transaction_row(40, -20.0)) - transaction_accessor._schema_profile = UNSUFFIXED_PROFILE - holding_accessor = DatabaseAccessor.__new__(DatabaseAccessor) - holding_accessor._con = StaticConnection(investment_holding_row()) - holding_accessor._schema_profile = UNSUFFIXED_PROFILE + transaction_row = investment_transaction_row(40, -20.0) + transaction_row.pop("ZNUMBEROFSHARES1") + transaction_row.pop("ZPRICEPERSHARE1") + holding_row = investment_holding_row() + holding_row.pop("ZNUMBEROFSHARES1") + holding_row.pop("ZPRICEPERSHARE1") + transaction_accessor = initialized_memory_accessor( + [transaction_row], + [(40, "InvestmentBuyTransaction", 0)], + ) + holding_accessor = initialized_memory_accessor( + [holding_row], + [(24, "InvestmentHolding", 0)], + ) transaction = transaction_accessor.get_record(40, InvestmentBuyTransaction) holding = holding_accessor.get_record_by_gid("holding", InvestmentHolding) diff --git a/tests/unit/test_schema_cache_binding.py b/tests/unit/test_schema_cache_binding.py new file mode 100644 index 0000000..2b0b99a --- /dev/null +++ b/tests/unit/test_schema_cache_binding.py @@ -0,0 +1,594 @@ +from contextlib import contextmanager +from pathlib import Path +import sqlite3 + +import pytest + +from moneywiz_api import MoneywizApi +from moneywiz_api.database_accessor import DatabaseAccessor, DatabaseSchemaError +from moneywiz_api.model.investment_holding import InvestmentHolding +from moneywiz_api.schema_profile import detect_schema_profile + + +SCHEMA_CHANGED = "database schema changed; close and reopen the accessor" + + +def open_writer(path: Path): + connection = sqlite3.connect(path) + connection.execute("PRAGMA journal_mode=WAL") + return connection + + +def create_account_store(path: Path, *, extra_metadata=()) -> None: + connection = open_writer(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", + [ + (8, "SyncObject", 0, 0), + (9, "Account", 8, 0), + (10, "CashAccount", 9, 1), + *extra_metadata, + ], + ) + connection.execute( + "CREATE TABLE ZSYNCOBJECT (" + "Z_PK INTEGER PRIMARY KEY, Z_ENT INTEGER, ZOBJECTCREATIONDATE FLOAT, " + "ZGID TEXT, ZDISPLAYORDER INTEGER, ZGROUPID INTEGER, ZNAME TEXT, " + "ZCURRENCYNAME TEXT, ZOPENINGBALANCE FLOAT, ZINFO TEXT, ZUSER INTEGER)" + ) + connection.execute( + "INSERT INTO ZSYNCOBJECT VALUES " + "(1, 10, 0, 'account-1', 1, 1, 'Before', 'EUR', 0, NULL, 1)" + ) + connection.commit() + connection.close() + + +def remap_cash_account(path: Path) -> None: + connection = open_writer(path) + connection.execute( + "UPDATE Z_PRIMARYKEY SET Z_NAME = 'RetiredCashAccount' WHERE Z_ENT = 10" + ) + connection.execute("INSERT INTO Z_PRIMARYKEY VALUES (11, 'CashAccount', 9, 1)") + connection.execute( + "UPDATE ZSYNCOBJECT SET Z_ENT = 11, ZNAME = 'After' WHERE Z_PK = 1" + ) + connection.commit() + connection.close() + + +def create_holding_store(path: Path) -> None: + connection = open_writer(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", + [(8, "SyncObject", 0, 0), (24, "InvestmentHolding", 8, 1)], + ) + connection.execute( + "CREATE TABLE ZSYNCOBJECT (" + "Z_PK INTEGER PRIMARY KEY, Z_ENT INTEGER, ZOBJECTCREATIONDATE FLOAT, " + "ZGID TEXT, ZINVESTMENTACCOUNT INTEGER, ZOPENNINGNUMBEROFSHARES FLOAT, " + "ZNUMBEROFSHARES FLOAT, ZPRICEPERSHARE FLOAT, ZSYMBOL TEXT, " + "ZHOLDINGTYPE TEXT, ZDESC TEXT, ZISPRICEPERSHAREAVAILABLEONLINE INTEGER, " + "ZINVESTMENTOBJECTTYPE INTEGER, ZCOSTBASISOFMISSINGOBSHARES FLOAT)" + ) + connection.execute( + "INSERT INTO ZSYNCOBJECT VALUES " + "(1, 24, 0, 'holding-1', 5, NULL, 2, 10, 'ACME', NULL, 'Acme', 0, 0, 0)" + ) + connection.commit() + connection.close() + + +def migrate_holding_aliases(path: Path) -> None: + connection = open_writer(path) + connection.execute( + "ALTER TABLE ZSYNCOBJECT RENAME COLUMN ZNUMBEROFSHARES TO ZNUMBEROFSHARES1" + ) + connection.execute( + "ALTER TABLE ZSYNCOBJECT RENAME COLUMN ZPRICEPERSHARE TO ZPRICEPERSHARE1" + ) + connection.commit() + connection.close() + + +def create_relationship_store(path: Path) -> None: + create_account_store( + path, + extra_metadata=( + (2, "CategoryAssigment", 8, 1), + (36, "Tag", 8, 1), + (37, "Transaction", 8, 1), + (50, "WithdrawRefundTransactionLink", 8, 1), + ), + ) + connection = open_writer(path) + connection.execute( + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)" + ) + connection.execute("INSERT INTO ZCATEGORYASSIGMENT VALUES (1, 20, 30, 5)") + connection.execute( + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, ZWITHDRAWTRANSACTION INTEGER)" + ) + connection.execute("INSERT INTO ZWITHDRAWREFUNDTRANSACTIONLINK VALUES (2, 31, 32)") + connection.execute( + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" + ) + connection.execute("INSERT INTO Z_37TAGS VALUES (30, 40)") + connection.commit() + connection.close() + + +def assert_old_account_generation( + api, manager, record, report, schema_identity +) -> None: + assert api.account_manager is manager + assert manager.get(1) is record + assert manager.get_by_gid("account-1") is record + assert manager.load_report is report + assert api.accessor._schema_identity is schema_identity + assert api._loaded_managers == {"accounts"} + snapshot = api.snapshot(("accounts",)).as_dict() + assert snapshot["records"]["accounts"][0]["name"] == "Before" + assert snapshot["completeness"]["managers"]["accounts"] == report.as_dict() + + +def test_metadata_remap_refuses_and_preserves_published_generation(tmp_path) -> None: + path = tmp_path / "metadata-remap.sqlite" + create_account_store(path) + with MoneywizApi(path, managers=("accounts",)) as api: + manager = api.account_manager + record = manager.get(1) + records = manager._records + gid_index = manager._gid_to_id + report = manager.load_report + schema_identity = api.accessor._schema_identity + cached_maps = ( + api.accessor._ent_to_typename, + api.accessor._ent_to_super, + api.accessor._typename_to_ent, + ) + remap_cash_account(path) + + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + api.load(("accounts",)) + + assert manager._records is records + assert manager._gid_to_id is gid_index + assert ( + api.accessor._ent_to_typename, + api.accessor._ent_to_super, + api.accessor._typename_to_ent, + ) == cached_maps + assert_old_account_generation(api, manager, record, report, schema_identity) + assert not api.accessor._con.in_transaction + + with MoneywizApi(path, managers=("accounts",)) as reopened: + assert reopened.account_manager.get(1).name == "After" + assert reopened.completeness().managers["accounts"].complete + + +@pytest.mark.parametrize( + "mutation", + [ + "UPDATE Z_PRIMARYKEY SET Z_NAME = 'CashAccount2' WHERE Z_ENT = 10", + "UPDATE Z_PRIMARYKEY SET Z_SUPER = 8 WHERE Z_ENT = 10", + "INSERT INTO Z_PRIMARYKEY VALUES (11, 'FutureAccount', 9, 0)", + "DELETE FROM Z_PRIMARYKEY WHERE Z_ENT = 10", + ], +) +def test_each_entity_metadata_change_refuses_reload(tmp_path, mutation) -> None: + path = tmp_path / "metadata-change.sqlite" + create_account_store(path) + with MoneywizApi(path, managers=("accounts",)) as api: + connection = open_writer(path) + connection.execute(mutation) + connection.commit() + connection.close() + + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + api.load(("accounts",)) + + assert api.account_manager.get(1).name == "Before" + + +def test_complete_metadata_inventory_has_no_thousand_row_cutoff(tmp_path) -> None: + path = tmp_path / "large-metadata.sqlite" + filler = tuple((ent_id, f"Entity{ent_id}", 8, 0) for ent_id in range(11, 1011)) + create_account_store(path, extra_metadata=filler) + connection = open_writer(path) + connection.execute( + "UPDATE Z_PRIMARYKEY SET Z_ENT = 1011 WHERE Z_NAME = 'CashAccount'" + ) + connection.execute("UPDATE ZSYNCOBJECT SET Z_ENT = 1011") + connection.commit() + connection.close() + + with MoneywizApi(path, managers=("accounts",)) as api: + assert len(api.accessor._schema_identity[1]) == 1003 + assert api.accessor.ent_for("CashAccount") == 1011 + assert "CashAccount" in api.accessor.descendant_typenames(("Account",)) + assert api.account_manager.get(1).name == "Before" + + +@pytest.mark.parametrize("duplicate", ["id", "name"]) +def test_duplicate_entity_map_keys_are_rejected(tmp_path, duplicate) -> None: + path = tmp_path / f"duplicate-{duplicate}.sqlite" + create_account_store(path) + connection = open_writer(path) + if duplicate == "id": + connection.execute( + "INSERT INTO Z_PRIMARYKEY VALUES (10, 'OtherCashAccount', 9, 0)" + ) + else: + connection.execute("INSERT INTO Z_PRIMARYKEY VALUES (11, 'CashAccount', 9, 0)") + connection.commit() + connection.close() + + with pytest.raises(DatabaseSchemaError, match="duplicate entity"): + DatabaseAccessor(path) + + +def test_physical_alias_drift_refuses_profiled_getters(tmp_path) -> None: + path = tmp_path / "holding-aliases.sqlite" + create_holding_store(path) + with DatabaseAccessor(path) as accessor: + baseline_profile = accessor.schema_profile + assert accessor.get_record(1, InvestmentHolding).number_of_shares == 2 + migrate_holding_aliases(path) + + for operation in ( + lambda: accessor.get_record(1, InvestmentHolding), + lambda: accessor.get_record_by_gid("holding-1", InvestmentHolding), + ): + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + operation() + assert accessor.schema_profile is baseline_profile + assert not accessor._con.in_transaction + + with DatabaseAccessor(path) as reopened: + assert reopened.schema_profile.profile_id == "suffixed-investment-columns" + assert reopened.get_record(1, InvestmentHolding).number_of_shares == 2 + + +@pytest.mark.parametrize( + "reader", + [ + "query_objects", + "read_category_assignments", + "get_category_assignment", + "read_refund_maps", + "get_refund_maps", + "read_tags_map", + "get_tags_map", + ], +) +def test_every_cache_dependent_reader_refuses_physical_drift(tmp_path, reader) -> None: + path = tmp_path / f"reader-{reader}.sqlite" + create_relationship_store(path) + with DatabaseAccessor(path) as accessor: + initial_profile = accessor.schema_profile + connection = open_writer(path) + connection.execute("ALTER TABLE ZCATEGORYASSIGMENT ADD COLUMN ZEXTRA") + connection.commit() + connection.close() + assert detect_schema_profile(accessor._con) == initial_profile + + operation = getattr(accessor, reader) + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + operation(["CashAccount"]) if reader == "query_objects" else operation() + + +@pytest.mark.parametrize( + "ddl", + [ + "CREATE TABLE Z_99TAGS (Z_99TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + "DROP TABLE ZCATEGORYASSIGMENT", + "ALTER TABLE ZCATEGORYASSIGMENT ADD COLUMN ZEXTRA", + ], +) +def test_relationship_schema_create_drop_and_alter_refuse_with_equal_profile( + tmp_path, ddl +) -> None: + path = tmp_path / "relationship-ddl.sqlite" + create_relationship_store(path) + with DatabaseAccessor(path) as accessor: + profile = accessor.schema_profile + connection = open_writer(path) + connection.execute(ddl) + connection.commit() + connection.close() + assert detect_schema_profile(accessor._con) == profile + + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + accessor.read_tags_map() + + +def test_schema_refusal_preserves_transaction_relationship_generation(tmp_path) -> None: + path = tmp_path / "relationship-state.sqlite" + create_relationship_store(path) + with MoneywizApi(path, managers=("accounts", "transactions")) as api: + manager = api.transaction_manager + report = manager.load_report + categories = manager.category_assignment + refunds = manager.refund_maps + tags = manager.tags_map + records = manager._records + gid_index = manager._gid_to_id + schema_identity = api.accessor._schema_identity + loaded_managers = set(api._loaded_managers) + connection = open_writer(path) + connection.execute("ALTER TABLE ZCATEGORYASSIGMENT ADD COLUMN ZEXTRA") + connection.commit() + connection.close() + + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + api.load(("transactions",)) + + assert api.transaction_manager is manager + assert manager.load_report is report + assert manager.category_assignment is categories + assert manager.refund_maps is refunds + assert manager.tags_map is tags + assert manager._records is records + assert manager._gid_to_id is gid_index + assert api.accessor._schema_identity is schema_identity + assert api._loaded_managers == loaded_managers + assert api.completeness().managers["transactions"] is report + + +def test_valid_direct_and_nested_reads_share_snapshot_ownership(tmp_path) -> None: + path = tmp_path / "nested.sqlite" + create_relationship_store(path) + with DatabaseAccessor(path) as accessor: + assert not accessor._con.in_transaction + with accessor.read_transaction(): + assert accessor._con.in_transaction + assert accessor.query_objects(["CashAccount"])[0]["Z_PK"] == 1 + assert accessor.get_record(1).gid == "account-1" + assert accessor.get_record_by_gid("account-1").id == 1 + assert accessor.read_category_assignments()[0] == {30: [(20, 5)]} + assert accessor.get_category_assignment() == {30: [(20, 5)]} + assert accessor.read_refund_maps()[0] == {31: 32} + assert accessor.get_refund_maps() == {31: 32} + assert accessor.read_tags_map()[0] == {30: [40]} + assert accessor.get_tags_map() == {30: [40]} + with accessor.read_transaction(): + assert accessor._con.in_transaction + assert accessor._con.in_transaction + assert not accessor._con.in_transaction + + +def test_normal_data_and_zmax_growth_reload_successfully(tmp_path) -> None: + path = tmp_path / "growth.sqlite" + create_account_store(path) + with MoneywizApi(path, managers=("accounts",)) as api: + manager = api.account_manager + connection = open_writer(path) + connection.execute("UPDATE Z_PRIMARYKEY SET Z_MAX = 2 WHERE Z_ENT = 10") + connection.execute("UPDATE ZSYNCOBJECT SET ZNAME = 'Edited' WHERE Z_PK = 1") + connection.execute( + "INSERT INTO ZSYNCOBJECT VALUES " + "(2, 10, 0, 'account-2', 2, 1, 'Added', 'EUR', 0, NULL, 1)" + ) + connection.commit() + connection.close() + + report = api.load(("accounts",)).managers["accounts"] + + assert api.account_manager is manager + assert report.complete + assert report.source_ids == (1, 2) + assert manager.get(1).name == "Edited" + assert manager.get(2).name == "Added" + + +def test_initialization_publishes_binding_only_after_transaction_exit( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "initialization-publication.sqlite" + create_account_store(path) + original_transaction = DatabaseAccessor._raw_read_transaction + exit_states = [] + + @contextmanager + def observe_transaction(accessor): + with original_transaction(accessor): + yield + exit_states.append( + (hasattr(accessor, "_schema_identity"), accessor._con.in_transaction) + ) + + monkeypatch.setattr(DatabaseAccessor, "_raw_read_transaction", observe_transaction) + + with DatabaseAccessor(path) as accessor: + assert exit_states == [(False, False)] + assert accessor._schema_identity + + +def test_missing_schema_binding_cannot_bypass_public_guard() -> None: + accessor = DatabaseAccessor.__new__(DatabaseAccessor) + accessor._con = sqlite3.connect(":memory:") + + with pytest.raises(DatabaseSchemaError, match="cache is not initialized"): + accessor.query_objects([]) + + assert not accessor._con.in_transaction + accessor.close() + + +def test_initialization_race_binds_one_old_snapshot_then_refuses(tmp_path, monkeypatch): + import moneywiz_api.database_accessor as accessor_module + + path = tmp_path / "initialization-race.sqlite" + create_holding_store(path) + original_detect = accessor_module.detect_schema_profile + migration_committed = False + + def migrate_then_detect(connection): + nonlocal migration_committed + writer = open_writer(path) + writer.execute( + "ALTER TABLE ZSYNCOBJECT RENAME COLUMN ZNUMBEROFSHARES TO ZNUMBEROFSHARES1" + ) + writer.execute( + "ALTER TABLE ZSYNCOBJECT RENAME COLUMN ZPRICEPERSHARE TO ZPRICEPERSHARE1" + ) + writer.execute( + "UPDATE Z_PRIMARYKEY SET Z_NAME = 'RetiredHolding' WHERE Z_ENT = 24" + ) + writer.execute( + "INSERT INTO Z_PRIMARYKEY VALUES (25, 'InvestmentHolding', 8, 1)" + ) + writer.execute("UPDATE ZSYNCOBJECT SET Z_ENT = 25") + writer.commit() + writer.close() + migration_committed = True + return original_detect(connection) + + monkeypatch.setattr(accessor_module, "detect_schema_profile", migrate_then_detect) + accessor = DatabaseAccessor(path) + + assert migration_committed + assert accessor.schema_profile.profile_id == "unsuffixed-investment-columns" + assert accessor.ent_for("InvestmentHolding") == 24 + assert (24, "InvestmentHolding", 8) in accessor._schema_identity[1] + assert (25, "InvestmentHolding", 8) not in accessor._schema_identity[1] + with accessor._raw_read_transaction(): + assert accessor._read_schema_identity() != accessor._schema_identity + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + accessor.query_objects(["InvestmentHolding"]) + accessor.close() + + monkeypatch.setattr(accessor_module, "detect_schema_profile", original_detect) + with DatabaseAccessor(path) as reopened: + assert reopened.schema_profile.profile_id == "suffixed-investment-columns" + assert reopened.ent_for("InvestmentHolding") == 25 + + +def test_first_load_schema_race_closes_constructor_accessor(tmp_path, monkeypatch): + import moneywiz_api.moneywiz_api as api_module + + path = tmp_path / "first-load.sqlite" + create_account_store(path) + accessor = DatabaseAccessor(path) + remap_cash_account(path) + monkeypatch.setattr(api_module, "DatabaseAccessor", lambda _path: accessor) + + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + api_module.MoneywizApi(path, managers=("accounts",)) + + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + accessor._con.execute("SELECT 1") + + +def test_reload_race_reads_old_snapshot_then_next_load_refuses( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "reload-race.sqlite" + create_account_store(path) + with MoneywizApi(path, managers=("accounts",)) as api: + original_verify = api.accessor._verify_schema_identity + migration_committed = False + + def verify_then_migrate(): + nonlocal migration_committed + original_verify() + if not migration_committed: + remap_cash_account(path) + migration_committed = True + + monkeypatch.setattr( + api.accessor, "_verify_schema_identity", verify_then_migrate + ) + + report = api.load(("accounts",)).managers["accounts"] + + assert migration_committed + assert report.complete + assert report.source_ids == (1,) + assert api.account_manager.get(1).name == "Before" + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + api.load(("accounts",)) + assert api.account_manager.get(1).name == "Before" + + +def test_transient_signature_query_error_preserves_state_and_retry( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "signature-error.sqlite" + create_account_store(path) + with MoneywizApi(path, managers=("accounts",)) as api: + manager = api.account_manager + record = manager.get(1) + report = manager.load_report + schema_identity = api.accessor._schema_identity + original_reader = api.accessor._read_schema_identity + + def fail_signature_query(): + raise sqlite3.OperationalError("PRIVATE_SCHEMA_QUERY_ERROR") + + monkeypatch.setattr(api.accessor, "_read_schema_identity", fail_signature_query) + with pytest.raises(DatabaseSchemaError, match="could not be verified") as error: + api.load(("accounts",)) + assert "PRIVATE_SCHEMA_QUERY_ERROR" not in str(error.value) + assert_old_account_generation(api, manager, record, report, schema_identity) + assert not api.accessor._con.in_transaction + + monkeypatch.setattr(api.accessor, "_read_schema_identity", original_reader) + connection = open_writer(path) + connection.execute("UPDATE ZSYNCOBJECT SET ZNAME = 'Retried' WHERE Z_PK = 1") + connection.commit() + connection.close() + assert api.load(("accounts",)).complete + assert manager.get(1).name == "Retried" + + +def test_invalid_metadata_refusal_can_retry_after_metadata_restore(tmp_path) -> None: + path = tmp_path / "invalid-metadata.sqlite" + create_account_store(path) + with MoneywizApi(path, managers=("accounts",)) as api: + connection = open_writer(path) + connection.execute("UPDATE Z_PRIMARYKEY SET Z_NAME = NULL WHERE Z_ENT = 10") + connection.commit() + connection.close() + + with pytest.raises(DatabaseSchemaError, match="invalid metadata"): + api.load(("accounts",)) + assert api.account_manager.get(1).name == "Before" + assert not api.accessor._con.in_transaction + + connection = open_writer(path) + connection.execute( + "UPDATE Z_PRIMARYKEY SET Z_NAME = 'CashAccount' WHERE Z_ENT = 10" + ) + connection.execute("UPDATE ZSYNCOBJECT SET ZNAME = 'Restored' WHERE Z_PK = 1") + connection.commit() + connection.close() + assert api.load(("accounts",)).complete + assert api.account_manager.get(1).name == "Restored" + + +def test_existing_unverified_transaction_is_checked_and_not_rolled_back( + tmp_path, +) -> None: + path = tmp_path / "caller-transaction.sqlite" + create_account_store(path) + with DatabaseAccessor(path) as accessor: + remap_cash_account(path) + accessor._con.execute("BEGIN") + + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + accessor.query_objects(["CashAccount"]) + + assert accessor._con.in_transaction + accessor._con.rollback() From 09bfe2cda6be85c3166b8cf495408bd7afb86924 Mon Sep 17 00:00:00 2001 From: MarcoMC Date: Sun, 13 Sep 2026 03:15:00 +0200 Subject: [PATCH 05/10] Distinguish unloaded managers from complete empty reads Publish manager completeness only after observation finishes, preserving atomic API reload behavior and existing report serialization. Cover fresh, scoped, failed and retried loads, and document transaction-only category assignment counts with alternative-owner controls. --- README.md | 8 +- src/moneywiz_api/managers/record_manager.py | 19 +++- .../managers/transaction_manager.py | 14 ++- src/moneywiz_api/read_result.py | 11 ++- tests/unit/test_api_atomic_reload.py | 38 ++++++++ tests/unit/test_read_completeness.py | 95 +++++++++++++++++++ tests/unit/test_relationship_completeness.py | 32 +++++++ 7 files changed, 207 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 97ef228..c1e1618 100644 --- a/README.md +++ b/README.md @@ -85,9 +85,15 @@ Supported manager names are `accounts`, `payees`, `categories`, `transactions`, parsed IDs plus identity-only diagnostics for every skipped row. Transaction completeness also reports category, refund, and tag relationship storage as `present`, `absent`, or `unknown`; unknown or malformed storage makes the read -partial. `snapshot` contains JSON-safe records, completeness, and the selected +partial. Category relationship counts cover transaction assignments only; rows +owned by budgets, scheduled transactions, or history items are outside that +map. `snapshot` contains JSON-safe records, completeness, and the selected schema profile. +A public manager that has not completed a load exposes a `load_report` with +status `unloaded` and `complete == False`. A successful zero-row load is instead +`complete`, so an empty result is distinguishable from an unobserved manager. + Database paths must identify existing files. Connections are opened read-only, and each multi-manager load uses one consistent SQLite read transaction. A later scoped load atomically refreshes the union of requested and already loaded diff --git a/src/moneywiz_api/managers/record_manager.py b/src/moneywiz_api/managers/record_manager.py index 4d3f742..45bc772 100644 --- a/src/moneywiz_api/managers/record_manager.py +++ b/src/moneywiz_api/managers/record_manager.py @@ -28,7 +28,7 @@ class RecordManager(ABC, Generic[T]): def __init__(self): self._records: Dict[ID, T] = {} self._gid_to_id: Dict[GID, ID] = {} - self._load_report = ManagerLoadReport() + self._load_report = ManagerLoadReport.unloaded() @property @abstractmethod @@ -41,9 +41,15 @@ def entity_roots(self) -> tuple[str, ...]: return () def load(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: + report = self._load_records(db_accessor) + self._load_report = report + return report + + def _load_records(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: + """Read rows while keeping the public report unloaded until publication.""" self._records = {} self._gid_to_id = {} - self._load_report = ManagerLoadReport() + self._load_report = ManagerLoadReport.unloaded() typenames = list(self.ents) if self.entity_roots: @@ -87,12 +93,17 @@ def load(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: continue parsed_ids.append(obj.id) - self._load_report = ManagerLoadReport( + return ManagerLoadReport( source_ids=tuple(source_ids), parsed_ids=tuple(parsed_ids), skipped=tuple(skipped), ) - return self._load_report + + def _discard_incomplete_load(self) -> None: + """Clear state when a direct manager load did not finish.""" + self._records = {} + self._gid_to_id = {} + self._load_report = ManagerLoadReport.unloaded() def _adopt_loaded_state(self, staged: "RecordManager[T]") -> None: """Publish a successfully staged load without replacing this manager.""" diff --git a/src/moneywiz_api/managers/transaction_manager.py b/src/moneywiz_api/managers/transaction_manager.py index 53eb02e..3e7dd13 100644 --- a/src/moneywiz_api/managers/transaction_manager.py +++ b/src/moneywiz_api/managers/transaction_manager.py @@ -63,10 +63,16 @@ def load(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: self.category_assignment = {} self.refund_maps = {} self.tags_map = {} - report = super().load(db_accessor) - category_assignment, category_report = db_accessor.read_category_assignments() - refund_maps, refund_report = db_accessor.read_refund_maps() - tags_map, tags_report = db_accessor.read_tags_map() + try: + report = self._load_records(db_accessor) + category_assignment, category_report = ( + db_accessor.read_category_assignments() + ) + refund_maps, refund_report = db_accessor.read_refund_maps() + tags_map, tags_report = db_accessor.read_tags_map() + except Exception: + self._discard_incomplete_load() + raise self.category_assignment = category_assignment self.refund_maps = refund_maps self.tags_map = tags_map diff --git a/src/moneywiz_api/read_result.py b/src/moneywiz_api/read_result.py index 75431a7..90f5b36 100644 --- a/src/moneywiz_api/read_result.py +++ b/src/moneywiz_api/read_result.py @@ -108,6 +108,12 @@ class ManagerLoadReport: parsed_ids: tuple[ID, ...] = () skipped: tuple[SkippedRecord, ...] = () relationships: Mapping[str, RelationshipLoadReport] = field(default_factory=dict) + observed: bool = True + + @classmethod + def unloaded(cls) -> "ManagerLoadReport": + """Return evidence that no complete manager observation was published.""" + return cls(observed=False) @property def source_count(self) -> int: @@ -120,13 +126,16 @@ def parsed_count(self) -> int: @property def complete(self) -> bool: return ( - self.source_count == self.parsed_count + self.observed + and self.source_count == self.parsed_count and not self.skipped and all(report.complete for report in self.relationships.values()) ) @property def status(self) -> str: + if not self.observed: + return "unloaded" if self.complete: return "complete" if self.parsed_count or any( diff --git a/tests/unit/test_api_atomic_reload.py b/tests/unit/test_api_atomic_reload.py index 550e532..6db5161 100644 --- a/tests/unit/test_api_atomic_reload.py +++ b/tests/unit/test_api_atomic_reload.py @@ -4,6 +4,7 @@ import pytest from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.managers.transaction_manager import TransactionManager from moneywiz_api.read_result import RelationshipLoadReport, RelationshipStorage from moneywiz_api.schema_profile import SchemaProfile @@ -168,11 +169,13 @@ def test_failed_later_load_keeps_published_union_and_relationships( assert account_manager._records is old_state["account_records"] assert account_manager._gid_to_id is old_state["account_gid_index"] assert account_manager.load_report is old_state["account_report"] + assert account_manager.load_report.status == "complete" assert account_manager.get(1) is old_state["account"] assert account_manager.get_by_gid("account-1") is old_state["account"] assert transaction_manager._records is old_state["transaction_records"] assert transaction_manager._gid_to_id is old_state["transaction_gid_index"] assert transaction_manager.load_report is old_state["transaction_report"] + assert transaction_manager.load_report.status == "complete" assert transaction_manager.get(10) is old_state["transaction"] assert transaction_manager.get_by_gid("transaction-10") is old_state["transaction"] assert transaction_manager.category_assignment is old_state["categories"] @@ -222,6 +225,7 @@ def test_failed_new_selection_remains_unloaded_then_successful_retry_publishes( assert api._loaded_managers == {"accounts"} assert account_manager.get(1) is old_account assert transaction_manager.records() == {} + assert transaction_manager.load_report.status == "unloaded" with pytest.raises(ValueError, match="manager not loaded: transactions"): api.completeness(("transactions",)) @@ -241,6 +245,40 @@ def test_failed_new_selection_remains_unloaded_then_successful_retry_publishes( assert report.complete +@pytest.mark.parametrize("failure", ["category", "refund", "tags"]) +def test_direct_transaction_relationship_failure_stays_unloaded_then_retries( + failure, +) -> None: + accessor = SyntheticAccessor() + accessor.fail_at = failure + manager = TransactionManager() + + with pytest.raises(RuntimeError, match=f"synthetic {failure} failure"): + manager.load(accessor) + + assert manager.load_report.status == "unloaded" + assert not manager.load_report.complete + assert manager.records() == {} + assert manager.category_assignment == {} + assert manager.refund_maps == {} + assert manager.tags_map == {} + + accessor.fail_at = None + report = manager.load(accessor) + + assert report.status == "complete" + assert report.complete + assert manager.get(10) is not None + + accessor.fail_at = failure + with pytest.raises(RuntimeError, match=f"synthetic {failure} failure"): + manager.load(accessor) + + assert manager.load_report.status == "unloaded" + assert not manager.load_report.complete + assert manager.records() == {} + + def test_successful_publication_occurs_after_transaction_exit(monkeypatch) -> None: accessor = SyntheticAccessor() adoption_states = [] diff --git a/tests/unit/test_read_completeness.py b/tests/unit/test_read_completeness.py index 7c3d675..f220eb6 100644 --- a/tests/unit/test_read_completeness.py +++ b/tests/unit/test_read_completeness.py @@ -4,10 +4,19 @@ import pytest +from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.managers.category_manager import CategoryManager +from moneywiz_api.managers.investment_holding_manager import ( + InvestmentHoldingManager, +) +from moneywiz_api.managers.payee_manager import PayeeManager from moneywiz_api.managers.record_manager import RecordManager +from moneywiz_api.managers.tag_manager import TagManager +from moneywiz_api.managers.transaction_manager import TransactionManager from moneywiz_api.model.record import Record from moneywiz_api.read_result import ( LoadErrorKind, + ManagerLoadReport, RelationshipLoadReport, RelationshipStorage, ) @@ -28,6 +37,34 @@ def entity_roots(self): return ("ExampleRecord",) +@pytest.mark.parametrize( + "manager_type", + [ + AccountManager, + PayeeManager, + CategoryManager, + TransactionManager, + InvestmentHoldingManager, + TagManager, + ], +) +def test_fresh_public_manager_reports_unloaded(manager_type) -> None: + report = manager_type().load_report + + assert not report.observed + assert report.status == "unloaded" + assert not report.complete + assert report.as_dict()["status"] == "unloaded" + + +def test_default_report_remains_a_successful_empty_observation() -> None: + report = ManagerLoadReport() + + assert report.observed + assert report.complete + assert report.status == "complete" + + def record_row(record_id=1, gid="record-1", ent=1): return { "Z_ENT": ent, @@ -70,6 +107,8 @@ def test_report_counts_duplicate_and_unknown_rows_without_partial_mutation() -> LoadErrorKind.DUPLICATE_GID, LoadErrorKind.UNKNOWN_ENTITY, ] + assert report.observed + assert report.status == "partial" assert list(manager.records()) == [1] @@ -86,6 +125,55 @@ def test_each_load_resets_records_and_diagnostics() -> None: assert manager.load_errors == [] +def test_failed_first_load_is_unloaded_and_can_retry() -> None: + manager = ExampleManager() + + class FailingAccessor(RecordAccessor): + def query_objects(self, _typenames): + raise RuntimeError("synthetic query failure") + + with pytest.raises(RuntimeError, match="synthetic query failure"): + manager.load(FailingAccessor([])) + + assert manager.load_report.status == "unloaded" + assert not manager.load_report.complete + assert manager.records() == {} + + report = manager.load(RecordAccessor([])) + + assert report.observed + assert report.status == "complete" + assert report.complete + + +def test_failed_direct_reload_replaces_prior_report_with_unloaded_state() -> None: + manager = ExampleManager() + manager.load(RecordAccessor([record_row()])) + + class FailingAccessor(RecordAccessor): + def query_objects(self, _typenames): + raise RuntimeError("synthetic reload failure") + + with pytest.raises(RuntimeError, match="synthetic reload failure"): + manager.load(FailingAccessor([])) + + assert manager.load_report.status == "unloaded" + assert not manager.load_report.complete + assert manager.records() == {} + + +def test_completed_load_with_only_invalid_rows_reports_error() -> None: + manager = ExampleManager() + + report = manager.load( + RecordAccessor([record_row(ent=2)], {1: "ExampleRecord", 2: "FutureRecord"}) + ) + + assert report.observed + assert report.status == "error" + assert not report.complete + + class FakeAccessor: def __init__(self, _db_file): self.schema_profile = SchemaProfile( @@ -144,6 +232,7 @@ def test_api_supports_scoped_loads_and_json_safe_snapshot(monkeypatch) -> None: assert snapshot["schema_profile"]["profile_id"] == ( "unsuffixed-investment-columns" ) + assert api.payee_manager.load_report.status == "unloaded" assert api.accessor.transaction_entries == 1 json.dumps(snapshot) @@ -167,8 +256,14 @@ def test_api_rejects_unloaded_or_unknown_manager(monkeypatch) -> None: monkeypatch.setattr(api_module, "DatabaseAccessor", FakeAccessor) api = api_module.MoneywizApi("unused.sqlite", managers=()) + assert all( + api._managers[name].load_report.status == "unloaded" + for name in api.MANAGER_NAMES + ) with pytest.raises(ValueError, match="manager not loaded"): api.completeness(("accounts",)) + with pytest.raises(ValueError, match="manager not loaded"): + api.snapshot(("accounts",)) with pytest.raises(ValueError, match="unknown manager"): api.load(("missing",)) diff --git a/tests/unit/test_relationship_completeness.py b/tests/unit/test_relationship_completeness.py index bad088b..ce90fc8 100644 --- a/tests/unit/test_relationship_completeness.py +++ b/tests/unit/test_relationship_completeness.py @@ -188,6 +188,38 @@ def test_shifted_relationship_layouts_report_valid_and_skipped_rows(tmp_path) -> ) +def test_category_counts_exclude_supported_nontransaction_owners(tmp_path) -> None: + path = tmp_path / "nontransaction-category-owners.sqlite" + create_custom_relationship_schema( + path, + metadata=((2, "CategoryAssigment", 0, 0),), + tables=( + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZAMOUNT FLOAT, " + "ZTRANSACTION INTEGER, ZBUDGET INTEGER, " + "ZSCHEDULEDTRANSACITION INTEGER, ZSTRINGHISTORYITEM INTEGER)", + ), + ) + with sqlite3.connect(path) as connection: + connection.executemany( + "INSERT INTO ZCATEGORYASSIGMENT VALUES (?, ?, ?, ?, ?, ?, ?)", + [ + (1, 10, 5.0, 100, None, None, None), + (2, 11, 2.0, None, 200, None, None), + (3, 12, 3.0, None, None, 300, None), + (4, 13, 4.0, None, None, None, 400), + ], + ) + + with DatabaseAccessor(path) as accessor: + categories, report = accessor.read_category_assignments() + + assert categories == {100: [(10, 5)]} + assert report.source_ids == (1,) + assert report.parsed_ids == (1,) + assert report.complete + + @pytest.mark.parametrize( "reader_name", ["read_category_assignments", "read_refund_maps", "read_tags_map"], From 129d876d944921d4112b6698ed8684ee9a01166c Mon Sep 17 00:00:00 2001 From: MarcoMC Date: Sun, 13 Sep 2026 03:55:33 +0200 Subject: [PATCH 06/10] Close entity coverage and published evidence ownership gaps Validate observed entity ancestry within each verified read transaction and discover descendants for all manager roots. Own nested completeness and snapshot containers immutably while exporting detached mutable JSON values. Preserve scoped isolation, valid empty models, atomic reloads and dataclass compatibility with paired regression matrices. --- README.md | 7 + src/moneywiz_api/database_accessor.py | 27 ++ src/moneywiz_api/managers/category_manager.py | 4 + .../managers/investment_holding_manager.py | 4 + src/moneywiz_api/managers/payee_manager.py | 4 + src/moneywiz_api/managers/tag_manager.py | 4 + src/moneywiz_api/read_result.py | 67 +++- tests/unit/optimized_validation_probe.py | 6 + tests/unit/test_category_diagnostics.py | 3 + tests/unit/test_entity_source_coverage.py | 366 ++++++++++++++++++ tests/unit/test_read_result_ownership.py | 218 +++++++++++ 11 files changed, 707 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_entity_source_coverage.py create mode 100644 tests/unit/test_read_result_ownership.py diff --git a/README.md b/README.md index c1e1618..acd6801 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,10 @@ owned by budgets, scheduled transactions, or history items are outside that map. `snapshot` contains JSON-safe records, completeness, and the selected schema profile. +Published completeness and snapshot values own immutable nested mappings and +sequences, so later caller or manager changes cannot rewrite earlier evidence. +Their `as_dict()` and `json_safe()` outputs are detached mutable JSON trees. + A public manager that has not completed a load exposes a `load_report` with status `unloaded` and `complete == False`. A successful zero-row load is instead `complete`, so an empty result is distinguishable from an unobserved manager. @@ -102,6 +106,9 @@ the previously published records and completeness evidence remain available. Schema metadata and physical table definitions are bound when the accessor is opened; if either changes, cache-dependent reads require closing and reopening the API instead of combining current rows with stale schema information. +Each verified read also rejects physical rows with NULL, unmapped, missing-parent, +or cyclic entity ancestry. Unknown descendants of a requested manager root are +included as skipped source rows instead of disappearing from completeness. ## Contribution diff --git a/src/moneywiz_api/database_accessor.py b/src/moneywiz_api/database_accessor.py index f674025..a52805c 100644 --- a/src/moneywiz_api/database_accessor.py +++ b/src/moneywiz_api/database_accessor.py @@ -363,11 +363,38 @@ def _verify_schema_identity(self) -> None: "database schema changed; close and reopen the accessor" ) + def _verify_source_eligibility(self) -> None: + """Reject rows whose entity ancestry cannot be classified safely.""" + try: + rows = self._con.execute( + "SELECT DISTINCT Z_ENT FROM ZSYNCOBJECT ORDER BY Z_ENT" + ).fetchall() + except sqlite3.Error as exc: + raise DatabaseSchemaError( + "database entity sources could not be verified" + ) from exc + + for row in rows: + current = row["Z_ENT"] + if not isinstance(current, int) or current not in self._ent_to_super: + raise DatabaseSchemaError( + "database contains rows with unclassifiable entity ancestry" + ) + visited: set[int] = set() + while current != 0: + if current in visited or current not in self._ent_to_super: + raise DatabaseSchemaError( + "database contains rows with unclassifiable entity ancestry" + ) + visited.add(current) + current = self._ent_to_super[current] + @contextmanager def read_transaction(self): """Keep cache-dependent reads on one verified SQLite snapshot.""" with self._raw_read_transaction(): self._verify_schema_identity() + self._verify_source_eligibility() yield def __enter__(self): diff --git a/src/moneywiz_api/managers/category_manager.py b/src/moneywiz_api/managers/category_manager.py index e124e13..b149c95 100644 --- a/src/moneywiz_api/managers/category_manager.py +++ b/src/moneywiz_api/managers/category_manager.py @@ -15,6 +15,10 @@ def ents(self) -> Dict[str, Callable]: "Category": Category, } + @property + def entity_roots(self) -> tuple[str, ...]: + return ("Category",) + def get_name_chain(self, category_id: ID) -> List[str]: ret: List[str] = [] current = self.get(category_id) diff --git a/src/moneywiz_api/managers/investment_holding_manager.py b/src/moneywiz_api/managers/investment_holding_manager.py index 069d02c..2e1e68a 100644 --- a/src/moneywiz_api/managers/investment_holding_manager.py +++ b/src/moneywiz_api/managers/investment_holding_manager.py @@ -17,6 +17,10 @@ def ents(self) -> Dict[str, Callable]: "InvestmentHolding": InvestmentHolding, } + @property + def entity_roots(self) -> tuple[str, ...]: + return ("InvestmentHolding",) + def construct_record( self, constructor: Callable, record, db_accessor: DatabaseAccessor ): diff --git a/src/moneywiz_api/managers/payee_manager.py b/src/moneywiz_api/managers/payee_manager.py index 714d0a2..1c92748 100644 --- a/src/moneywiz_api/managers/payee_manager.py +++ b/src/moneywiz_api/managers/payee_manager.py @@ -13,3 +13,7 @@ def ents(self) -> Dict[str, Callable]: return { "Payee": Payee, } + + @property + def entity_roots(self) -> tuple[str, ...]: + return ("Payee",) diff --git a/src/moneywiz_api/managers/tag_manager.py b/src/moneywiz_api/managers/tag_manager.py index 57de588..8a4bb32 100644 --- a/src/moneywiz_api/managers/tag_manager.py +++ b/src/moneywiz_api/managers/tag_manager.py @@ -13,3 +13,7 @@ def ents(self) -> Dict[str, Callable]: return { "Tag": Tag, } + + @property + def entity_roots(self) -> tuple[str, ...]: + return ("Tag",) diff --git a/src/moneywiz_api/read_result.py b/src/moneywiz_api/read_result.py index 90f5b36..c23016a 100644 --- a/src/moneywiz_api/read_result.py +++ b/src/moneywiz_api/read_result.py @@ -2,6 +2,7 @@ from __future__ import annotations +from copy import deepcopy from dataclasses import asdict, dataclass, field from datetime import date, datetime from decimal import Decimal @@ -12,6 +13,48 @@ from moneywiz_api.types import ID +class _FrozenMapping(dict): + """Recursively own published mapping values and refuse ordinary mutation.""" + + def __init__(self, values=()): + if getattr(self, "_initialized", False): + raise TypeError("published result mappings are immutable") + dict.__init__( + self, + ((key, _freeze_value(value)) for key, value in dict(values).items()), + ) + self._initialized = True + + @staticmethod + def _immutable(*_args, **_kwargs): + raise TypeError("published result mappings are immutable") + + __setitem__ = _immutable + __delitem__ = _immutable + __ior__ = _immutable + clear = _immutable + pop = _immutable + popitem = _immutable + setdefault = _immutable + update = _immutable + + def __deepcopy__(self, memo): + return type(self)( + (deepcopy(key, memo), deepcopy(value, memo)) for key, value in self.items() + ) + + +def _freeze_value(value: Any) -> Any: + """Recursively freeze mappings and sequences at a result boundary.""" + if isinstance(value, _FrozenMapping): + return value + if isinstance(value, Mapping): + return _FrozenMapping(value) + if isinstance(value, (tuple, list)): + return tuple(_freeze_value(item) for item in value) + return value + + class LoadErrorKind(str, Enum): """Bounded classifications for records omitted from a read.""" @@ -60,6 +103,11 @@ class RelationshipLoadReport: parsed_ids: tuple[ID | str, ...] = () skipped: tuple[SkippedRecord, ...] = () + def __post_init__(self) -> None: + object.__setattr__(self, "source_ids", tuple(self.source_ids)) + object.__setattr__(self, "parsed_ids", tuple(self.parsed_ids)) + object.__setattr__(self, "skipped", tuple(self.skipped)) + @property def source_count(self) -> int: return len(self.source_ids) @@ -110,6 +158,12 @@ class ManagerLoadReport: relationships: Mapping[str, RelationshipLoadReport] = field(default_factory=dict) observed: bool = True + def __post_init__(self) -> None: + object.__setattr__(self, "source_ids", tuple(self.source_ids)) + object.__setattr__(self, "parsed_ids", tuple(self.parsed_ids)) + object.__setattr__(self, "skipped", tuple(self.skipped)) + object.__setattr__(self, "relationships", _FrozenMapping(self.relationships)) + @classmethod def unloaded(cls) -> "ManagerLoadReport": """Return evidence that no complete manager observation was published.""" @@ -165,6 +219,9 @@ class ApiCompleteness: managers: Mapping[str, ManagerLoadReport] + def __post_init__(self) -> None: + object.__setattr__(self, "managers", _FrozenMapping(self.managers)) + @property def complete(self) -> bool: return bool(self.managers) and all( @@ -195,15 +252,19 @@ def as_dict(self) -> dict[str, Any]: class ReadSnapshot: """JSON-safe records and completeness metadata from one API instance.""" - records: Mapping[str, tuple[dict[str, Any], ...]] + records: Mapping[str, tuple[Mapping[str, Any], ...]] completeness: ApiCompleteness schema_profile: Mapping[str, Any] + def __post_init__(self) -> None: + object.__setattr__(self, "records", _FrozenMapping(self.records)) + object.__setattr__(self, "schema_profile", _FrozenMapping(self.schema_profile)) + def as_dict(self) -> dict[str, Any]: return { - "schema_profile": dict(self.schema_profile), + "schema_profile": json_safe(self.schema_profile), "completeness": self.completeness.as_dict(), - "records": {name: list(records) for name, records in self.records.items()}, + "records": json_safe(self.records), } diff --git a/tests/unit/optimized_validation_probe.py b/tests/unit/optimized_validation_probe.py index 728dbe1..1b1ebfa 100644 --- a/tests/unit/optimized_validation_probe.py +++ b/tests/unit/optimized_validation_probe.py @@ -168,6 +168,9 @@ def accessor_for(row): class PayeeAccessor: + def descendant_typenames(self, _roots): + return ["Payee"] + def query_objects(self, _typenames): return [ { @@ -190,6 +193,9 @@ class CategoryAccessor: def __init__(self, rows): self.rows = rows + def descendant_typenames(self, _roots): + return ["Category"] + def query_objects(self, _typenames): return self.rows diff --git a/tests/unit/test_category_diagnostics.py b/tests/unit/test_category_diagnostics.py index fd5f274..78b4ca8 100644 --- a/tests/unit/test_category_diagnostics.py +++ b/tests/unit/test_category_diagnostics.py @@ -26,6 +26,9 @@ class CategoryAccessor: def __init__(self, rows): self.rows = rows + def descendant_typenames(self, _roots): + return ["Category"] + def query_objects(self, _typenames): return self.rows diff --git a/tests/unit/test_entity_source_coverage.py b/tests/unit/test_entity_source_coverage.py new file mode 100644 index 0000000..dac97c0 --- /dev/null +++ b/tests/unit/test_entity_source_coverage.py @@ -0,0 +1,366 @@ +from pathlib import Path +import sqlite3 + +import pytest + +from moneywiz_api import MoneywizApi +from moneywiz_api.database_accessor import DatabaseAccessor, DatabaseSchemaError +from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.managers.category_manager import CategoryManager +from moneywiz_api.managers.investment_holding_manager import ( + InvestmentHoldingManager, +) +from moneywiz_api.managers.payee_manager import PayeeManager +from moneywiz_api.managers.tag_manager import TagManager +from moneywiz_api.managers.transaction_manager import TransactionManager + + +UNCLASSIFIABLE = "database contains rows with unclassifiable entity ancestry" + + +def common_row(record_id: int, ent_id: int | None, gid: str) -> dict: + return { + "Z_PK": record_id, + "Z_ENT": ent_id, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": gid, + } + + +def account_row() -> dict: + return { + **common_row(1, 10, "account-1"), + "ZDISPLAYORDER": 1, + "ZGROUPID": 1, + "ZNAME": "Account", + "ZCURRENCYNAME": "EUR", + "ZOPENINGBALANCE": 0.0, + "ZINFO": None, + "ZUSER": 1, + } + + +def payee_row() -> dict: + return { + **common_row(1, 28, "payee-1"), + "ZNAME5": "Payee", + "ZUSER7": 1, + } + + +def category_row() -> dict: + return { + **common_row(1, 19, "category-1"), + "ZNAME2": "Category", + "ZPARENTCATEGORY": None, + "ZTYPE2": 1, + "ZUSER3": 1, + } + + +def holding_row() -> dict: + return { + **common_row(1, 24, "holding-1"), + "ZINVESTMENTACCOUNT": 1, + "ZOPENNINGNUMBEROFSHARES": None, + "ZNUMBEROFSHARES": 2.0, + "ZPRICEPERSHARE": 10.0, + "ZSYMBOL": "ACME", + "ZHOLDINGTYPE": None, + "ZDESC": "Acme", + "ZISPRICEPERSHAREAVAILABLEONLINE": 0, + "ZINVESTMENTOBJECTTYPE": 0, + "ZCOSTBASISOFMISSINGOBSHARES": 0.0, + } + + +def tag_row() -> dict: + return { + **common_row(1, 35, "tag-1"), + "ZNAME6": "Tag", + "ZUSER8": 1, + } + + +def transaction_row() -> dict: + return { + **common_row(1, 38, "transaction-1"), + "ZRECONCILED": 0, + "ZAMOUNT1": 5.0, + "ZDESC2": "Deposit", + "ZDATE1": 0.0, + "ZNOTES1": None, + "ZACCOUNT2": 1, + "ZPAYEE2": None, + "ZORIGINALCURRENCY": "EUR", + "ZORIGINALAMOUNT": 5.0, + "ZORIGINALEXCHANGERATE": 1.0, + } + + +MANAGER_CASES = [ + pytest.param( + AccountManager, "Account", 9, "CashAccount", 10, account_row, id="account" + ), + pytest.param(PayeeManager, "Payee", 28, "Payee", 28, payee_row, id="payee"), + pytest.param( + CategoryManager, "Category", 19, "Category", 19, category_row, id="category" + ), + pytest.param( + InvestmentHoldingManager, + "InvestmentHolding", + 24, + "InvestmentHolding", + 24, + holding_row, + id="investment-holding", + ), + pytest.param(TagManager, "Tag", 35, "Tag", 35, tag_row, id="tag"), + pytest.param( + TransactionManager, + "Transaction", + 37, + "DepositTransaction", + 38, + transaction_row, + id="transaction", + ), +] + + +def create_store(path: Path, metadata: list[tuple[int, str, int]], rows: list[dict]): + connection = sqlite3.connect(path) + connection.execute("PRAGMA journal_mode=WAL") + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.executemany("INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, 0)", metadata) + columns = list( + dict.fromkeys( + ["Z_PK", "Z_ENT", "ZOBJECTCREATIONDATE", "ZGID"] + + [column for row in rows for column in row] + + ["ZNUMBEROFSHARES", "ZPRICEPERSHARE"] + ) + ) + definitions = ", ".join( + f'"{column}" INTEGER' if column in {"Z_PK", "Z_ENT"} else f'"{column}"' + for column in columns + ) + connection.execute(f"CREATE TABLE ZSYNCOBJECT ({definitions})") + for row in rows: + insert_row(connection, row) + connection.commit() + connection.close() + + +def insert_row(connection: sqlite3.Connection, row: dict) -> None: + columns = tuple(row) + names = ", ".join(f'"{column}"' for column in columns) + placeholders = ", ".join("?" for _ in columns) + connection.execute( + f"INSERT INTO ZSYNCOBJECT ({names}) VALUES ({placeholders})", + tuple(row[column] for column in columns), + ) + + +@pytest.mark.parametrize( + ("manager_type", "root_name", "root_id", "known_name", "known_id", "row_factory"), + MANAGER_CASES, +) +@pytest.mark.parametrize("transitive", [False, True], ids=["direct", "transitive"]) +def test_all_managers_count_unknown_descendants( + tmp_path, + manager_type, + root_name, + root_id, + known_name, + known_id, + row_factory, + transitive, +) -> None: + path = tmp_path / f"{root_name}-{transitive}.sqlite" + metadata = [(8, "SyncObject", 0), (root_id, root_name, 8)] + if known_id != root_id: + metadata.append((known_id, known_name, root_id)) + unknown_parent = root_id + if transitive: + metadata.append((90, f"Future{root_name}Parent", root_id)) + unknown_parent = 90 + unknown_name = f"Future{root_name}" + metadata.append((91, unknown_name, unknown_parent)) + create_store( + path, + metadata, + [row_factory(), common_row(2, 91, f"future-{root_name.lower()}")], + ) + + with DatabaseAccessor(path) as accessor: + report = manager_type().load(accessor) + + assert set(report.source_ids) == {1, 2} + assert report.parsed_ids == (1,) + assert len(report.skipped) == 1 + assert report.skipped[0].record_id == 2 + assert report.skipped[0].entity == unknown_name + assert report.skipped[0].error.value == "unknown_entity" + + +def orphan_metadata(kind: str) -> tuple[list[tuple[int, str, int]], int | None]: + metadata = [(8, "SyncObject", 0), (9, "Account", 8), (10, "CashAccount", 9)] + if kind == "null": + return metadata, None + if kind == "unmapped": + return metadata, 99 + if kind == "missing-parent": + return [*metadata, (90, "FuturePayee", 99)], 90 + if kind == "cycle": + return [*metadata, (90, "FuturePayee", 91), (91, "FuturePayeeParent", 90)], 90 + raise AssertionError("unsupported synthetic orphan kind") + + +@pytest.mark.parametrize("kind", ["null", "unmapped", "missing-parent", "cycle"]) +@pytest.mark.parametrize("include_valid", [False, True], ids=["orphan-only", "mixed"]) +def test_unclassifiable_observed_entity_refuses_source_read( + tmp_path, kind, include_valid +) -> None: + path = tmp_path / f"{kind}-{include_valid}.sqlite" + metadata, ent_id = orphan_metadata(kind) + rows = [common_row(2, ent_id, "orphan")] + if include_valid: + rows.insert(0, account_row()) + create_store(path, metadata, rows) + + with DatabaseAccessor(path) as accessor: + with pytest.raises(DatabaseSchemaError, match=f"^{UNCLASSIFIABLE}$"): + accessor.query_objects(["CashAccount"]) + + +@pytest.mark.parametrize( + "operation", + [ + lambda accessor: accessor.query_objects(["CashAccount"]), + lambda accessor: accessor.get_record(1), + lambda accessor: accessor.get_record_by_gid("account-1"), + lambda accessor: accessor.read_category_assignments(), + lambda accessor: accessor.get_category_assignment(), + lambda accessor: accessor.read_refund_maps(), + lambda accessor: accessor.get_refund_maps(), + lambda accessor: accessor.read_tags_map(), + lambda accessor: accessor.get_tags_map(), + ], +) +def test_every_public_cache_dependent_reader_refuses_orphan( + tmp_path, operation +) -> None: + path = tmp_path / "reader-orphan.sqlite" + metadata, ent_id = orphan_metadata("unmapped") + create_store(path, metadata, [account_row(), common_row(2, ent_id, "orphan")]) + + with DatabaseAccessor(path) as accessor: + with pytest.raises(DatabaseSchemaError, match=f"^{UNCLASSIFIABLE}$"): + operation(accessor) + + +def test_unobserved_broken_metadata_and_absent_optional_roots_are_allowed( + tmp_path, +) -> None: + path = tmp_path / "unused-broken-metadata.sqlite" + create_store(path, [(8, "SyncObject", 0), (90, "FutureUnused", 99)], []) + + with DatabaseAccessor(path) as accessor: + reports = [ + manager_type().load(accessor) + for manager_type in ( + AccountManager, + PayeeManager, + CategoryManager, + TransactionManager, + InvestmentHoldingManager, + TagManager, + ) + ] + + assert all(report.source_count == 0 and report.complete for report in reports) + + +def test_known_unrelated_malformed_row_does_not_block_account_scope(tmp_path) -> None: + path = tmp_path / "known-unrelated.sqlite" + metadata = [ + (8, "SyncObject", 0), + (9, "Account", 8), + (10, "CashAccount", 9), + (80, "KnownUnrelated", 8), + ] + create_store(path, metadata, [account_row(), common_row(2, 80, "unrelated")]) + + with DatabaseAccessor(path) as accessor: + report = AccountManager().load(accessor) + + assert report.complete + assert report.source_ids == (1,) + assert report.parsed_ids == (1,) + + +def test_api_initial_orphan_failure_closes_accessor(tmp_path, monkeypatch) -> None: + import moneywiz_api.moneywiz_api as api_module + + path = tmp_path / "initial-orphan.sqlite" + metadata, ent_id = orphan_metadata("unmapped") + create_store(path, metadata, [common_row(2, ent_id, "orphan")]) + accessor = DatabaseAccessor(path) + monkeypatch.setattr(api_module, "DatabaseAccessor", lambda _path: accessor) + + with pytest.raises(DatabaseSchemaError, match=f"^{UNCLASSIFIABLE}$"): + api_module.MoneywizApi(path, managers=("accounts",)) + + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + accessor._con.execute("SELECT 1") + + +def test_same_schema_orphan_reload_preserves_generation_and_can_retry(tmp_path) -> None: + path = tmp_path / "reload-orphan.sqlite" + metadata, _ = orphan_metadata("unmapped") + create_store(path, metadata, [account_row()]) + + with MoneywizApi(path, managers=("accounts",)) as api: + manager = api.account_manager + record = manager.get(1) + report = manager.load_report + writer = sqlite3.connect(path) + insert_row(writer, common_row(2, 99, "orphan")) + writer.commit() + writer.close() + + with pytest.raises(DatabaseSchemaError, match=f"^{UNCLASSIFIABLE}$"): + api.load(("accounts",)) + + assert api.account_manager is manager + assert manager.get(1) is record + assert manager.load_report is report + + writer = sqlite3.connect(path) + writer.execute("DELETE FROM ZSYNCOBJECT WHERE Z_PK = 2") + writer.commit() + writer.close() + + retry = api.load(("accounts",)).managers["accounts"] + + assert retry.complete + assert retry.source_ids == (1,) + + +def test_explicit_query_names_do_not_expand_to_other_known_scopes(tmp_path) -> None: + path = tmp_path / "exact-query.sqlite" + metadata = [ + (8, "SyncObject", 0), + (9, "Account", 8), + (10, "CashAccount", 9), + (28, "Payee", 8), + ] + create_store(path, metadata, [account_row(), payee_row()]) + + with DatabaseAccessor(path) as accessor: + rows = accessor.query_objects(["CashAccount"]) + + assert [row["Z_PK"] for row in rows] == [1] diff --git a/tests/unit/test_read_result_ownership.py b/tests/unit/test_read_result_ownership.py new file mode 100644 index 0000000..dfb4df5 --- /dev/null +++ b/tests/unit/test_read_result_ownership.py @@ -0,0 +1,218 @@ +from copy import deepcopy +from dataclasses import asdict, fields, replace +from datetime import date +from decimal import Decimal + +import pytest + +from moneywiz_api.managers.tag_manager import TagManager +from moneywiz_api.model.record import Record +from moneywiz_api.read_result import ( + ApiCompleteness, + LoadErrorKind, + ManagerLoadReport, + ReadSnapshot, + RelationshipLoadReport, + RelationshipStorage, + SkippedRecord, + json_safe, +) + + +def incomplete_relationship() -> RelationshipLoadReport: + return RelationshipLoadReport( + storage=RelationshipStorage.UNKNOWN, + source_ids=["source-1"], + skipped=[ + SkippedRecord( + record_id="source-1", + entity="Tag", + error=LoadErrorKind.MISSING_FIELD, + exception_type="KeyError", + ) + ], + ) + + +def test_report_constructors_copy_sequences_and_relationship_mapping() -> None: + source_ids = [1] + parsed_ids = [] + skipped = [ + SkippedRecord(1, "FutureRecord", LoadErrorKind.UNKNOWN_ENTITY, "ValueError") + ] + relationships = {"transaction_tags": incomplete_relationship()} + report = ManagerLoadReport( + source_ids=source_ids, + parsed_ids=parsed_ids, + skipped=skipped, + relationships=relationships, + ) + + source_ids.append(2) + parsed_ids.append(1) + skipped.clear() + relationships.clear() + + assert report.source_ids == (1,) + assert report.parsed_ids == () + assert len(report.skipped) == 1 + assert set(report.relationships) == {"transaction_tags"} + assert not report.complete + with pytest.raises(TypeError, match="immutable"): + report.relationships.clear() + + +def test_aggregate_owns_manager_mapping_and_cannot_become_complete() -> None: + managers = { + "incomplete": ManagerLoadReport( + relationships={"transaction_tags": incomplete_relationship()} + ), + "complete": ManagerLoadReport(), + } + completeness = ApiCompleteness(managers) + + del managers["incomplete"] + + assert set(completeness.managers) == {"incomplete", "complete"} + assert not completeness.complete + with pytest.raises(TypeError, match="immutable"): + completeness.managers.update({"replacement": ManagerLoadReport()}) + + +def test_snapshot_owns_nested_records_and_schema_profile() -> None: + nested_values = [1] + row = {"amount": Decimal("1.20"), "nested": {"values": nested_values}} + records = {"accounts": [row]} + schema_profile = {"aliases": {"columns": ["ZVALUE"]}} + snapshot = ReadSnapshot( + records=records, + completeness=ApiCompleteness({"accounts": ManagerLoadReport()}), + schema_profile=schema_profile, + ) + + nested_values.append(2) + row["nested"]["extra"] = True + records.clear() + schema_profile["aliases"]["columns"].append("ZVALUE1") + + retained_row = snapshot.records["accounts"][0] + assert retained_row["nested"] == {"values": (1,)} + assert snapshot.schema_profile == {"aliases": {"columns": ("ZVALUE",)}} + with pytest.raises(TypeError, match="immutable"): + retained_row["nested"]["extra"] = True + with pytest.raises(AttributeError, match="append"): + retained_row["nested"]["values"].append(3) + + +def test_result_outputs_are_mutable_detached_json_trees() -> None: + report = ManagerLoadReport( + relationships={"transaction_tags": incomplete_relationship()} + ) + snapshot = ReadSnapshot( + records={ + "accounts": ( + { + "amount": Decimal("1.20"), + "date": date(2026, 9, 13), + "nested": {"values": [1]}, + }, + ) + }, + completeness=ApiCompleteness({"accounts": report}), + schema_profile={"aliases": {"columns": ["ZVALUE"]}}, + ) + + report_output = report.as_dict() + snapshot_output = snapshot.as_dict() + safe_output = json_safe(snapshot) + report_output["relationships"].clear() + snapshot_output["records"]["accounts"][0]["nested"]["values"].append(2) + snapshot_output["schema_profile"]["aliases"]["columns"].append("ZVALUE1") + safe_output["records"]["accounts"][0]["nested"]["values"].append(3) + + assert not report.complete + assert snapshot.records["accounts"][0]["nested"]["values"] == (1,) + assert snapshot.schema_profile["aliases"]["columns"] == ("ZVALUE",) + assert snapshot_output["records"]["accounts"][0]["amount"] == "1.20" + assert snapshot_output["records"]["accounts"][0]["date"] == "2026-09-13" + + +def test_dataclass_asdict_replace_and_constructor_contracts_remain_supported() -> None: + unloaded = ManagerLoadReport.unloaded() + default = ManagerLoadReport() + report = replace( + default, + relationships={"transaction_tags": incomplete_relationship()}, + ) + snapshot = ReadSnapshot( + records={"accounts": ({"nested": [1]},)}, + completeness=ApiCompleteness({"accounts": report}), + schema_profile={"aliases": ["ZVALUE"]}, + ) + + report_dict = asdict(report) + snapshot_dict = asdict(snapshot) + copied_records = deepcopy(snapshot.records) + replaced_snapshot = replace(snapshot, schema_profile={"aliases": ["ZVALUE1"]}) + + assert [item.name for item in fields(ManagerLoadReport)] == [ + "source_ids", + "parsed_ids", + "skipped", + "relationships", + "observed", + ] + assert unloaded.status == "unloaded" + assert default.complete + assert report_dict["relationships"]["transaction_tags"]["storage"] == ( + RelationshipStorage.UNKNOWN + ) + assert snapshot_dict["records"]["accounts"][0]["nested"] == (1,) + assert copied_records == snapshot.records + assert copied_records is not snapshot.records + assert replaced_snapshot.schema_profile["aliases"] == ("ZVALUE1",) + + +def test_relationship_sequences_are_copied_and_as_dict_is_detached() -> None: + source_ids = ["source-1"] + parsed_ids = ["source-1"] + report = RelationshipLoadReport( + RelationshipStorage.PRESENT, + source_ids=source_ids, + parsed_ids=parsed_ids, + ) + + source_ids.append("source-2") + parsed_ids.clear() + output = report.as_dict() + output["source_ids"].append("source-3") + output["parsed_ids"].clear() + + assert report.source_ids == ("source-1",) + assert report.parsed_ids == ("source-1",) + assert report.complete + + +def test_live_manager_and_model_remain_mutable_after_snapshot_publication() -> None: + record = Record( + { + "Z_PK": 1, + "Z_ENT": 35, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": "record-1", + } + ) + manager = TagManager() + manager.records()[record.id] = record + snapshot = ReadSnapshot( + records={"tags": (record.as_dict(),)}, + completeness=ApiCompleteness({"tags": ManagerLoadReport()}), + schema_profile={}, + ) + + record.gid = "changed-record" + manager.records()[2] = record + + assert manager.get(2) is record + assert record.gid == "changed-record" + assert snapshot.records["tags"][0]["gid"] == "record-1" From 3264256979b5c1ca3924e2b11c2cd91e10ebed7d Mon Sep 17 00:00:00 2001 From: MarcoMC Date: Sun, 13 Sep 2026 09:57:04 +0200 Subject: [PATCH 07/10] Keep public manager loads coherent and validate raw identity domains Share one transaction lifecycle across all manager entrypoints and publish reports only after successful exit. Validate integer identities and raw reconciliation state before conversion, retaining optional references and safe diagnostics. Cover real WAL interleaves, nested ownership and mutation-sensitive failure regressions. --- README.md | 10 + src/moneywiz_api/database_accessor.py | 45 +- src/moneywiz_api/managers/record_manager.py | 19 +- .../managers/transaction_manager.py | 29 +- src/moneywiz_api/model/account.py | 8 +- src/moneywiz_api/model/category.py | 10 +- src/moneywiz_api/model/investment_holding.py | 5 +- src/moneywiz_api/model/payee.py | 3 +- src/moneywiz_api/model/record.py | 8 +- src/moneywiz_api/model/tag.py | 3 +- src/moneywiz_api/model/transaction.py | 87 +++- src/moneywiz_api/validation.py | 10 + tests/unit/optimized_validation_probe.py | 53 ++- tests/unit/test_account_diagnostics.py | 6 + tests/unit/test_api_atomic_reload.py | 79 +++- tests/unit/test_category_diagnostics.py | 5 + tests/unit/test_identity_domains.py | 418 ++++++++++++++++++ .../test_manager_transaction_lifecycle.py | 212 +++++++++ tests/unit/test_optimized_validation.py | 32 +- tests/unit/test_profiled_investment_values.py | 5 + tests/unit/test_read_completeness.py | 25 +- tests/unit/test_relationship_completeness.py | 139 ++++++ tests/unit/test_transaction_compatibility.py | 5 + 23 files changed, 1161 insertions(+), 55 deletions(-) create mode 100644 tests/unit/test_identity_domains.py create mode 100644 tests/unit/test_manager_transaction_lifecycle.py diff --git a/README.md b/README.md index acd6801..ba95275 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,16 @@ and each multi-manager load uses one consistent SQLite read transaction. A later scoped load atomically refreshes the union of requested and already loaded managers so one snapshot never mixes database generations. If a reload fails, the previously published records and completeness evidence remain available. +Direct `manager.load(accessor)` calls use the same boundary: records, +transaction relationships, and the published report come from one read +snapshot, while a failed load resets that manager to `unloaded`. + +Core Data identity and relationship endpoint fields must arrive as uncoerced +integers; nullable payee and category-parent references remain supported. +Transaction reconciliation accepts only raw integer `0` or `1` before exposing +the value as `bool`. Refused rows remain visible through identity-only +completeness diagnostics without embedding malformed payload values. + Schema metadata and physical table definitions are bound when the accessor is opened; if either changes, cache-dependent reads require closing and reopening the API instead of combining current rows with stale schema information. diff --git a/src/moneywiz_api/database_accessor.py b/src/moneywiz_api/database_accessor.py index a52805c..ea2952d 100644 --- a/src/moneywiz_api/database_accessor.py +++ b/src/moneywiz_api/database_accessor.py @@ -22,7 +22,7 @@ ) from moneywiz_api.schema_profile import SchemaProfile, detect_schema_profile from moneywiz_api.types import ENT_ID, ID, GID -from moneywiz_api.validation import require_valid +from moneywiz_api.validation import require_integer_identity, require_valid TRANSACTION_TAG_TABLE_RE = re.compile(r"Z_\d+TAGS") @@ -481,11 +481,16 @@ def _read_category_assignments( "WHERE ZTRANSACTION IS NOT NULL" ).fetchall() for row in rows: - source_id = row.get("Z_PK") + raw_source_id = row.get("Z_PK") + source_id = raw_source_id if type(raw_source_id) is int else None source_ids.append(source_id) try: require_valid( - source_id is not None, "category assignment ID is required" + raw_source_id is not None, "category assignment ID is required" + ) + require_integer_identity( + raw_source_id, + "category assignment ID must be an uncoerced integer", ) category_id = row["ZCATEGORY"] transaction_id = row["ZTRANSACTION"] @@ -497,6 +502,14 @@ def _read_category_assignments( transaction_id is not None, "category assignment transaction endpoint is required", ) + require_integer_identity( + category_id, + "category assignment category endpoint must be an uncoerced integer", + ) + require_integer_identity( + transaction_id, + "category assignment transaction endpoint must be an uncoerced integer", + ) amount = RDH.get_decimal(row, "ZAMOUNT") transaction_map[transaction_id].append((category_id, amount)) except Exception as exc: @@ -554,10 +567,14 @@ def _read_refund_maps( f'SELECT {", ".join(columns)} FROM "{table_name}"' ).fetchall() for row in rows: - source_id = row.get("Z_PK") + raw_source_id = row.get("Z_PK") + source_id = raw_source_id if type(raw_source_id) is int else None source_ids.append(source_id) try: - require_valid(source_id is not None, "refund link ID is required") + require_valid(raw_source_id is not None, "refund link ID is required") + require_integer_identity( + raw_source_id, "refund link ID must be an uncoerced integer" + ) refund_id = row["ZREFUNDTRANSACTION"] withdraw_id = row["ZWITHDRAWTRANSACTION"] require_valid( @@ -566,6 +583,14 @@ def _read_refund_maps( require_valid( withdraw_id is not None, "withdraw transaction endpoint is required" ) + require_integer_identity( + refund_id, + "refund transaction endpoint must be an uncoerced integer", + ) + require_integer_identity( + withdraw_id, + "withdraw transaction endpoint must be an uncoerced integer", + ) if refund_id in refund_to_withdraw: raise ValueError() refund_to_withdraw[refund_id] = withdraw_id @@ -631,7 +656,7 @@ def _read_tags_map( tag_id = row.get(tag_column) source_id = ( f"{transaction_id}:{tag_id}" - if transaction_id is not None and tag_id is not None + if type(transaction_id) is int and type(tag_id) is int else f"row:{position}" ) source_ids.append(source_id) @@ -643,6 +668,14 @@ def _read_tags_map( require_valid( tag_id is not None, "transaction-tag tag endpoint is required" ) + require_integer_identity( + transaction_id, + "transaction-tag transaction endpoint must be an uncoerced integer", + ) + require_integer_identity( + tag_id, + "transaction-tag tag endpoint must be an uncoerced integer", + ) transactions_to_tags[transaction_id].append(tag_id) except Exception as exc: skipped.append( diff --git a/src/moneywiz_api/managers/record_manager.py b/src/moneywiz_api/managers/record_manager.py index 45bc772..c5a55b6 100644 --- a/src/moneywiz_api/managers/record_manager.py +++ b/src/moneywiz_api/managers/record_manager.py @@ -41,16 +41,22 @@ def entity_roots(self) -> tuple[str, ...]: return () def load(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: - report = self._load_records(db_accessor) + self._discard_incomplete_load() + try: + with db_accessor.read_transaction(): + report = self._load_in_transaction(db_accessor) + except Exception: + self._discard_incomplete_load() + raise self._load_report = report return report + def _load_in_transaction(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: + """Load all manager state inside the caller-owned read snapshot.""" + return self._load_records(db_accessor) + def _load_records(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: """Read rows while keeping the public report unloaded until publication.""" - self._records = {} - self._gid_to_id = {} - self._load_report = ManagerLoadReport.unloaded() - typenames = list(self.ents) if self.entity_roots: discovered = db_accessor.descendant_typenames(self.entity_roots) @@ -62,7 +68,8 @@ def _load_records(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: skipped: list[SkippedRecord] = [] for record in records: - record_id = record.get("Z_PK") + raw_record_id = record.get("Z_PK") + record_id = raw_record_id if type(raw_record_id) is int else None source_ids.append(record_id) typename = None try: diff --git a/src/moneywiz_api/managers/transaction_manager.py b/src/moneywiz_api/managers/transaction_manager.py index 3e7dd13..26b7af1 100644 --- a/src/moneywiz_api/managers/transaction_manager.py +++ b/src/moneywiz_api/managers/transaction_manager.py @@ -59,24 +59,15 @@ def construct_record( return constructor(record, schema_profile=db_accessor.schema_profile) return super().construct_record(constructor, record, db_accessor) - def load(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: - self.category_assignment = {} - self.refund_maps = {} - self.tags_map = {} - try: - report = self._load_records(db_accessor) - category_assignment, category_report = ( - db_accessor.read_category_assignments() - ) - refund_maps, refund_report = db_accessor.read_refund_maps() - tags_map, tags_report = db_accessor.read_tags_map() - except Exception: - self._discard_incomplete_load() - raise + def _load_in_transaction(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: + report = super()._load_in_transaction(db_accessor) + category_assignment, category_report = db_accessor.read_category_assignments() + refund_maps, refund_report = db_accessor.read_refund_maps() + tags_map, tags_report = db_accessor.read_tags_map() self.category_assignment = category_assignment self.refund_maps = refund_maps self.tags_map = tags_map - self._load_report = replace( + return replace( report, relationships={ "category_assignments": category_report, @@ -84,7 +75,13 @@ def load(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: "transaction_tags": tags_report, }, ) - return self._load_report + + def _discard_incomplete_load(self) -> None: + """Clear records and relationships after an incomplete load.""" + super()._discard_incomplete_load() + self.category_assignment = {} + self.refund_maps = {} + self.tags_map = {} def _adopt_loaded_state(self, staged: "TransactionManager") -> None: """Publish records and relationship state from one completed load.""" diff --git a/src/moneywiz_api/model/account.py b/src/moneywiz_api/model/account.py index a235589..0688853 100644 --- a/src/moneywiz_api/model/account.py +++ b/src/moneywiz_api/model/account.py @@ -6,7 +6,7 @@ from moneywiz_api.types import ID from moneywiz_api.model.raw_data_handler import RawDataHandler as RDH from moneywiz_api.model.record import Record -from moneywiz_api.validation import require_valid +from moneywiz_api.validation import require_integer_identity, require_valid @dataclass @@ -42,6 +42,9 @@ def validate(self) -> None: self.display_order is not None, "account display order is required" ) require_valid(self.group_id is not None, "account group identity is required") + require_integer_identity( + self.group_id, "account group identity must be an uncoerced integer" + ) require_valid(self.name is not None, "account name is required") require_valid(self.currency is not None, "account currency is required") require_valid( @@ -49,6 +52,9 @@ def validate(self) -> None: ) # info is nullable in valid MoneyWiz stores. require_valid(self.user is not None, "account owner is required") + require_integer_identity( + self.user, "account owner must be an uncoerced integer" + ) @dataclass diff --git a/src/moneywiz_api/model/category.py b/src/moneywiz_api/model/category.py index 8f851d7..e1df291 100644 --- a/src/moneywiz_api/model/category.py +++ b/src/moneywiz_api/model/category.py @@ -3,7 +3,7 @@ from moneywiz_api.model.record import Record from moneywiz_api.types import CategoryType, ID -from moneywiz_api.validation import require_valid +from moneywiz_api.validation import require_integer_identity, require_valid @dataclass @@ -28,8 +28,16 @@ def __init__(self, row): # Validate require_valid(self.name is not None, "category name is required") + require_integer_identity( + self.parent_id, + "category parent must be an uncoerced integer", + optional=True, + ) require_valid(self.type is not None, "category type is required") require_valid(self.user is not None, "category owner is required") + require_integer_identity( + self.user, "category owner must be an uncoerced integer" + ) @staticmethod def _convert_type(type_: Optional[int]) -> CategoryType: diff --git a/src/moneywiz_api/model/investment_holding.py b/src/moneywiz_api/model/investment_holding.py index e9d7aab..651e040 100644 --- a/src/moneywiz_api/model/investment_holding.py +++ b/src/moneywiz_api/model/investment_holding.py @@ -6,7 +6,7 @@ from moneywiz_api.model.record import Record from moneywiz_api.schema_profile import SchemaProfile from moneywiz_api.types import ID -from moneywiz_api.validation import require_valid +from moneywiz_api.validation import require_integer_identity, require_valid @dataclass @@ -80,6 +80,9 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): def validate(self): require_valid(self.account is not None, "holding account is required") + require_integer_identity( + self.account, "holding account must be an uncoerced integer" + ) require_valid(self.number_of_shares is not None, "holding quantity is required") # price_per_share can be None when no current quote is stored. require_valid(self.symbol is not None, "holding symbol is required") diff --git a/src/moneywiz_api/model/payee.py b/src/moneywiz_api/model/payee.py index b251c2a..86a8438 100644 --- a/src/moneywiz_api/model/payee.py +++ b/src/moneywiz_api/model/payee.py @@ -2,7 +2,7 @@ from moneywiz_api.model.record import Record from moneywiz_api.types import ID -from moneywiz_api.validation import require_valid +from moneywiz_api.validation import require_integer_identity, require_valid @dataclass @@ -24,3 +24,4 @@ def __init__(self, row): # Validate require_valid(self.name is not None, "payee name is required") require_valid(self.user is not None, "payee owner is required") + require_integer_identity(self.user, "payee owner must be an uncoerced integer") diff --git a/src/moneywiz_api/model/record.py b/src/moneywiz_api/model/record.py index a5483af..c148a5a 100644 --- a/src/moneywiz_api/model/record.py +++ b/src/moneywiz_api/model/record.py @@ -4,7 +4,7 @@ from moneywiz_api.types import ID, ENT_ID from moneywiz_api.model.raw_data_handler import RawDataHandler as RDH -from moneywiz_api.validation import require_valid +from moneywiz_api.validation import require_integer_identity, require_valid @dataclass @@ -27,9 +27,15 @@ def __init__(self, row): # Validate require_valid(self._raw, "record source must not be empty") require_valid(self._ent, "record entity must be set") + require_integer_identity( + self._ent, "record entity must be an uncoerced integer" + ) require_valid(self._created_at, "record creation date must be set") require_valid(self.gid, "record global identity must be set") require_valid(self.id, "record identity must be set") + require_integer_identity( + self.id, "record identity must be an uncoerced integer" + ) def ent(self) -> ENT_ID: return self._ent diff --git a/src/moneywiz_api/model/tag.py b/src/moneywiz_api/model/tag.py index a5305a3..c152677 100644 --- a/src/moneywiz_api/model/tag.py +++ b/src/moneywiz_api/model/tag.py @@ -2,7 +2,7 @@ from moneywiz_api.model.record import Record from moneywiz_api.types import ID -from moneywiz_api.validation import require_valid +from moneywiz_api.validation import require_integer_identity, require_valid @dataclass @@ -24,3 +24,4 @@ def __init__(self, row): # Validate require_valid(self.name is not None, "tag name is required") require_valid(self.user is not None, "tag owner is required") + require_integer_identity(self.user, "tag owner must be an uncoerced integer") diff --git a/src/moneywiz_api/model/transaction.py b/src/moneywiz_api/model/transaction.py index 651a8ac..c73553e 100644 --- a/src/moneywiz_api/model/transaction.py +++ b/src/moneywiz_api/model/transaction.py @@ -10,7 +10,7 @@ from moneywiz_api.model.record import Record from moneywiz_api.schema_profile import SchemaProfile from moneywiz_api.types import ID -from moneywiz_api.validation import require_valid +from moneywiz_api.validation import require_integer_identity, require_valid ABS_TOLERANCE = 0.001 @@ -30,7 +30,16 @@ class Transaction(Record, ABC): def __init__(self, row): super().__init__(row) - self.reconciled = row["ZRECONCILED"] == 1 + raw_reconciled = row["ZRECONCILED"] + require_integer_identity( + raw_reconciled, + "transaction reconciled state must be an uncoerced integer", + ) + require_valid( + raw_reconciled in (0, 1), + "transaction reconciled state must be zero or one", + ) + self.reconciled = raw_reconciled == 1 self.amount = RDH.get_decimal(row, "ZAMOUNT1") self.description = row["ZDESC2"] self.datetime = RDH.get_datetime(row, "ZDATE1") @@ -83,8 +92,16 @@ def __init__(self, row): def validate(self): require_valid(self.account is not None, "deposit account is required") + require_integer_identity( + self.account, "deposit account must be an uncoerced integer" + ) require_valid(self.amount is not None, "deposit amount is required") # self.payee can be None + require_integer_identity( + self.payee, + "deposit payee must be an uncoerced integer", + optional=True, + ) require_valid( self.original_currency is not None, "deposit original currency is required" ) @@ -150,15 +167,26 @@ def __init__(self, row): def validate(self): require_valid(self.account is not None, "exchange account is required") + require_integer_identity( + self.account, "exchange account must be an uncoerced integer" + ) require_valid( self.from_investment_holding is not None, "exchange source holding is required", ) + require_integer_identity( + self.from_investment_holding, + "exchange source holding must be an uncoerced integer", + ) require_valid(self.from_symbol, "exchange source symbol is required") require_valid( self.to_investment_holding is not None, "exchange destination holding is required", ) + require_integer_identity( + self.to_investment_holding, + "exchange destination holding must be an uncoerced integer", + ) require_valid(self.to_symbol, "exchange destination symbol is required") require_valid( self.from_number_of_shares <= 0, @@ -235,6 +263,9 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): def validate(self): require_valid(self.account is not None, "investment buy account is required") + require_integer_identity( + self.account, "investment buy account must be an uncoerced integer" + ) require_valid(self.amount is not None, "investment buy amount is required") require_valid(self.amount <= 0, "investment buy amount must not be positive") require_valid(self.fee is not None, "investment buy fee is required") @@ -249,6 +280,10 @@ def validate(self): self.investment_holding is not None, "investment buy holding is required", ) + require_integer_identity( + self.investment_holding, + "investment buy holding must be an uncoerced integer", + ) require_valid( self.number_of_shares is not None, "investment buy quantity is required", @@ -321,6 +356,9 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): def validate(self): require_valid(self.account is not None, "investment sell account is required") + require_integer_identity( + self.account, "investment sell account must be an uncoerced integer" + ) require_valid(self.amount is not None, "investment sell amount is required") require_valid(self.fee is not None, "investment sell fee is required") @@ -336,6 +374,10 @@ def validate(self): self.investment_holding is not None, "investment sell holding is required", ) + require_integer_identity( + self.investment_holding, + "investment sell holding must be an uncoerced integer", + ) require_valid( self.number_of_shares is not None, "investment sell quantity is required", @@ -382,6 +424,9 @@ def __init__(self, row): def validate(self): require_valid(self.account is not None, "reconcile account is required") + require_integer_identity( + self.account, "reconcile account must be an uncoerced integer" + ) require_valid( self.reconcile_amount is not None or self.reconcile_number_of_shares is not None, @@ -425,6 +470,14 @@ def __init__(self, row): def validate(self): require_valid(self.account is not None, "refund account is required") + require_integer_identity( + self.account, "refund account must be an uncoerced integer" + ) + require_integer_identity( + self.payee, + "refund payee must be an uncoerced integer", + optional=True, + ) require_valid(self.amount is not None, "refund amount is required") require_valid(self.amount > 0, "refund amount must be positive") @@ -514,16 +567,27 @@ def __init__(self, row): def validate(self): require_valid(self.account is not None, "transfer deposit account is required") + require_integer_identity( + self.account, "transfer deposit account must be an uncoerced integer" + ) require_valid(self.amount is not None, "transfer deposit amount is required") require_valid(self.amount > 0, "transfer deposit amount must be positive") require_valid( self.sender_account is not None, "transfer deposit sender account is required", ) + require_integer_identity( + self.sender_account, + "transfer deposit sender account must be an uncoerced integer", + ) require_valid( self.sender_transaction is not None, "transfer deposit sender transaction is required", ) + require_integer_identity( + self.sender_transaction, + "transfer deposit sender transaction must be an uncoerced integer", + ) require_valid( self.original_amount is not None, "transfer deposit original amount is required", @@ -633,16 +697,27 @@ def validate(self): require_valid( self.account is not None, "transfer withdrawal account is required" ) + require_integer_identity( + self.account, "transfer withdrawal account must be an uncoerced integer" + ) require_valid(self.amount is not None, "transfer withdrawal amount is required") require_valid(self.amount < 0, "transfer withdrawal amount must be negative") require_valid( self.recipient_account is not None, "transfer withdrawal recipient account is required", ) + require_integer_identity( + self.recipient_account, + "transfer withdrawal recipient account must be an uncoerced integer", + ) require_valid( self.recipient_transaction is not None, "transfer withdrawal recipient transaction is required", ) + require_integer_identity( + self.recipient_transaction, + "transfer withdrawal recipient transaction must be an uncoerced integer", + ) require_valid( self.original_amount is not None, "transfer withdrawal original amount is required", @@ -732,8 +807,16 @@ def __init__(self, row): def validate(self): require_valid(self.account is not None, "withdrawal account is required") + require_integer_identity( + self.account, "withdrawal account must be an uncoerced integer" + ) require_valid(self.amount is not None, "withdrawal amount is required") # self.payee can be None + require_integer_identity( + self.payee, + "withdrawal payee must be an uncoerced integer", + optional=True, + ) require_valid( self.original_currency is not None, "withdrawal original currency is required", diff --git a/src/moneywiz_api/validation.py b/src/moneywiz_api/validation.py index 260aa64..ffcd695 100644 --- a/src/moneywiz_api/validation.py +++ b/src/moneywiz_api/validation.py @@ -2,3 +2,13 @@ def require_valid(condition: object, message: str) -> None: """Raise the historical validation exception even under optimized Python.""" if not condition: raise AssertionError(message) + + +def require_integer_identity( + value: object, message: str, *, optional: bool = False +) -> None: + """Require an uncoerced integer identity, optionally allowing ``None``.""" + if optional and value is None: + return + if type(value) is not int: + raise AssertionError(message) diff --git a/tests/unit/optimized_validation_probe.py b/tests/unit/optimized_validation_probe.py index 1b1ebfa..debf68f 100644 --- a/tests/unit/optimized_validation_probe.py +++ b/tests/unit/optimized_validation_probe.py @@ -6,6 +6,7 @@ import sqlite3 import sys import tempfile +from contextlib import contextmanager from decimal import Decimal from pathlib import Path @@ -168,6 +169,10 @@ def accessor_for(row): class PayeeAccessor: + @contextmanager + def read_transaction(self): + yield + def descendant_typenames(self, _roots): return ["Payee"] @@ -183,6 +188,15 @@ def query_objects(self, _typenames): "ZNAME5": "B", "ZUSER7": None, }, + { + **common_row( + Z_ENT=28, + ZGID="payee-invalid-identity", + Z_PK="PRIVATE_PAYLOAD", + ), + "ZNAME5": "C", + "ZUSER7": 1, + }, ] def typename_for(self, _ent_id): @@ -193,6 +207,10 @@ class CategoryAccessor: def __init__(self, rows): self.rows = rows + @contextmanager + def read_transaction(self): + yield + def descendant_typenames(self, _roots): return ["Category"] @@ -357,6 +375,8 @@ def relationship_probe(): (2, None, 101, 2.0), (3, 12, 102, "PRIVATE_PAYLOAD"), (4, 13, None, 4.0), + (5, "PRIVATE_PAYLOAD", 103, 1.0), + ("PRIVATE_PAYLOAD", 14, 104, 1.0), ], ) connection.execute( @@ -365,13 +385,19 @@ def relationship_probe(): ) connection.executemany( "INSERT INTO ZWITHDRAWREFUNDTRANSACTIONLINK VALUES (?, ?, ?)", - [(5, 103, 104), (6, 105, None)], + [ + (5, 103, 104), + (6, 105, None), + (7, "PRIVATE_PAYLOAD", 106), + ("PRIVATE_PAYLOAD", 107, 108), + ], ) connection.execute( "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" ) connection.executemany( - "INSERT INTO Z_37TAGS VALUES (?, ?)", [(100, 200), (101, None)] + "INSERT INTO Z_37TAGS VALUES (?, ?)", + [(100, 200), (101, None), ("PRIVATE_PAYLOAD", 201)], ) connection.commit() connection.close() @@ -462,6 +488,7 @@ def main(): ), "record_valid": capture(lambda: Record(common_row()).gid), "record_invalid_identity": capture(lambda: Record(common_row(ZGID=""))), + "record_coerced_identity": capture(lambda: Record(common_row(Z_PK=True))), "account_nullable_info": capture( lambda: { "info": valid_account.info, @@ -492,6 +519,28 @@ def main(): "deposit_invalid_sign": capture( lambda: DepositTransaction(transaction_row(ZORIGINALAMOUNT=-10.0)) ), + "deposit_coerced_account": capture( + lambda: DepositTransaction(transaction_row(ZACCOUNT2="PRIVATE_PAYLOAD")) + ), + "reconciled": { + label: capture( + lambda value=value: ( + DepositTransaction( + transaction_row(ZRECONCILED=value) + ).reconciled + ) + ) + for label, value in ( + ("none", None), + ("false_boolean", False), + ("true_boolean", True), + ("zero", 0), + ("one", 1), + ("two", 2), + ("float", 1.0), + ("text", "PRIVATE_PAYLOAD"), + ) + }, "deposit_invalid_fx": capture( lambda: DepositTransaction(transaction_row(ZAMOUNT1=20.01)) ), diff --git a/tests/unit/test_account_diagnostics.py b/tests/unit/test_account_diagnostics.py index 340cfc1..52e7dc1 100644 --- a/tests/unit/test_account_diagnostics.py +++ b/tests/unit/test_account_diagnostics.py @@ -1,3 +1,5 @@ +from contextlib import contextmanager + import pytest from moneywiz_api.managers.account_manager import AccountManager @@ -49,6 +51,10 @@ class AccountAccessor: def __init__(self, rows): self.rows = rows + @contextmanager + def read_transaction(self): + yield + def descendant_typenames(self, _roots): return ["CreditCardAccount"] diff --git a/tests/unit/test_api_atomic_reload.py b/tests/unit/test_api_atomic_reload.py index 6db5161..12aad5b 100644 --- a/tests/unit/test_api_atomic_reload.py +++ b/tests/unit/test_api_atomic_reload.py @@ -56,22 +56,29 @@ def __init__(self): self.generation = "old" self.fail_at = None self.extra_invalid_account = False - self.in_transaction = False + self.transaction_depth = 0 self.transaction_entries = 0 self.transaction_exits = 0 + self.exit_failure_depths = [] + + @property + def in_transaction(self): + return self.transaction_depth > 0 @contextmanager def read_transaction(self): - if self.fail_at == "enter": + entered_depth = self.transaction_depth + 1 + if self.fail_at == "enter" and entered_depth == 1: raise RuntimeError("synthetic transaction enter failure") self.transaction_entries += 1 - self.in_transaction = True + self.transaction_depth = entered_depth try: yield finally: - self.in_transaction = False + self.transaction_depth -= 1 self.transaction_exits += 1 - if self.fail_at == "exit": + if self.fail_at == "exit" and entered_depth == 1: + self.exit_failure_depths.append(entered_depth) raise RuntimeError("synthetic transaction exit failure") def descendant_typenames(self, _roots): @@ -208,6 +215,59 @@ def test_transaction_context_failure_does_not_publish(monkeypatch, failure) -> N assert not accessor.in_transaction +def test_outer_exit_failure_retains_all_published_state_and_can_retry( + monkeypatch, +) -> None: + accessor = SyntheticAccessor() + api = api_for(monkeypatch, accessor, ("accounts", "transactions")) + account_manager = api.account_manager + transaction_manager = api.transaction_manager + loaded_managers = api._loaded_managers + old_state = { + "account_records": account_manager._records, + "account_gid_index": account_manager._gid_to_id, + "account_report": account_manager.load_report, + "account": account_manager.get(1), + "transaction_records": transaction_manager._records, + "transaction_gid_index": transaction_manager._gid_to_id, + "transaction_report": transaction_manager.load_report, + "transaction": transaction_manager.get(10), + "categories": transaction_manager.category_assignment, + "refunds": transaction_manager.refund_maps, + "tags": transaction_manager.tags_map, + } + accessor.generation = "new" + accessor.fail_at = "exit" + + with pytest.raises(RuntimeError, match="synthetic transaction exit failure"): + api.load(("transactions",)) + + assert accessor.exit_failure_depths == [1] + assert accessor.transaction_depth == 0 + assert api.account_manager is account_manager + assert api.transaction_manager is transaction_manager + assert api._loaded_managers is loaded_managers + assert api._loaded_managers == {"accounts", "transactions"} + assert account_manager._records is old_state["account_records"] + assert account_manager._gid_to_id is old_state["account_gid_index"] + assert account_manager.load_report is old_state["account_report"] + assert account_manager.get(1) is old_state["account"] + assert transaction_manager._records is old_state["transaction_records"] + assert transaction_manager._gid_to_id is old_state["transaction_gid_index"] + assert transaction_manager.load_report is old_state["transaction_report"] + assert transaction_manager.get(10) is old_state["transaction"] + assert transaction_manager.category_assignment is old_state["categories"] + assert transaction_manager.refund_maps is old_state["refunds"] + assert transaction_manager.tags_map is old_state["tags"] + + accessor.fail_at = None + report = api.load(("transactions",)) + + assert report.complete + assert api.account_manager.get(2).name == "New" + assert api.transaction_manager.get(20).account == 2 + + def test_failed_new_selection_remains_unloaded_then_successful_retry_publishes( monkeypatch, ) -> None: @@ -281,11 +341,12 @@ def test_direct_transaction_relationship_failure_stays_unloaded_then_retries( def test_successful_publication_occurs_after_transaction_exit(monkeypatch) -> None: accessor = SyntheticAccessor() - adoption_states = [] + adoption_depths = [] original_adopt = AccountManager._adopt_loaded_state def observe_adoption(self, staged): - adoption_states.append(accessor.in_transaction) + assert accessor.transaction_depth == 0 + adoption_depths.append(accessor.transaction_depth) original_adopt(self, staged) monkeypatch.setattr(AccountManager, "_adopt_loaded_state", observe_adoption) @@ -294,8 +355,8 @@ def observe_adoption(self, staged): api.load(("accounts",)) - assert adoption_states == [False, False] - assert accessor.transaction_entries == accessor.transaction_exits == 2 + assert adoption_depths == [0, 0] + assert accessor.transaction_entries == accessor.transaction_exits == 4 def test_partial_staged_generation_is_published(monkeypatch) -> None: diff --git a/tests/unit/test_category_diagnostics.py b/tests/unit/test_category_diagnostics.py index 78b4ca8..26e6781 100644 --- a/tests/unit/test_category_diagnostics.py +++ b/tests/unit/test_category_diagnostics.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager import json import pytest @@ -26,6 +27,10 @@ class CategoryAccessor: def __init__(self, rows): self.rows = rows + @contextmanager + def read_transaction(self): + yield + def descendant_typenames(self, _roots): return ["Category"] diff --git a/tests/unit/test_identity_domains.py b/tests/unit/test_identity_domains.py new file mode 100644 index 0000000..8519163 --- /dev/null +++ b/tests/unit/test_identity_domains.py @@ -0,0 +1,418 @@ +from contextlib import contextmanager +from decimal import Decimal +import json + +import pytest + +from moneywiz_api.managers.transaction_manager import TransactionManager +from moneywiz_api.model.account import CashAccount +from moneywiz_api.model.category import Category +from moneywiz_api.model.investment_holding import InvestmentHolding +from moneywiz_api.model.payee import Payee +from moneywiz_api.model.record import Record +from moneywiz_api.model.tag import Tag +from moneywiz_api.model.transaction import ( + DepositTransaction, + InvestmentBuyTransaction, + InvestmentExchangeTransaction, + InvestmentSellTransaction, + ReconcileTransaction, + RefundTransaction, + TransferBudgetTransaction, + TransferDepositTransaction, + TransferWithdrawTransaction, + WithdrawTransaction, +) +from moneywiz_api.read_result import RelationshipLoadReport, RelationshipStorage +from tests.unit.accessor_test_support import initialized_memory_accessor + + +INVALID_IDENTITIES = (True, 1.0, "PRIVATE_PAYLOAD", Decimal("1")) + + +def record_row(**overrides): + row = { + "Z_ENT": 37, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": "record-1", + "Z_PK": 1, + } + row.update(overrides) + return row + + +def account_row(**overrides): + row = { + **record_row(Z_ENT=10, ZGID="account-1"), + "ZDISPLAYORDER": 0, + "ZGROUPID": 1, + "ZNAME": "Cash", + "ZCURRENCYNAME": "EUR", + "ZOPENINGBALANCE": 0.0, + "ZINFO": None, + "ZUSER": 2, + } + row.update(overrides) + return row + + +def payee_row(**overrides): + row = { + **record_row(Z_ENT=28, ZGID="payee-1"), + "ZNAME5": "Payee", + "ZUSER7": 2, + } + row.update(overrides) + return row + + +def category_row(**overrides): + row = { + **record_row(Z_ENT=19, ZGID="category-1"), + "ZNAME2": "Category", + "ZPARENTCATEGORY": None, + "ZTYPE2": 1, + "ZUSER3": 2, + } + row.update(overrides) + return row + + +def tag_row(**overrides): + row = { + **record_row(Z_ENT=35, ZGID="tag-1"), + "ZNAME6": "Tag", + "ZUSER8": 2, + } + row.update(overrides) + return row + + +def holding_row(**overrides): + row = { + **record_row(Z_ENT=24, ZGID="holding-1"), + "ZINVESTMENTACCOUNT": 2, + "ZOPENNINGNUMBEROFSHARES": None, + "ZNUMBEROFSHARES": 2.0, + "ZPRICEPERSHARE": 10.0, + "ZSYMBOL": "ACME", + "ZHOLDINGTYPE": None, + "ZDESC": "Acme", + "ZISPRICEPERSHAREAVAILABLEONLINE": 0, + "ZINVESTMENTOBJECTTYPE": 0, + "ZCOSTBASISOFMISSINGOBSHARES": 0.0, + } + row.update(overrides) + return row + + +def transaction_row(**overrides): + row = { + **record_row(Z_ENT=37, ZGID="transaction-1"), + "ZRECONCILED": 0, + "ZAMOUNT1": 10.0, + "ZDESC2": "Transaction", + "ZDATE1": 0.0, + "ZNOTES1": None, + "ZACCOUNT2": 2, + "ZPAYEE2": None, + "ZORIGINALCURRENCY": "EUR", + "ZORIGINALAMOUNT": 10.0, + "ZORIGINALEXCHANGERATE": 1.0, + } + row.update(overrides) + return row + + +def exchange_row(**overrides): + row = { + **transaction_row(Z_ENT=38, ZAMOUNT1=0.0), + "ZFROMINVESTMENTHOLDING": 3, + "ZFROMSYMBOL": "AAA", + "ZTOINVESTMENTHOLDING": 4, + "ZTOSYMBOL": "BBB", + "ZFROMNUMBEROFSHARES": -2.0, + "ZTONUMBEROFSHARES": 1.0, + "ZORIGINALFEE": 0.0, + "ZORIGINALFEECURRENCY": "AAA", + } + row.update(overrides) + return row + + +def buy_row(**overrides): + row = { + **transaction_row(Z_ENT=40, ZAMOUNT1=-20.0), + "ZFEE2": 0.0, + "ZINVESTMENTHOLDING": 3, + "ZNUMBEROFSHARES1": 2.0, + "ZPRICEPERSHARE1": 10.0, + } + row.update(overrides) + return row + + +def sell_row(**overrides): + row = { + **buy_row(Z_ENT=41, ZAMOUNT1=20.0), + } + row.update(overrides) + return row + + +def reconcile_row(**overrides): + row = { + **transaction_row(Z_ENT=42, ZAMOUNT1=0.0), + "ZRECONCILEAMOUNT": 0.0, + "ZRECONCILENUMBEROFSHARES": None, + } + row.update(overrides) + return row + + +def refund_row(**overrides): + row = {**transaction_row(Z_ENT=43), "ZPAYEE2": 5} + row.update(overrides) + return row + + +def transfer_budget_row(**overrides): + row = transaction_row(Z_ENT=44) + row.update(overrides) + return row + + +def transfer_deposit_row(**overrides): + row = { + **transaction_row(Z_ENT=45), + "ZSENDERACCOUNT": 3, + "ZSENDERTRANSACTION": 4, + "ZORIGINALSENDERAMOUNT": -10.0, + "ZORIGINALSENDERCURRENCY": "EUR", + "ZORIGINALFEE": None, + "ZORIGINALFEECURRENCY": None, + } + row.update(overrides) + return row + + +def transfer_withdraw_row(**overrides): + row = { + **transaction_row(Z_ENT=46, ZAMOUNT1=-10.0, ZORIGINALAMOUNT=-10.0), + "ZRECIPIENTACCOUNT1": 3, + "ZRECIPIENTTRANSACTION": 4, + "ZORIGINALRECIPIENTAMOUNT": 10.0, + "ZORIGINALRECIPIENTCURRENCY": "EUR", + "ZORIGINALFEE": None, + "ZORIGINALFEECURRENCY": None, + } + row.update(overrides) + return row + + +def withdraw_row(**overrides): + row = transaction_row(Z_ENT=47, ZAMOUNT1=-10.0, ZORIGINALAMOUNT=-10.0) + row.update(overrides) + return row + + +IDENTITY_FIELDS = ( + (Record, record_row, "Z_PK"), + (Record, record_row, "Z_ENT"), + (CashAccount, account_row, "ZGROUPID"), + (CashAccount, account_row, "ZUSER"), + (Payee, payee_row, "ZUSER7"), + (Category, category_row, "ZPARENTCATEGORY"), + (Category, category_row, "ZUSER3"), + (Tag, tag_row, "ZUSER8"), + (InvestmentHolding, holding_row, "ZINVESTMENTACCOUNT"), + (DepositTransaction, transaction_row, "ZACCOUNT2"), + (DepositTransaction, transaction_row, "ZPAYEE2"), + (InvestmentExchangeTransaction, exchange_row, "ZACCOUNT2"), + (InvestmentExchangeTransaction, exchange_row, "ZFROMINVESTMENTHOLDING"), + (InvestmentExchangeTransaction, exchange_row, "ZTOINVESTMENTHOLDING"), + (InvestmentBuyTransaction, buy_row, "ZACCOUNT2"), + (InvestmentBuyTransaction, buy_row, "ZINVESTMENTHOLDING"), + (InvestmentSellTransaction, sell_row, "ZACCOUNT2"), + (InvestmentSellTransaction, sell_row, "ZINVESTMENTHOLDING"), + (ReconcileTransaction, reconcile_row, "ZACCOUNT2"), + (RefundTransaction, refund_row, "ZACCOUNT2"), + (RefundTransaction, refund_row, "ZPAYEE2"), + (TransferDepositTransaction, transfer_deposit_row, "ZACCOUNT2"), + (TransferDepositTransaction, transfer_deposit_row, "ZSENDERACCOUNT"), + (TransferDepositTransaction, transfer_deposit_row, "ZSENDERTRANSACTION"), + (TransferWithdrawTransaction, transfer_withdraw_row, "ZACCOUNT2"), + (TransferWithdrawTransaction, transfer_withdraw_row, "ZRECIPIENTACCOUNT1"), + (TransferWithdrawTransaction, transfer_withdraw_row, "ZRECIPIENTTRANSACTION"), + (WithdrawTransaction, withdraw_row, "ZACCOUNT2"), + (WithdrawTransaction, withdraw_row, "ZPAYEE2"), +) + + +@pytest.mark.parametrize(("constructor", "row_factory", "field"), IDENTITY_FIELDS) +@pytest.mark.parametrize("invalid_identity", INVALID_IDENTITIES) +def test_model_identity_fields_reject_coerced_domains( + constructor, row_factory, field, invalid_identity +) -> None: + with pytest.raises(AssertionError, match="uncoerced integer"): + constructor(row_factory(**{field: invalid_identity})) + + +@pytest.mark.parametrize(("constructor", "row_factory", "field"), IDENTITY_FIELDS) +def test_model_identity_fields_preserve_valid_integers( + constructor, row_factory, field +) -> None: + constructor(row_factory(**{field: 7})) + + +@pytest.mark.parametrize( + ("constructor", "row_factory", "field"), + [ + (CashAccount, account_row, "ZGROUPID"), + (CashAccount, account_row, "ZUSER"), + (Payee, payee_row, "ZUSER7"), + (Category, category_row, "ZPARENTCATEGORY"), + (Tag, tag_row, "ZUSER8"), + (InvestmentHolding, holding_row, "ZINVESTMENTACCOUNT"), + (DepositTransaction, transaction_row, "ZACCOUNT2"), + (DepositTransaction, transaction_row, "ZPAYEE2"), + ], +) +def test_non_record_identity_fields_preserve_integer_zero( + constructor, row_factory, field +) -> None: + constructor(row_factory(**{field: 0})) + + +@pytest.mark.parametrize( + ("constructor", "row_factory", "field"), + [ + (Category, category_row, "ZPARENTCATEGORY"), + (DepositTransaction, transaction_row, "ZPAYEE2"), + (RefundTransaction, refund_row, "ZPAYEE2"), + (WithdrawTransaction, withdraw_row, "ZPAYEE2"), + ], +) +def test_optional_identity_fields_accept_none(constructor, row_factory, field) -> None: + constructor(row_factory(**{field: None})) + + +TRANSACTION_ROWS = ( + (DepositTransaction, transaction_row), + (InvestmentExchangeTransaction, exchange_row), + (InvestmentBuyTransaction, buy_row), + (InvestmentSellTransaction, sell_row), + (ReconcileTransaction, reconcile_row), + (RefundTransaction, refund_row), + (TransferBudgetTransaction, transfer_budget_row), + (TransferDepositTransaction, transfer_deposit_row), + (TransferWithdrawTransaction, transfer_withdraw_row), + (WithdrawTransaction, withdraw_row), +) + + +@pytest.mark.parametrize(("constructor", "row_factory"), TRANSACTION_ROWS) +@pytest.mark.parametrize(("raw_value", "expected"), [(0, False), (1, True)]) +def test_reconciled_raw_integer_domain_is_preserved( + constructor, row_factory, raw_value, expected +) -> None: + assert constructor(row_factory(ZRECONCILED=raw_value)).reconciled is expected + + +@pytest.mark.parametrize(("constructor", "row_factory"), TRANSACTION_ROWS) +@pytest.mark.parametrize("raw_value", [None, True, 2, 1.0, "PRIVATE_PAYLOAD"]) +def test_reconciled_refuses_non_binary_or_coerced_values( + constructor, row_factory, raw_value +) -> None: + with pytest.raises(AssertionError) as error: + constructor(row_factory(ZRECONCILED=raw_value)) + assert "PRIVATE_PAYLOAD" not in str(error.value) + + +def test_manager_skips_invalid_reconciled_row_and_keeps_valid_row() -> None: + rows = [ + transaction_row(ZRECONCILED=2), + transaction_row(Z_PK=2, ZGID="transaction-2", ZRECONCILED=1), + ] + + report = TransactionManager().load(TransactionAccessor(rows)) + + assert report.source_ids == (1, 2) + assert report.parsed_ids == (2,) + assert report.skipped[0].error.value == "validation" + assert report.status == "partial" + + +@pytest.mark.parametrize(("raw_value", "expected"), [(0, False), (1, True)]) +def test_accessor_preserves_binary_reconciled_domain(raw_value, expected) -> None: + accessor = initialized_memory_accessor( + [transaction_row(ZRECONCILED=raw_value)], + [(37, "DepositTransaction", 0)], + ) + + transaction = accessor.get_record(1, DepositTransaction) + + assert transaction.reconciled is expected + + +@pytest.mark.parametrize("raw_value", [None, 2, 1.0, "PRIVATE_PAYLOAD"]) +def test_accessor_refuses_invalid_reconciled_domain(raw_value) -> None: + accessor = initialized_memory_accessor( + [transaction_row(ZRECONCILED=raw_value)], + [(37, "DepositTransaction", 0)], + ) + + with pytest.raises(AssertionError): + accessor.get_record(1, DepositTransaction) + + +class TransactionAccessor: + def __init__(self, rows): + self.rows = rows + + @contextmanager + def read_transaction(self): + yield + + def descendant_typenames(self, _roots): + return ["DepositTransaction"] + + def query_objects(self, _typenames): + return self.rows + + def typename_for(self, ent_id): + return {37: "DepositTransaction"}.get(ent_id) + + def read_category_assignments(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def read_refund_maps(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def read_tags_map(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + +@pytest.mark.parametrize( + "invalid_row", + [ + transaction_row(Z_PK="PRIVATE_PAYLOAD"), + transaction_row(Z_ENT="PRIVATE_PAYLOAD"), + transaction_row(ZACCOUNT2="PRIVATE_PAYLOAD"), + transaction_row(ZRECONCILED="PRIVATE_PAYLOAD"), + ], +) +def test_manager_diagnostics_do_not_publish_invalid_identity_payloads( + invalid_row, caplog +) -> None: + valid_row = transaction_row(Z_PK=2, ZGID="transaction-2") + caplog.set_level("DEBUG") + + report = TransactionManager().load(TransactionAccessor([invalid_row, valid_row])) + serialized = json.dumps(report.as_dict(), sort_keys=True) + + assert report.source_count == 2 + assert report.parsed_ids == (2,) + assert len(report.skipped) == 1 + assert "PRIVATE_PAYLOAD" not in serialized + assert "PRIVATE_PAYLOAD" not in caplog.text diff --git a/tests/unit/test_manager_transaction_lifecycle.py b/tests/unit/test_manager_transaction_lifecycle.py new file mode 100644 index 0000000..73f9065 --- /dev/null +++ b/tests/unit/test_manager_transaction_lifecycle.py @@ -0,0 +1,212 @@ +from contextlib import contextmanager +from decimal import Decimal +import sqlite3 + +import pytest + +from moneywiz_api.database_accessor import DatabaseAccessor +from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.managers.category_manager import CategoryManager +from moneywiz_api.managers.investment_holding_manager import ( + InvestmentHoldingManager, +) +from moneywiz_api.managers.payee_manager import PayeeManager +from moneywiz_api.managers.tag_manager import TagManager +from moneywiz_api.managers.transaction_manager import TransactionManager +from moneywiz_api.read_result import RelationshipLoadReport, RelationshipStorage + + +BUILTIN_MANAGERS = ( + AccountManager, + PayeeManager, + CategoryManager, + TransactionManager, + InvestmentHoldingManager, + TagManager, +) + + +class EmptyAccessor: + def __init__(self): + self.entries = 0 + self.exits = 0 + self.depth = 0 + self.max_depth = 0 + self.fail = None + + @contextmanager + def read_transaction(self): + if self.fail == "enter": + raise RuntimeError("synthetic transaction enter failure") + self.entries += 1 + self.depth += 1 + self.max_depth = max(self.max_depth, self.depth) + try: + yield + finally: + self.depth -= 1 + self.exits += 1 + if self.fail == "exit": + raise RuntimeError("synthetic transaction exit failure") + + def descendant_typenames(self, _roots): + assert self.depth == 1 + return [] + + def query_objects(self, _typenames): + assert self.depth == 1 + if self.fail == "query": + raise RuntimeError("synthetic query failure") + return [] + + def read_category_assignments(self): + assert self.depth == 1 + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def read_refund_maps(self): + assert self.depth == 1 + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def read_tags_map(self): + assert self.depth == 1 + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + +@pytest.mark.parametrize("manager_type", BUILTIN_MANAGERS) +def test_each_public_manager_load_owns_one_outer_transaction(manager_type) -> None: + accessor = EmptyAccessor() + manager = manager_type() + + report = manager.load(accessor) + + assert report.complete + assert accessor.entries == accessor.exits == 1 + assert accessor.max_depth == 1 + + +@pytest.mark.parametrize("manager_type", BUILTIN_MANAGERS) +@pytest.mark.parametrize("failure", ["enter", "query", "exit"]) +def test_each_public_manager_failure_discards_observation( + manager_type, failure +) -> None: + accessor = EmptyAccessor() + manager = manager_type() + assert manager.load(accessor).complete + accessor.fail = failure + + with pytest.raises(RuntimeError, match="synthetic"): + manager.load(accessor) + + assert manager.records() == {} + assert manager.load_report.status == "unloaded" + if isinstance(manager, TransactionManager): + assert manager.category_assignment == {} + assert manager.refund_maps == {} + assert manager.tags_map == {} + accessor.fail = None + assert manager.load(accessor).complete + + +def _create_wal_transaction_store(path) -> None: + connection = sqlite3.connect(path) + connection.execute("PRAGMA journal_mode=WAL") + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, 0)", + [ + (2, "CategoryAssigment", 0), + (8, "SyncObject", 0), + (35, "Tag", 8), + (36, "Transaction", 8), + (37, "DepositTransaction", 36), + (50, "WithdrawRefundTransactionLink", 0), + ], + ) + connection.execute( + "CREATE TABLE ZSYNCOBJECT (" + "Z_PK INTEGER, Z_ENT INTEGER, ZOBJECTCREATIONDATE FLOAT, ZGID TEXT, " + "ZRECONCILED INTEGER, ZAMOUNT1 FLOAT, ZDESC2 TEXT, ZDATE1 FLOAT, " + "ZNOTES1 TEXT, ZACCOUNT2 INTEGER, ZPAYEE2 INTEGER, " + "ZORIGINALCURRENCY TEXT, ZORIGINALAMOUNT FLOAT, " + "ZORIGINALEXCHANGERATE FLOAT)" + ) + connection.execute( + "INSERT INTO ZSYNCOBJECT VALUES " + "(100, 37, 0, 'transaction-100', 0, 5, 'old', 0, NULL, 1, NULL, " + "'EUR', 5, 1)" + ) + connection.execute( + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)" + ) + connection.execute("INSERT INTO ZCATEGORYASSIGMENT VALUES (1, 7, 100, 5)") + connection.execute( + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, ZWITHDRAWTRANSACTION INTEGER)" + ) + connection.execute("INSERT INTO ZWITHDRAWREFUNDTRANSACTIONLINK VALUES (2, 100, 90)") + connection.execute( + "CREATE TABLE Z_36TAGS (Z_36TRANSACTIONS INTEGER, Z_35TAGS INTEGER)" + ) + connection.execute("INSERT INTO Z_36TAGS VALUES (100, 9)") + connection.commit() + connection.close() + + +def _mutate_wal_transaction_store(path) -> None: + connection = sqlite3.connect(path) + connection.execute("UPDATE ZSYNCOBJECT SET ZAMOUNT1 = 6, ZORIGINALAMOUNT = 6") + connection.execute("UPDATE ZCATEGORYASSIGMENT SET ZAMOUNT = 7") + connection.execute( + "UPDATE ZWITHDRAWREFUNDTRANSACTIONLINK SET ZWITHDRAWTRANSACTION = 91" + ) + connection.execute("UPDATE Z_36TAGS SET Z_35TAGS = 10") + connection.commit() + connection.close() + + +@pytest.mark.parametrize( + "interleave_after", + ["transaction_rows", "category_assignments", "refund_links"], +) +def test_direct_transaction_manager_load_uses_one_wal_generation( + tmp_path, monkeypatch, interleave_after +) -> None: + path = tmp_path / "manager-generation.sqlite" + _create_wal_transaction_store(path) + + with DatabaseAccessor(path) as accessor: + method_name = { + "transaction_rows": "query_objects", + "category_assignments": "read_category_assignments", + "refund_links": "read_refund_maps", + }[interleave_after] + original = getattr(accessor, method_name) + mutated = False + + def read_then_mutate(*args, **kwargs): + nonlocal mutated + result = original(*args, **kwargs) + if not mutated: + _mutate_wal_transaction_store(path) + mutated = True + return result + + monkeypatch.setattr(accessor, method_name, read_then_mutate) + manager = TransactionManager() + report = manager.load(accessor) + + assert report.complete + assert all(item.complete for item in report.relationships.values()) + assert manager.get(100).amount == Decimal("5.0") + assert manager.category_assignment == {100: [(7, Decimal("5.0"))]} + assert manager.refund_maps == {100: 90} + assert manager.tags_map == {100: [9]} + + +def test_transaction_manager_uses_shared_public_load_boundary() -> None: + assert "load" not in TransactionManager.__dict__ + assert "_load_in_transaction" in TransactionManager.__dict__ diff --git a/tests/unit/test_optimized_validation.py b/tests/unit/test_optimized_validation.py index 907603e..7cc74f0 100644 --- a/tests/unit/test_optimized_validation.py +++ b/tests/unit/test_optimized_validation.py @@ -63,6 +63,7 @@ def test_probe_covers_success_error_and_completeness_paths( } assert cases["date_string"]["type"] == "AssertionError" assert cases["record_invalid_identity"]["type"] == "AssertionError" + assert cases["record_coerced_identity"]["type"] == "AssertionError" assert cases["account_nullable_info"]["status"] == "ok" assert cases["account_invalid_name_public"]["type"] == "AssertionError" direct_accounts = cases["direct_accounts"] @@ -99,10 +100,28 @@ def test_probe_covers_success_error_and_completeness_paths( named_entities[name]["type"] == "AssertionError" for name in ("payee_invalid", "tag_invalid", "category_invalid") ) + assert cases["manager"]["source_ids"] == [10, 11, None] assert cases["manager"]["parsed_count"] == 1 - assert cases["manager"]["skipped"][0]["error"] == "validation" + assert all( + skipped["error"] == "validation" for skipped in cases["manager"]["skipped"] + ) assert cases["holding_missing_quantity"]["type"] == "AssertionError" assert cases["deposit_invalid_sign"]["type"] == "AssertionError" + assert cases["deposit_coerced_account"]["type"] == "AssertionError" + reconciled = cases["reconciled"] + assert reconciled["zero"] == {"status": "ok", "value": False} + assert reconciled["one"] == {"status": "ok", "value": True} + assert all( + reconciled[value]["type"] == "AssertionError" + for value in ( + "none", + "false_boolean", + "true_boolean", + "two", + "float", + "text", + ) + ) assert cases["deposit_invalid_fx"]["type"] == "AssertionError" assert cases["deposit_zero_rate"] == {"status": "ok", "value": None} assert cases["investment_buy_invalid_quantity"]["type"] == "AssertionError" @@ -111,12 +130,15 @@ def test_probe_covers_success_error_and_completeness_paths( relationships = cases["relationships"] assert relationships["category_report"]["parsed_count"] == 1 - assert [item["error"] for item in relationships["category_report"]["skipped"]] == [ - "validation", - "validation", - ] + assert all( + item["error"] == "validation" + for item in relationships["category_report"]["skipped"] + ) + assert relationships["category_report"]["source_ids"][-1] is None assert relationships["refund_report"]["parsed_count"] == 1 + assert relationships["refund_report"]["source_ids"][-1] is None assert relationships["tag_report"]["parsed_count"] == 1 + assert relationships["tag_report"]["source_ids"][-1] == "row:2" assert not relationships["aggregate"]["complete"] serialized = json.dumps(cases["sentinel"], sort_keys=True) diff --git a/tests/unit/test_profiled_investment_values.py b/tests/unit/test_profiled_investment_values.py index 4a78d9b..22c9415 100644 --- a/tests/unit/test_profiled_investment_values.py +++ b/tests/unit/test_profiled_investment_values.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager from decimal import Decimal import pytest @@ -137,6 +138,10 @@ def __init__(self, typename: str, row: dict): self.typename = typename self.row = row + @contextmanager + def read_transaction(self): + yield + def query_objects(self, _typenames): return [self.row] diff --git a/tests/unit/test_read_completeness.py b/tests/unit/test_read_completeness.py index f220eb6..c0686e6 100644 --- a/tests/unit/test_read_completeness.py +++ b/tests/unit/test_read_completeness.py @@ -79,6 +79,10 @@ def __init__(self, rows, typenames=None): self.rows = rows self.typenames = typenames or {1: "ExampleRecord"} + @contextmanager + def read_transaction(self): + yield + def descendant_typenames(self, _roots): return list(self.typenames.values()) @@ -185,17 +189,27 @@ def __init__(self, _db_file): ) self.queries = [] self.transaction_entries = 0 + self.transaction_depth = 0 + self.max_transaction_depth = 0 self.closed = False @contextmanager def read_transaction(self): self.transaction_entries += 1 - yield + self.transaction_depth += 1 + self.max_transaction_depth = max( + self.max_transaction_depth, self.transaction_depth + ) + try: + yield + finally: + self.transaction_depth -= 1 def descendant_typenames(self, _roots): return [] def query_objects(self, typenames): + assert self.transaction_depth == 2 self.queries.append(tuple(typenames)) return [] @@ -203,12 +217,15 @@ def typename_for(self, _ent_id): return None def read_category_assignments(self): + assert self.transaction_depth == 2 return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) def read_refund_maps(self): + assert self.transaction_depth == 2 return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) def read_tags_map(self): + assert self.transaction_depth == 2 return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) def close(self): @@ -233,7 +250,8 @@ def test_api_supports_scoped_loads_and_json_safe_snapshot(monkeypatch) -> None: "unsuffixed-investment-columns" ) assert api.payee_manager.load_report.status == "unloaded" - assert api.accessor.transaction_entries == 1 + assert api.accessor.transaction_entries == 3 + assert api.accessor.max_transaction_depth == 2 json.dumps(snapshot) assert api.accessor.closed @@ -247,7 +265,8 @@ def test_api_preserves_default_eager_loading(monkeypatch) -> None: with api_module.MoneywizApi("unused.sqlite") as api: assert list(api.completeness().managers) == list(api.MANAGER_NAMES) assert len(api.accessor.queries) == len(api.MANAGER_NAMES) - assert api.accessor.transaction_entries == 1 + assert api.accessor.transaction_entries == 7 + assert api.accessor.max_transaction_depth == 2 def test_api_rejects_unloaded_or_unknown_manager(monkeypatch) -> None: diff --git a/tests/unit/test_relationship_completeness.py b/tests/unit/test_relationship_completeness.py index ce90fc8..dd7b782 100644 --- a/tests/unit/test_relationship_completeness.py +++ b/tests/unit/test_relationship_completeness.py @@ -1,3 +1,4 @@ +import json import sqlite3 import pytest @@ -188,6 +189,144 @@ def test_shifted_relationship_layouts_report_valid_and_skipped_rows(tmp_path) -> ) +@pytest.mark.parametrize( + ( + "reader_name", + "metadata", + "table_statement", + "valid_row", + "invalid_row", + "expected", + "expected_sources", + "expected_parsed", + ), + [ + ( + "read_category_assignments", + ((2, "CategoryAssigment", 0, 0),), + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)", + (1, 10, 100, 5.0), + (2, "PRIVATE_PAYLOAD", 101, 6.0), + {100: [(10, 5)]}, + (1, 2), + (1,), + ), + ( + "read_category_assignments", + ((2, "CategoryAssigment", 0, 0),), + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)", + (1, 10, 100, 5.0), + (2, 11, "PRIVATE_PAYLOAD", 6.0), + {100: [(10, 5)]}, + (1, 2), + (1,), + ), + ( + "read_category_assignments", + ((2, "CategoryAssigment", 0, 0),), + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)", + (1, 10, 100, 5.0), + ("PRIVATE_PAYLOAD", 11, 101, 6.0), + {100: [(10, 5)]}, + (1, None), + (1,), + ), + ( + "read_refund_maps", + ((50, "WithdrawRefundTransactionLink", 0, 0),), + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, " + "ZWITHDRAWTRANSACTION INTEGER)", + (1, 100, 90), + (2, "PRIVATE_PAYLOAD", 91), + {100: 90}, + (1, 2), + (1,), + ), + ( + "read_refund_maps", + ((50, "WithdrawRefundTransactionLink", 0, 0),), + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, " + "ZWITHDRAWTRANSACTION INTEGER)", + (1, 100, 90), + (2, 101, "PRIVATE_PAYLOAD"), + {100: 90}, + (1, 2), + (1,), + ), + ( + "read_refund_maps", + ((50, "WithdrawRefundTransactionLink", 0, 0),), + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, " + "ZWITHDRAWTRANSACTION INTEGER)", + (1, 100, 90), + ("PRIVATE_PAYLOAD", 101, 91), + {100: 90}, + (1, None), + (1,), + ), + ( + "read_tags_map", + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + (100, 200), + ("PRIVATE_PAYLOAD", 201), + {100: [200]}, + ("100:200", "row:1"), + ("100:200",), + ), + ( + "read_tags_map", + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + (100, 200), + (101, "PRIVATE_PAYLOAD"), + {100: [200]}, + ("100:200", "row:1"), + ("100:200",), + ), + ], +) +def test_integer_affinity_relationships_refuse_noninteger_raw_identities( + tmp_path, + reader_name, + metadata, + table_statement, + valid_row, + invalid_row, + expected, + expected_sources, + expected_parsed, +) -> None: + path = tmp_path / f"{reader_name}-identity-domain.sqlite" + create_custom_relationship_schema( + path, metadata=metadata, tables=(table_statement,) + ) + table_name = table_statement.split()[2] + with sqlite3.connect(path) as connection: + placeholders = ", ".join("?" for _ in valid_row) + connection.executemany( + f"INSERT INTO {table_name} VALUES ({placeholders})", + (valid_row, invalid_row), + ) + + with DatabaseAccessor(path) as accessor: + data, report = getattr(accessor, reader_name)() + + assert data == expected + assert report.source_ids == expected_sources + assert report.parsed_ids == expected_parsed + assert report.source_count == 2 + assert report.parsed_count == 1 + assert report.skipped[0].error.value == "validation" + assert "PRIVATE_PAYLOAD" not in json.dumps(report.as_dict(), sort_keys=True) + + def test_category_counts_exclude_supported_nontransaction_owners(tmp_path) -> None: path = tmp_path / "nontransaction-category-owners.sqlite" create_custom_relationship_schema( diff --git a/tests/unit/test_transaction_compatibility.py b/tests/unit/test_transaction_compatibility.py index 907706b..aa7a0e6 100644 --- a/tests/unit/test_transaction_compatibility.py +++ b/tests/unit/test_transaction_compatibility.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager from decimal import Decimal import pytest @@ -67,6 +68,10 @@ class TransferWithdrawAccessor: def __init__(self, rows): self.rows = rows + @contextmanager + def read_transaction(self): + yield + def query_objects(self, _typenames): return self.rows From cd1f8b5b81426ea7bb1f05fefcc9501453390d7c Mon Sep 17 00:00:00 2001 From: MarcoMC Date: Sun, 13 Sep 2026 10:44:08 +0200 Subject: [PATCH 08/10] Bound JSON snapshots and reuse verified read admission Reject malformed global IDs and supported text metadata without coercion. Validate recursive snapshot values and keys before publication while preserving raw model output. Admit each active outer read scope once, reverify independent scopes and latch transaction loss. Cover identity, serialization, WAL, external ownership and unsafe admission mutants. --- README.md | 18 +- src/moneywiz_api/database_accessor.py | 48 ++++- src/moneywiz_api/model/account.py | 10 +- src/moneywiz_api/model/category.py | 7 +- src/moneywiz_api/model/payee.py | 7 +- src/moneywiz_api/model/record.py | 7 +- src/moneywiz_api/model/tag.py | 7 +- src/moneywiz_api/read_result.py | 40 +++- src/moneywiz_api/validation.py | 8 + tests/unit/optimized_validation_probe.py | 27 +++ tests/unit/test_accessor_admission.py | 251 +++++++++++++++++++++++ tests/unit/test_identity_domains.py | 127 ++++++++++++ tests/unit/test_optimized_validation.py | 12 +- tests/unit/test_read_result_ownership.py | 156 +++++++++++++- 14 files changed, 702 insertions(+), 23 deletions(-) create mode 100644 tests/unit/test_accessor_admission.py diff --git a/README.md b/README.md index ba95275..f822c16 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,10 @@ completeness also reports category, refund, and tag relationship storage as partial. Category relationship counts cover transaction assignments only; rows owned by budgets, scheduled transactions, or history items are outside that map. `snapshot` contains JSON-safe records, completeness, and the selected -schema profile. +schema profile. Snapshot construction refuses unsupported leaves such as BLOBs, +arbitrary objects, non-finite built-in floats, unsupported mapping keys, and +keys that collide after normalization. It does not encode or stringify those +values. Legacy model `as_dict()` methods remain raw compatibility interfaces. Published completeness and snapshot values own immutable nested mappings and sequences, so later caller or manager changes cannot rewrite earlier evidence. @@ -106,9 +109,16 @@ the previously published records and completeness evidence remain available. Direct `manager.load(accessor)` calls use the same boundary: records, transaction relationships, and the published report come from one read snapshot, while a failed load resets that manager to `unloaded`. - -Core Data identity and relationship endpoint fields must arrive as uncoerced -integers; nullable payee and category-parent references remain supported. +Nested accessor reads reuse the outer scope's successful schema and source +admission. Each independent public scope revalidates, including scopes inside a +caller-owned SQLite transaction. If SQLite ends an admitted transaction, later +nested reads refuse the lost snapshot until the outer scope unwinds. + +Core Data record global IDs must be nonempty strings. Identity and relationship +endpoint fields must arrive as uncoerced integers; nullable payee and +category-parent references remain supported. Account, payee, category, and tag +names must be strings while preserving existing empty-string behavior; account +info also permits `None`. Transaction reconciliation accepts only raw integer `0` or `1` before exposing the value as `bool`. Refused rows remain visible through identity-only completeness diagnostics without embedding malformed payload values. diff --git a/src/moneywiz_api/database_accessor.py b/src/moneywiz_api/database_accessor.py index ea2952d..ddbd208 100644 --- a/src/moneywiz_api/database_accessor.py +++ b/src/moneywiz_api/database_accessor.py @@ -79,6 +79,8 @@ def dict_factory(cursor, row): def _initialize_schema_cache(self) -> None: """Bind all schema-dependent caches from one SQLite snapshot.""" + self._admission_depth = 0 + self._admission_broken = False with self._raw_read_transaction(): if not self._table_exists("Z_PRIMARYKEY") or not self._table_exists( "ZSYNCOBJECT" @@ -340,9 +342,15 @@ def _raw_read_transaction(self): try: yield finally: - if owns_transaction and self._con.in_transaction: + if owns_transaction and self._transaction_is_active(): self._con.rollback() + def _transaction_is_active(self) -> bool: + try: + return self._con.in_transaction + except sqlite3.Error: + return False + def _verify_schema_identity(self) -> None: try: baseline = self._schema_identity @@ -392,10 +400,42 @@ def _verify_source_eligibility(self) -> None: @contextmanager def read_transaction(self): """Keep cache-dependent reads on one verified SQLite snapshot.""" + admission_depth = getattr(self, "_admission_depth", 0) + admission_broken = getattr(self, "_admission_broken", False) + if admission_depth and (admission_broken or not self._transaction_is_active()): + self._admission_broken = True + raise DatabaseSchemaError( + "database read transaction was interrupted; close and reopen the accessor" + ) + with self._raw_read_transaction(): - self._verify_schema_identity() - self._verify_source_eligibility() - yield + owns_admission = admission_depth == 0 + if owns_admission: + self._verify_schema_identity() + self._verify_source_eligibility() + if not self._transaction_is_active(): + raise DatabaseSchemaError( + "database read transaction was interrupted; " + "close and reopen the accessor" + ) + self._admission_depth += 1 + try: + yield + except BaseException: + if not self._transaction_is_active(): + self._admission_broken = True + raise + else: + if self._admission_broken or not self._transaction_is_active(): + self._admission_broken = True + raise DatabaseSchemaError( + "database read transaction was interrupted; " + "close and reopen the accessor" + ) + finally: + self._admission_depth -= 1 + if owns_admission: + self._admission_broken = False def __enter__(self): return self diff --git a/src/moneywiz_api/model/account.py b/src/moneywiz_api/model/account.py index 0688853..a9b4848 100644 --- a/src/moneywiz_api/model/account.py +++ b/src/moneywiz_api/model/account.py @@ -6,7 +6,11 @@ from moneywiz_api.types import ID from moneywiz_api.model.raw_data_handler import RawDataHandler as RDH from moneywiz_api.model.record import Record -from moneywiz_api.validation import require_integer_identity, require_valid +from moneywiz_api.validation import ( + require_integer_identity, + require_text, + require_valid, +) @dataclass @@ -46,11 +50,15 @@ def validate(self) -> None: self.group_id, "account group identity must be an uncoerced integer" ) require_valid(self.name is not None, "account name is required") + require_text(self.name, "account name must be an uncoerced string") require_valid(self.currency is not None, "account currency is required") require_valid( self.opening_balance is not None, "account opening balance is required" ) # info is nullable in valid MoneyWiz stores. + require_text( + self.info, "account info must be an uncoerced string", optional=True + ) require_valid(self.user is not None, "account owner is required") require_integer_identity( self.user, "account owner must be an uncoerced integer" diff --git a/src/moneywiz_api/model/category.py b/src/moneywiz_api/model/category.py index e1df291..71c5886 100644 --- a/src/moneywiz_api/model/category.py +++ b/src/moneywiz_api/model/category.py @@ -3,7 +3,11 @@ from moneywiz_api.model.record import Record from moneywiz_api.types import CategoryType, ID -from moneywiz_api.validation import require_integer_identity, require_valid +from moneywiz_api.validation import ( + require_integer_identity, + require_text, + require_valid, +) @dataclass @@ -28,6 +32,7 @@ def __init__(self, row): # Validate require_valid(self.name is not None, "category name is required") + require_text(self.name, "category name must be an uncoerced string") require_integer_identity( self.parent_id, "category parent must be an uncoerced integer", diff --git a/src/moneywiz_api/model/payee.py b/src/moneywiz_api/model/payee.py index 86a8438..b71be40 100644 --- a/src/moneywiz_api/model/payee.py +++ b/src/moneywiz_api/model/payee.py @@ -2,7 +2,11 @@ from moneywiz_api.model.record import Record from moneywiz_api.types import ID -from moneywiz_api.validation import require_integer_identity, require_valid +from moneywiz_api.validation import ( + require_integer_identity, + require_text, + require_valid, +) @dataclass @@ -23,5 +27,6 @@ def __init__(self, row): # Validate require_valid(self.name is not None, "payee name is required") + require_text(self.name, "payee name must be an uncoerced string") require_valid(self.user is not None, "payee owner is required") require_integer_identity(self.user, "payee owner must be an uncoerced integer") diff --git a/src/moneywiz_api/model/record.py b/src/moneywiz_api/model/record.py index c148a5a..c11a188 100644 --- a/src/moneywiz_api/model/record.py +++ b/src/moneywiz_api/model/record.py @@ -4,7 +4,11 @@ from moneywiz_api.types import ID, ENT_ID from moneywiz_api.model.raw_data_handler import RawDataHandler as RDH -from moneywiz_api.validation import require_integer_identity, require_valid +from moneywiz_api.validation import ( + require_integer_identity, + require_text, + require_valid, +) @dataclass @@ -32,6 +36,7 @@ def __init__(self, row): ) require_valid(self._created_at, "record creation date must be set") require_valid(self.gid, "record global identity must be set") + require_text(self.gid, "record global identity must be an uncoerced string") require_valid(self.id, "record identity must be set") require_integer_identity( self.id, "record identity must be an uncoerced integer" diff --git a/src/moneywiz_api/model/tag.py b/src/moneywiz_api/model/tag.py index c152677..b1e2cd0 100644 --- a/src/moneywiz_api/model/tag.py +++ b/src/moneywiz_api/model/tag.py @@ -2,7 +2,11 @@ from moneywiz_api.model.record import Record from moneywiz_api.types import ID -from moneywiz_api.validation import require_integer_identity, require_valid +from moneywiz_api.validation import ( + require_integer_identity, + require_text, + require_valid, +) @dataclass @@ -23,5 +27,6 @@ def __init__(self, row): # Validate require_valid(self.name is not None, "tag name is required") + require_text(self.name, "tag name must be an uncoerced string") require_valid(self.user is not None, "tag owner is required") require_integer_identity(self.user, "tag owner must be an uncoerced integer") diff --git a/src/moneywiz_api/read_result.py b/src/moneywiz_api/read_result.py index c23016a..a85513c 100644 --- a/src/moneywiz_api/read_result.py +++ b/src/moneywiz_api/read_result.py @@ -3,10 +3,11 @@ from __future__ import annotations from copy import deepcopy -from dataclasses import asdict, dataclass, field +from dataclasses import asdict, dataclass, field, is_dataclass from datetime import date, datetime from decimal import Decimal from enum import Enum +import math from pathlib import Path from typing import Any, Mapping @@ -257,8 +258,11 @@ class ReadSnapshot: schema_profile: Mapping[str, Any] def __post_init__(self) -> None: - object.__setattr__(self, "records", _FrozenMapping(self.records)) - object.__setattr__(self, "schema_profile", _FrozenMapping(self.schema_profile)) + records = json_safe(self.records) + schema_profile = json_safe(self.schema_profile) + json_safe(self.completeness) + object.__setattr__(self, "records", _FrozenMapping(records)) + object.__setattr__(self, "schema_profile", _FrozenMapping(schema_profile)) def as_dict(self) -> dict[str, Any]: return { @@ -270,6 +274,12 @@ def as_dict(self) -> dict[str, Any]: def json_safe(value: Any) -> Any: """Convert public model values into deterministic JSON-compatible values.""" + if value is None or type(value) in (bool, int, str): + return value + if type(value) is float: + if not math.isfinite(value): + raise TypeError("snapshot float values must be finite") + return value if isinstance(value, Decimal): return str(value) if isinstance(value, (datetime, date)): @@ -277,11 +287,27 @@ def json_safe(value: Any) -> Any: if isinstance(value, Path): return str(value) if isinstance(value, Enum): - return value.value - if hasattr(value, "__dataclass_fields__"): + return json_safe(value.value) + if is_dataclass(value) and not isinstance(value, type): return json_safe(asdict(value)) if isinstance(value, Mapping): - return {str(key): json_safe(item) for key, item in value.items()} + converted = {} + for key, item in value.items(): + normalized_key = _json_safe_key(key) + if normalized_key in converted: + raise TypeError("snapshot mapping keys collide after normalization") + converted[normalized_key] = json_safe(item) + return converted if isinstance(value, (tuple, list)): return [json_safe(item) for item in value] - return value + raise TypeError("snapshot contains an unsupported value") + + +def _json_safe_key(value: Any) -> str: + if value is None or type(value) in (bool, int, str): + return str(value) + if type(value) is float: + if not math.isfinite(value): + raise TypeError("snapshot mapping keys must be finite") + return str(value) + raise TypeError("snapshot contains an unsupported mapping key") diff --git a/src/moneywiz_api/validation.py b/src/moneywiz_api/validation.py index ffcd695..3705ae1 100644 --- a/src/moneywiz_api/validation.py +++ b/src/moneywiz_api/validation.py @@ -12,3 +12,11 @@ def require_integer_identity( return if type(value) is not int: raise AssertionError(message) + + +def require_text(value: object, message: str, *, optional: bool = False) -> None: + """Require an uncoerced string, optionally allowing None.""" + if optional and value is None: + return + if type(value) is not str: + raise AssertionError(message) diff --git a/tests/unit/optimized_validation_probe.py b/tests/unit/optimized_validation_probe.py index debf68f..dfabc90 100644 --- a/tests/unit/optimized_validation_probe.py +++ b/tests/unit/optimized_validation_probe.py @@ -311,6 +311,10 @@ def named_entities_probe(): Payee, {**common_row(Z_ENT=28), "ZNAME5": "Payee", "ZUSER7": None}, ), + "payee_blob_name": ( + Payee, + {**common_row(Z_ENT=28), "ZNAME5": b"PRIVATE_PAYLOAD", "ZUSER7": 1}, + ), "tag_valid": ( Tag, {**common_row(Z_ENT=35), "ZNAME6": "Tag", "ZUSER8": 1}, @@ -319,6 +323,10 @@ def named_entities_probe(): Tag, {**common_row(Z_ENT=35), "ZNAME6": "Tag", "ZUSER8": None}, ), + "tag_blob_name": ( + Tag, + {**common_row(Z_ENT=35), "ZNAME6": b"PRIVATE_PAYLOAD", "ZUSER8": 1}, + ), "category_valid": ( Category, { @@ -339,6 +347,16 @@ def named_entities_probe(): "ZUSER3": None, }, ), + "category_blob_name": ( + Category, + { + **common_row(Z_ENT=19), + "ZNAME2": b"PRIVATE_PAYLOAD", + "ZPARENTCATEGORY": None, + "ZTYPE2": 1, + "ZUSER3": 1, + }, + ), } return { name: capture(lambda constructor=constructor, row=row: constructor(row).name) @@ -488,6 +506,9 @@ def main(): ), "record_valid": capture(lambda: Record(common_row()).gid), "record_invalid_identity": capture(lambda: Record(common_row(ZGID=""))), + "record_coerced_gid": capture( + lambda: Record(common_row(ZGID=b"PRIVATE_PAYLOAD")) + ), "record_coerced_identity": capture(lambda: Record(common_row(Z_PK=True))), "account_nullable_info": capture( lambda: { @@ -500,6 +521,12 @@ def main(): "account-1", CreditCardAccount ) ), + "account_blob_name": capture( + lambda: CashAccount(account_row(ZNAME=b"PRIVATE_PAYLOAD")) + ), + "account_blob_info": capture( + lambda: CashAccount(account_row(ZINFO=b"PRIVATE_PAYLOAD")) + ), "direct_accounts": direct_account_probe(), "category_types": category_type_probe(), "named_entities": named_entities_probe(), diff --git a/tests/unit/test_accessor_admission.py b/tests/unit/test_accessor_admission.py new file mode 100644 index 0000000..706f3ae --- /dev/null +++ b/tests/unit/test_accessor_admission.py @@ -0,0 +1,251 @@ +from contextlib import contextmanager +import sqlite3 + +import pytest + +from moneywiz_api import MoneywizApi +from moneywiz_api.database_accessor import DatabaseAccessor, DatabaseSchemaError +from moneywiz_api.managers.account_manager import AccountManager +from tests.unit.test_schema_cache_binding import ( + create_account_store, + create_relationship_store, + open_writer, +) + + +def count_admissions(monkeypatch): + counts = {"schema": 0, "source": 0} + original_schema = DatabaseAccessor._verify_schema_identity + original_source = DatabaseAccessor._verify_source_eligibility + + def verify_schema(accessor): + counts["schema"] += 1 + return original_schema(accessor) + + def verify_source(accessor): + counts["source"] += 1 + return original_source(accessor) + + monkeypatch.setattr(DatabaseAccessor, "_verify_schema_identity", verify_schema) + monkeypatch.setattr(DatabaseAccessor, "_verify_source_eligibility", verify_source) + return counts + + +def assert_admission_reset(accessor) -> None: + assert accessor._admission_depth == 0 + assert accessor._admission_broken is False + assert not accessor._con.in_transaction + + +def test_eager_api_re_load_uses_one_admission(tmp_path, monkeypatch) -> None: + path = tmp_path / "eager.sqlite" + create_relationship_store(path) + counts = count_admissions(monkeypatch) + + with MoneywizApi(path) as api: + assert api.completeness().complete + assert counts == {"schema": 1, "source": 1} + + +def test_direct_manager_and_independent_getters_admit_per_outer_scope( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "direct.sqlite" + create_account_store(path) + counts = count_admissions(monkeypatch) + + with DatabaseAccessor(path) as accessor: + assert AccountManager().load(accessor).complete + assert counts == {"schema": 1, "source": 1} + assert accessor.get_record(1).id == 1 + assert accessor.get_record_by_gid("account-1").id == 1 + assert counts == {"schema": 3, "source": 3} + + +def test_explicit_nested_scope_reuses_one_admission(tmp_path, monkeypatch) -> None: + path = tmp_path / "nested.sqlite" + create_account_store(path) + counts = count_admissions(monkeypatch) + + with DatabaseAccessor(path) as accessor: + with accessor.read_transaction(): + assert accessor.query_objects(["CashAccount"]) + assert accessor.get_record(1).id == 1 + assert accessor.get_record_by_gid("account-1").id == 1 + assert counts == {"schema": 1, "source": 1} + assert_admission_reset(accessor) + + +@pytest.mark.parametrize("failure", ["schema", "source"]) +def test_failed_admission_resets_and_allows_retry( + tmp_path, monkeypatch, failure +) -> None: + path = tmp_path / f"failed-{failure}.sqlite" + create_account_store(path) + accessor = DatabaseAccessor(path) + method_name = ( + f"_verify_{failure}_eligibility" + if failure == "source" + else ("_verify_schema_identity") + ) + original = getattr(accessor, method_name) + + def refuse(): + raise DatabaseSchemaError("synthetic admission refusal") + + monkeypatch.setattr(accessor, method_name, refuse) + with pytest.raises(DatabaseSchemaError, match="synthetic admission refusal"): + accessor.query_objects(["CashAccount"]) + assert_admission_reset(accessor) + + monkeypatch.setattr(accessor, method_name, original) + assert len(accessor.query_objects(["CashAccount"])) == 1 + accessor.close() + + +def test_nested_body_failure_does_not_poison_active_or_later_scope( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "body-error.sqlite" + create_account_store(path) + counts = count_admissions(monkeypatch) + + with DatabaseAccessor(path) as accessor: + with accessor.read_transaction(): + with pytest.raises(RuntimeError, match="synthetic body failure"): + with accessor.read_transaction(): + raise RuntimeError("synthetic body failure") + assert accessor.get_record(1).id == 1 + assert counts == {"schema": 1, "source": 1} + assert accessor.query_objects(["CashAccount"]) + assert counts == {"schema": 2, "source": 2} + + +def test_outer_exit_failure_resets_admission_and_allows_retry( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "exit-error.sqlite" + create_account_store(path) + accessor = DatabaseAccessor(path) + original = accessor._raw_read_transaction + + @contextmanager + def fail_after_exit(): + with original(): + yield + raise DatabaseSchemaError("synthetic outer exit failure") + + monkeypatch.setattr(accessor, "_raw_read_transaction", fail_after_exit) + with pytest.raises(DatabaseSchemaError, match="synthetic outer exit failure"): + accessor.query_objects(["CashAccount"]) + assert_admission_reset(accessor) + + monkeypatch.setattr(accessor, "_raw_read_transaction", original) + assert accessor.query_objects(["CashAccount"]) + accessor.close() + + +def test_public_close_inside_scope_refuses_interrupted_snapshot_and_cleans_depth( + tmp_path, +) -> None: + path = tmp_path / "closed.sqlite" + create_account_store(path) + accessor = DatabaseAccessor(path) + + with pytest.raises(DatabaseSchemaError, match="read transaction was interrupted"): + with accessor.read_transaction(): + accessor.close() + + assert accessor._admission_depth == 0 + assert accessor._admission_broken is False + + +def test_simulated_transaction_loss_latches_until_outer_scope_unwinds(tmp_path) -> None: + path = tmp_path / "simulated-loss.sqlite" + create_account_store(path) + + with DatabaseAccessor(path) as accessor: + with pytest.raises( + DatabaseSchemaError, match="read transaction was interrupted" + ): + with accessor.read_transaction(): + # Simulate a SQLite statement aborting the transaction. This does not + # assert that an interrupted SELECT itself performs a rollback. + accessor._con.rollback() + with accessor.read_transaction(): + pytest.fail("lost outer snapshot must not admit a nested read") + assert_admission_reset(accessor) + assert accessor.query_objects(["CashAccount"]) + + +def test_caller_owned_transaction_is_verified_per_public_outer_scope( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "caller-owned.sqlite" + create_account_store(path) + counts = count_admissions(monkeypatch) + + with DatabaseAccessor(path) as accessor: + accessor._con.execute("BEGIN") + with accessor.read_transaction(): + assert accessor.get_record(1).id == 1 + assert counts == {"schema": 1, "source": 1} + assert accessor._con.in_transaction + assert accessor.query_objects(["CashAccount"]) + assert counts == {"schema": 2, "source": 2} + assert accessor._con.in_transaction + accessor._con.rollback() + + +def test_wal_orphan_remains_outside_pinned_generation_then_next_scope_refuses( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "wal-orphan.sqlite" + create_account_store(path) + counts = count_admissions(monkeypatch) + + with DatabaseAccessor(path) as accessor: + with accessor.read_transaction(): + assert len(accessor.query_objects(["CashAccount"])) == 1 + writer = open_writer(path) + writer.execute( + "INSERT INTO ZSYNCOBJECT VALUES " + "(2, 999, 0, 'orphan', 2, 1, 'Orphan', 'EUR', 0, NULL, 1)" + ) + writer.commit() + writer.close() + assert len(accessor.query_objects(["CashAccount"])) == 1 + assert counts == {"schema": 1, "source": 1} + + with pytest.raises(DatabaseSchemaError, match="unclassifiable entity ancestry"): + accessor.query_objects(["CashAccount"]) + assert counts == {"schema": 2, "source": 2} + + writer = open_writer(path) + writer.execute("DELETE FROM ZSYNCOBJECT WHERE Z_PK = 2") + writer.commit() + writer.close() + assert len(accessor.query_objects(["CashAccount"])) == 1 + assert counts == {"schema": 3, "source": 3} + + +def test_nested_query_error_with_intact_transaction_reuses_admission( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "query-error.sqlite" + create_account_store(path) + counts = count_admissions(monkeypatch) + + with DatabaseAccessor(path) as accessor: + original = accessor._query_objects + + def fail_query(_typenames): + raise sqlite3.OperationalError("synthetic query error") + + with accessor.read_transaction(): + monkeypatch.setattr(accessor, "_query_objects", fail_query) + with pytest.raises(sqlite3.OperationalError, match="synthetic query error"): + accessor.query_objects(["CashAccount"]) + monkeypatch.setattr(accessor, "_query_objects", original) + assert accessor.query_objects(["CashAccount"]) + assert counts == {"schema": 1, "source": 1} diff --git a/tests/unit/test_identity_domains.py b/tests/unit/test_identity_domains.py index 8519163..c294118 100644 --- a/tests/unit/test_identity_domains.py +++ b/tests/unit/test_identity_domains.py @@ -5,6 +5,11 @@ import pytest from moneywiz_api.managers.transaction_manager import TransactionManager +from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.managers.category_manager import CategoryManager +from moneywiz_api.managers.investment_holding_manager import InvestmentHoldingManager +from moneywiz_api.managers.payee_manager import PayeeManager +from moneywiz_api.managers.tag_manager import TagManager from moneywiz_api.model.account import CashAccount from moneywiz_api.model.category import Category from moneywiz_api.model.investment_holding import InvestmentHolding @@ -24,10 +29,18 @@ WithdrawTransaction, ) from moneywiz_api.read_result import RelationshipLoadReport, RelationshipStorage +from moneywiz_api.schema_profile import SchemaProfile from tests.unit.accessor_test_support import initialized_memory_accessor INVALID_IDENTITIES = (True, 1.0, "PRIVATE_PAYLOAD", Decimal("1")) +PROFILE = SchemaProfile( + "unsuffixed-investment-columns", + "ZNUMBEROFSHARES", + "ZNUMBEROFSHARES", + "ZPRICEPERSHARE", + "ZPRICEPERSHARE", +) def record_row(**overrides): @@ -265,6 +278,43 @@ def test_model_identity_fields_preserve_valid_integers( constructor(row_factory(**{field: 7})) +@pytest.mark.parametrize("gid", ["record-1", "Δοκιμή", " ", "id:/?!"]) +def test_record_gid_preserves_nonempty_exact_strings(gid) -> None: + assert Record(record_row(ZGID=gid)).gid == gid + + +@pytest.mark.parametrize("gid", [None, "", b"PRIVATE_PAYLOAD", 1, True, object()]) +def test_record_gid_refuses_empty_or_coerced_values_without_payload(gid) -> None: + with pytest.raises(AssertionError) as error: + Record(record_row(ZGID=gid)) + + assert "PRIVATE_PAYLOAD" not in str(error.value) + + +@pytest.mark.parametrize( + ("constructor", "row_factory", "field"), + [ + (CashAccount, account_row, "ZNAME"), + (Payee, payee_row, "ZNAME5"), + (Category, category_row, "ZNAME2"), + (Tag, tag_row, "ZNAME6"), + ], +) +def test_native_named_entities_preserve_empty_strings_and_reject_blobs( + constructor, row_factory, field +) -> None: + assert constructor(row_factory(**{field: ""})).name == "" + with pytest.raises(AssertionError, match="uncoerced string"): + constructor(row_factory(**{field: b"PRIVATE_PAYLOAD"})) + + +def test_account_info_preserves_nullable_and_empty_text_contract() -> None: + assert CashAccount(account_row(ZINFO=None)).info is None + assert CashAccount(account_row(ZINFO="")).info == "" + with pytest.raises(AssertionError, match="uncoerced string"): + CashAccount(account_row(ZINFO=b"PRIVATE_PAYLOAD")) + + @pytest.mark.parametrize( ("constructor", "row_factory", "field"), [ @@ -366,6 +416,17 @@ def test_accessor_refuses_invalid_reconciled_domain(raw_value) -> None: accessor.get_record(1, DepositTransaction) +def test_accessor_enforces_exact_string_gid_domain() -> None: + accessor = initialized_memory_accessor( + [record_row(ZGID=b"PRIVATE_PAYLOAD")], + [(37, "Record", 0)], + ) + + with pytest.raises(AssertionError, match="uncoerced string"): + accessor.get_record(1) + accessor.close() + + class TransactionAccessor: def __init__(self, rows): self.rows = rows @@ -393,6 +454,72 @@ def read_tags_map(self): return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) +class ManagerAccessor(TransactionAccessor): + def __init__(self, rows, typenames): + super().__init__(rows) + self.typenames = typenames + self.schema_profile = PROFILE + + def descendant_typenames(self, _roots): + return list(self.typenames.values()) + + def typename_for(self, ent_id): + return self.typenames.get(ent_id) + + +@pytest.mark.parametrize( + ("manager_type", "row_factory", "entity", "entity_id"), + [ + (AccountManager, account_row, "CashAccount", 10), + (PayeeManager, payee_row, "Payee", 28), + (CategoryManager, category_row, "Category", 19), + (InvestmentHoldingManager, holding_row, "InvestmentHolding", 24), + (TransactionManager, transaction_row, "DepositTransaction", 37), + (TagManager, tag_row, "Tag", 35), + ], +) +def test_all_managers_skip_wrong_type_gid_and_keep_valid_row( + manager_type, row_factory, entity, entity_id +) -> None: + invalid = row_factory(ZGID=b"PRIVATE_PAYLOAD") + valid = row_factory(Z_PK=2, ZGID="valid-2") + + report = manager_type().load(ManagerAccessor([invalid, valid], {entity_id: entity})) + + assert report.source_ids == (1, 2) + assert report.parsed_ids == (2,) + assert report.skipped[0].error.value == "validation" + assert report.status == "partial" + assert "PRIVATE_PAYLOAD" not in json.dumps(report.as_dict(), sort_keys=True) + + +@pytest.mark.parametrize( + ("manager_type", "row_factory", "field", "entity", "entity_id"), + [ + (AccountManager, account_row, "ZNAME", "CashAccount", 10), + (PayeeManager, payee_row, "ZNAME5", "Payee", 28), + (CategoryManager, category_row, "ZNAME2", "Category", 19), + (TagManager, tag_row, "ZNAME6", "Tag", 35), + ], +) +def test_named_entity_managers_report_blob_names_as_validation( + manager_type, row_factory, field, entity, entity_id +) -> None: + report = manager_type().load( + ManagerAccessor( + [ + row_factory(**{field: b"PRIVATE_PAYLOAD"}), + row_factory(Z_PK=2, ZGID="valid-2", **{field: ""}), + ], + {entity_id: entity}, + ) + ) + + assert report.source_count == 2 + assert report.parsed_ids == (2,) + assert report.skipped[0].error.value == "validation" + + @pytest.mark.parametrize( "invalid_row", [ diff --git a/tests/unit/test_optimized_validation.py b/tests/unit/test_optimized_validation.py index 7cc74f0..dc73be6 100644 --- a/tests/unit/test_optimized_validation.py +++ b/tests/unit/test_optimized_validation.py @@ -63,9 +63,12 @@ def test_probe_covers_success_error_and_completeness_paths( } assert cases["date_string"]["type"] == "AssertionError" assert cases["record_invalid_identity"]["type"] == "AssertionError" + assert cases["record_coerced_gid"]["type"] == "AssertionError" assert cases["record_coerced_identity"]["type"] == "AssertionError" assert cases["account_nullable_info"]["status"] == "ok" assert cases["account_invalid_name_public"]["type"] == "AssertionError" + assert cases["account_blob_name"]["type"] == "AssertionError" + assert cases["account_blob_info"]["type"] == "AssertionError" direct_accounts = cases["direct_accounts"] assert all( outcomes["valid"] == {"status": "ok", "value": None} @@ -98,7 +101,14 @@ def test_probe_covers_success_error_and_completeness_paths( ) assert all( named_entities[name]["type"] == "AssertionError" - for name in ("payee_invalid", "tag_invalid", "category_invalid") + for name in ( + "payee_invalid", + "tag_invalid", + "category_invalid", + "payee_blob_name", + "tag_blob_name", + "category_blob_name", + ) ) assert cases["manager"]["source_ids"] == [10, 11, None] assert cases["manager"]["parsed_count"] == 1 diff --git a/tests/unit/test_read_result_ownership.py b/tests/unit/test_read_result_ownership.py index dfb4df5..7df3d3c 100644 --- a/tests/unit/test_read_result_ownership.py +++ b/tests/unit/test_read_result_ownership.py @@ -1,11 +1,16 @@ from copy import deepcopy -from dataclasses import asdict, fields, replace -from datetime import date +from dataclasses import asdict, dataclass, fields, replace +from datetime import date, datetime from decimal import Decimal +from enum import Enum +import json +from pathlib import Path +from types import SimpleNamespace import pytest from moneywiz_api.managers.tag_manager import TagManager +from moneywiz_api.moneywiz_api import MoneywizApi from moneywiz_api.model.record import Record from moneywiz_api.read_result import ( ApiCompleteness, @@ -17,6 +22,7 @@ SkippedRecord, json_safe, ) +from moneywiz_api.schema_profile import SchemaProfile def incomplete_relationship() -> RelationshipLoadReport: @@ -216,3 +222,149 @@ def test_live_manager_and_model_remain_mutable_after_snapshot_publication() -> N assert manager.get(2) is record assert record.gid == "changed-record" assert snapshot.records["tags"][0]["gid"] == "record-1" + + +class TextEnum(Enum): + VALUE = "value" + + +class BinaryEnum(Enum): + VALUE = b"PRIVATE_PAYLOAD" + + +@dataclass +class NestedValue: + value: object + + +class PrivatePayload: + def __repr__(self): + raise RuntimeError("PRIVATE_REPR") + + def __str__(self): + raise RuntimeError("PRIVATE_STR") + + +def test_json_safe_preserves_supported_scalar_and_converter_contract() -> None: + value = { + "none": None, + "bool": True, + "int": 3, + "float": -0.25, + "decimal": Decimal("NaN"), + "datetime": datetime(2026, 9, 13, 10, 30), + "date": date(2026, 9, 13), + "path": Path("relative/path"), + "enum": TextEnum.VALUE, + "dataclass": NestedValue(["nested"]), + } + + converted = json_safe(value) + + assert converted == { + "none": None, + "bool": True, + "int": 3, + "float": -0.25, + "decimal": "NaN", + "datetime": "2026-09-13T10:30:00", + "date": "2026-09-13", + "path": "relative/path", + "enum": "value", + "dataclass": {"value": ["nested"]}, + } + json.dumps(converted, allow_nan=False) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) +def test_json_safe_rejects_nonfinite_builtin_floats(value) -> None: + with pytest.raises(TypeError, match="float values must be finite"): + json_safe(value) + + +@pytest.mark.parametrize( + "value", + [b"PRIVATE_PAYLOAD", PrivatePayload(), BinaryEnum.VALUE, NestedValue(b"x")], +) +def test_json_safe_rejects_nested_unsupported_values_without_rendering(value) -> None: + with pytest.raises(TypeError) as error: + json_safe({"nested": [value]}) + + assert "PRIVATE" not in str(error.value) + + +def test_json_safe_key_domain_and_collision_are_deterministic() -> None: + assert json_safe({None: 0, True: 1, 2: 2, 1.5: 3, "name": 4}) == { + "None": 0, + "True": 1, + "2": 2, + "1.5": 3, + "name": 4, + } + with pytest.raises(TypeError, match="keys collide"): + json_safe({1: "integer", "1": "string"}) + + +@pytest.mark.parametrize( + "key", + [ + Decimal("1"), + TextEnum.VALUE, + Path("key"), + b"PRIVATE_PAYLOAD", + ("tuple",), + PrivatePayload(), + float("nan"), + ], +) +def test_json_safe_rejects_unsupported_keys_without_rendering(key) -> None: + with pytest.raises(TypeError) as error: + json_safe({key: "value"}) + + assert "PRIVATE" not in str(error.value) + + +def test_snapshot_refuses_unsupported_nested_values_before_publication() -> None: + with pytest.raises(TypeError, match="unsupported value"): + ReadSnapshot( + records={"accounts": ({"nested": [b"PRIVATE_PAYLOAD"]},)}, + completeness=ApiCompleteness({"accounts": ManagerLoadReport()}), + schema_profile={}, + ) + + +def test_api_snapshot_refuses_live_model_mutation_but_prior_snapshot_is_detached() -> ( + None +): + record = Record( + { + "Z_PK": 1, + "Z_ENT": 35, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": "record-1", + } + ) + manager = TagManager() + manager.records()[record.id] = record + manager._load_report = ManagerLoadReport(source_ids=(1,), parsed_ids=(1,)) + api = MoneywizApi.__new__(MoneywizApi) + api._managers = {"tags": manager} + api._loaded_managers = {"tags"} + api.accessor = SimpleNamespace( + schema_profile=SchemaProfile( + "unsuffixed-investment-columns", + "ZNUMBEROFSHARES", + "ZNUMBEROFSHARES", + "ZPRICEPERSHARE", + "ZPRICEPERSHARE", + ) + ) + + published = api.snapshot(("tags",)) + record.gid = b"PRIVATE_PAYLOAD" + + assert published.records["tags"][0]["gid"] == "record-1" + json.dumps(published.as_dict(), allow_nan=False) + assert record.as_dict()["gid"] == b"PRIVATE_PAYLOAD" + with pytest.raises(TypeError, match="unsupported value"): + api.snapshot(("tags",)) From 3020435cef7b9ae6b788ac6e779f54ac146e7489 Mon Sep 17 00:00:00 2001 From: MarcoMC Date: Sun, 13 Sep 2026 14:24:44 +0200 Subject: [PATCH 09/10] Harden P0 read validation and interrupted loads Reject invalid persisted scalar shapes before coercion and discard incomplete read state when initialization is interrupted. Add regression coverage for optimized validation, schema discovery, diagnostics, and cancellation. --- src/moneywiz_api/database_accessor.py | 4 +- src/moneywiz_api/managers/record_manager.py | 2 +- src/moneywiz_api/model/account.py | 4 ++ src/moneywiz_api/model/investment_holding.py | 11 +++- src/moneywiz_api/moneywiz_api.py | 2 +- tests/unit/optimized_validation_probe.py | 22 ++++++++ tests/unit/test_account_diagnostics.py | 24 ++++++++ tests/unit/test_api_atomic_reload.py | 27 +++++++++ tests/unit/test_database_accessor.py | 31 ++++++++++ tests/unit/test_optimized_validation.py | 15 +++++ tests/unit/test_profiled_investment_values.py | 56 +++++++++++++++++++ tests/unit/test_read_completeness.py | 33 +++++++++++ 12 files changed, 227 insertions(+), 4 deletions(-) diff --git a/src/moneywiz_api/database_accessor.py b/src/moneywiz_api/database_accessor.py index ddbd208..8e30f8d 100644 --- a/src/moneywiz_api/database_accessor.py +++ b/src/moneywiz_api/database_accessor.py @@ -71,8 +71,10 @@ def dict_factory(cursor, row): try: self._con.row_factory = dict_factory self._initialize_schema_cache() - except Exception as exc: + except BaseException as exc: self._con.close() + if not isinstance(exc, Exception): + raise if isinstance(exc, DatabaseSchemaError): raise raise DatabaseSchemaError("database schema could not be read") from exc diff --git a/src/moneywiz_api/managers/record_manager.py b/src/moneywiz_api/managers/record_manager.py index c5a55b6..e378751 100644 --- a/src/moneywiz_api/managers/record_manager.py +++ b/src/moneywiz_api/managers/record_manager.py @@ -45,7 +45,7 @@ def load(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: try: with db_accessor.read_transaction(): report = self._load_in_transaction(db_accessor) - except Exception: + except BaseException: self._discard_incomplete_load() raise self._load_report = report diff --git a/src/moneywiz_api/model/account.py b/src/moneywiz_api/model/account.py index a9b4848..accd1c3 100644 --- a/src/moneywiz_api/model/account.py +++ b/src/moneywiz_api/model/account.py @@ -45,6 +45,10 @@ def validate(self) -> None: require_valid( self.display_order is not None, "account display order is required" ) + require_integer_identity( + self.display_order, + "account display order must be an uncoerced integer", + ) require_valid(self.group_id is not None, "account group identity is required") require_integer_identity( self.group_id, "account group identity must be an uncoerced integer" diff --git a/src/moneywiz_api/model/investment_holding.py b/src/moneywiz_api/model/investment_holding.py index 651e040..39c4792 100644 --- a/src/moneywiz_api/model/investment_holding.py +++ b/src/moneywiz_api/model/investment_holding.py @@ -66,8 +66,17 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): self.symbol = row["ZSYMBOL"] self.holding_type = row["ZHOLDINGTYPE"] self.description = row["ZDESC"] + raw_price_per_share_available_online = row["ZISPRICEPERSHAREAVAILABLEONLINE"] + require_integer_identity( + raw_price_per_share_available_online, + "holding online-price state must be an uncoerced integer", + ) + require_valid( + raw_price_per_share_available_online in (0, 1), + "holding online-price state must be zero or one", + ) self.price_per_share_available_online = ( - row["ZISPRICEPERSHAREAVAILABLEONLINE"] == 1 + raw_price_per_share_available_online == 1 ) self._investment_object_type = row["ZINVESTMENTOBJECTTYPE"] diff --git a/src/moneywiz_api/moneywiz_api.py b/src/moneywiz_api/moneywiz_api.py index 17e8095..41fb65d 100644 --- a/src/moneywiz_api/moneywiz_api.py +++ b/src/moneywiz_api/moneywiz_api.py @@ -52,7 +52,7 @@ def __init__( try: self.load(managers) - except Exception: + except BaseException: self.close() raise diff --git a/tests/unit/optimized_validation_probe.py b/tests/unit/optimized_validation_probe.py index dfabc90..8ab4774 100644 --- a/tests/unit/optimized_validation_probe.py +++ b/tests/unit/optimized_validation_probe.py @@ -527,6 +527,9 @@ def main(): "account_blob_info": capture( lambda: CashAccount(account_row(ZINFO=b"PRIVATE_PAYLOAD")) ), + "account_coerced_display_order": capture( + lambda: CashAccount(account_row(ZDISPLAYORDER="PRIVATE_PAYLOAD")) + ), "direct_accounts": direct_account_probe(), "category_types": category_type_probe(), "named_entities": named_entities_probe(), @@ -537,6 +540,25 @@ def main(): "holding_missing_quantity": capture( lambda: InvestmentHolding(holding_row(ZNUMBEROFSHARES=None), PROFILE) ), + "holding_online_price_state": { + label: capture( + lambda value=value: ( + InvestmentHolding( + holding_row(ZISPRICEPERSHAREAVAILABLEONLINE=value), PROFILE + ).price_per_share_available_online + ) + ) + for label, value in ( + ("none", None), + ("false_boolean", False), + ("true_boolean", True), + ("zero", 0), + ("one", 1), + ("two", 2), + ("float", 1.0), + ("text", "PRIVATE_PAYLOAD"), + ) + }, "deposit_valid": capture( lambda: { "amount": valid_deposit.amount, diff --git a/tests/unit/test_account_diagnostics.py b/tests/unit/test_account_diagnostics.py index 52e7dc1..12be241 100644 --- a/tests/unit/test_account_diagnostics.py +++ b/tests/unit/test_account_diagnostics.py @@ -96,6 +96,30 @@ def test_required_account_identity_remains_required() -> None: assert report.skipped[0].exception_type == "AssertionError" +@pytest.mark.parametrize( + "raw_value", + [None, False, True, 1.0, "PRIVATE_PAYLOAD"], +) +def test_direct_accounts_reject_non_integer_display_order(raw_value) -> None: + with pytest.raises(AssertionError) as error: + CashAccount(account_row(ZDISPLAYORDER=raw_value)) + + assert "PRIVATE_PAYLOAD" not in str(error.value) + + +def test_account_manager_skips_invalid_display_order_before_sorting() -> None: + invalid = account_row(ZDISPLAYORDER="PRIVATE_PAYLOAD") + valid = account_row(Z_PK=2, ZGID="account-2", ZDISPLAYORDER=2) + manager = AccountManager() + + report = manager.load(AccountAccessor([invalid, valid])) + + assert not report.complete + assert report.parsed_ids == (2,) + assert report.skipped[0].error.value == "validation" + assert list(manager.records()) == [2] + + @pytest.mark.parametrize("constructor", ACCOUNT_CONSTRUCTORS) @pytest.mark.parametrize("info_present", [True, False]) def test_direct_account_constructors_accept_nullable_or_absent_info( diff --git a/tests/unit/test_api_atomic_reload.py b/tests/unit/test_api_atomic_reload.py index 12aad5b..4cdb1e6 100644 --- a/tests/unit/test_api_atomic_reload.py +++ b/tests/unit/test_api_atomic_reload.py @@ -140,6 +140,33 @@ def api_for(monkeypatch, accessor, managers): return api_module.MoneywizApi("unused.sqlite", managers=managers) +@pytest.mark.parametrize("interruption_type", [KeyboardInterrupt, SystemExit]) +def test_constructor_interruption_closes_accessor_and_reraises( + monkeypatch, interruption_type +) -> None: + import moneywiz_api.moneywiz_api as api_module + + accessor = SyntheticAccessor() + accessor.closed = False + interruption = interruption_type("synthetic API construction interruption") + + def interrupt_load(_self, _managers=None): + raise interruption + + def close(): + accessor.closed = True + + accessor.close = close + monkeypatch.setattr(api_module, "DatabaseAccessor", lambda _path: accessor) + monkeypatch.setattr(api_module.MoneywizApi, "load", interrupt_load) + + with pytest.raises(interruption_type) as error: + api_module.MoneywizApi("unused.sqlite") + + assert error.value is interruption + assert accessor.closed + + @pytest.mark.parametrize( "failure", ["transaction_query", "category", "refund", "tags"], diff --git a/tests/unit/test_database_accessor.py b/tests/unit/test_database_accessor.py index ac4b53c..f71328e 100644 --- a/tests/unit/test_database_accessor.py +++ b/tests/unit/test_database_accessor.py @@ -98,3 +98,34 @@ def tracked_connect(*args, **kwargs): assert len(opened) == 1 assert opened[0].was_closed + + +@pytest.mark.parametrize("interruption_type", [KeyboardInterrupt, SystemExit]) +def test_schema_initialization_interruption_closes_connection_and_reraises( + tmp_path, monkeypatch, interruption_type +) -> None: + path = tmp_path / "interrupted.sqlite" + sqlite3.connect(path).close() + original_connect = sqlite3.connect + opened = [] + interruption = interruption_type("synthetic schema initialization interruption") + + def tracked_connect(*args, **kwargs): + connection = original_connect(*args, factory=TrackingConnection, **kwargs) + opened.append(connection) + return connection + + def interrupt_initialization(_self): + raise interruption + + monkeypatch.setattr(sqlite3, "connect", tracked_connect) + monkeypatch.setattr( + DatabaseAccessor, "_initialize_schema_cache", interrupt_initialization + ) + + with pytest.raises(interruption_type) as error: + DatabaseAccessor(path) + + assert error.value is interruption + assert len(opened) == 1 + assert opened[0].was_closed diff --git a/tests/unit/test_optimized_validation.py b/tests/unit/test_optimized_validation.py index dc73be6..68dc259 100644 --- a/tests/unit/test_optimized_validation.py +++ b/tests/unit/test_optimized_validation.py @@ -69,6 +69,7 @@ def test_probe_covers_success_error_and_completeness_paths( assert cases["account_invalid_name_public"]["type"] == "AssertionError" assert cases["account_blob_name"]["type"] == "AssertionError" assert cases["account_blob_info"]["type"] == "AssertionError" + assert cases["account_coerced_display_order"]["type"] == "AssertionError" direct_accounts = cases["direct_accounts"] assert all( outcomes["valid"] == {"status": "ok", "value": None} @@ -116,6 +117,20 @@ def test_probe_covers_success_error_and_completeness_paths( skipped["error"] == "validation" for skipped in cases["manager"]["skipped"] ) assert cases["holding_missing_quantity"]["type"] == "AssertionError" + holding_online_price_state = cases["holding_online_price_state"] + assert holding_online_price_state["zero"] == {"status": "ok", "value": False} + assert holding_online_price_state["one"] == {"status": "ok", "value": True} + assert all( + holding_online_price_state[value]["type"] == "AssertionError" + for value in ( + "none", + "false_boolean", + "true_boolean", + "two", + "float", + "text", + ) + ) assert cases["deposit_invalid_sign"]["type"] == "AssertionError" assert cases["deposit_coerced_account"]["type"] == "AssertionError" reconciled = cases["reconciled"] diff --git a/tests/unit/test_profiled_investment_values.py b/tests/unit/test_profiled_investment_values.py index 22c9415..46fc711 100644 --- a/tests/unit/test_profiled_investment_values.py +++ b/tests/unit/test_profiled_investment_values.py @@ -95,6 +95,62 @@ def test_holding_uses_selected_profile_alias() -> None: assert holding.price_per_share == Decimal("10.0") +@pytest.mark.parametrize( + ("raw_value", "expected"), + [(0, False), (1, True)], +) +def test_holding_preserves_binary_online_price_domain(raw_value, expected) -> None: + row = investment_holding_row() + row["ZISPRICEPERSHAREAVAILABLEONLINE"] = raw_value + + assert ( + InvestmentHolding(row, UNSUFFIXED_PROFILE).price_per_share_available_online + is expected + ) + + +@pytest.mark.parametrize( + "raw_value", + [None, False, True, -1, 2, 1.0, "PRIVATE_PAYLOAD"], +) +def test_holding_rejects_non_binary_or_coerced_online_price_state( + raw_value, +) -> None: + row = investment_holding_row() + row["ZISPRICEPERSHAREAVAILABLEONLINE"] = raw_value + + with pytest.raises(AssertionError) as error: + InvestmentHolding(row, UNSUFFIXED_PROFILE) + + assert "PRIVATE_PAYLOAD" not in str(error.value) + + +def test_holding_manager_reports_invalid_online_price_state_as_incomplete() -> None: + invalid = investment_holding_row() + invalid["ZISPRICEPERSHAREAVAILABLEONLINE"] = 2 + valid = investment_holding_row() + valid.update( + { + "Z_PK": 25, + "ZGID": "holding-valid", + "ZISPRICEPERSHAREAVAILABLEONLINE": 1, + } + ) + + manager = InvestmentHoldingManager() + report = manager.load(ManagerAccessor("InvestmentHolding", invalid)) + + assert not report.complete + assert report.parsed_ids == () + assert report.skipped[0].error.value == "validation" + assert manager.records() == {} + + report = manager.load(ManagerAccessor("InvestmentHolding", valid)) + + assert report.complete + assert manager.get(25).price_per_share_available_online is True + + @pytest.mark.parametrize( ("constructor", "transaction_row"), [ diff --git a/tests/unit/test_read_completeness.py b/tests/unit/test_read_completeness.py index c0686e6..c402712 100644 --- a/tests/unit/test_read_completeness.py +++ b/tests/unit/test_read_completeness.py @@ -166,6 +166,39 @@ def query_objects(self, _typenames): assert manager.records() == {} +@pytest.mark.parametrize("interruption_type", [KeyboardInterrupt, SystemExit]) +def test_interrupted_reload_discards_partial_state_and_reraises( + interruption_type, +) -> None: + manager = ExampleManager() + manager.load(RecordAccessor([record_row(record_id=9, gid="record-9")])) + interruption = interruption_type("synthetic interruption") + + class InterruptingAccessor(RecordAccessor): + def __init__(self): + super().__init__([record_row(), record_row(record_id=2, gid="record-2")]) + self.lookups = 0 + + def typename_for(self, ent_id): + self.lookups += 1 + if self.lookups == 2: + raise interruption + return super().typename_for(ent_id) + + with pytest.raises(interruption_type) as error: + manager.load(InterruptingAccessor()) + + assert error.value is interruption + assert manager.records() == {} + assert manager._gid_to_id == {} + assert manager.load_report.status == "unloaded" + + report = manager.load(RecordAccessor([record_row(record_id=3, gid="record-3")])) + + assert report.complete + assert list(manager.records()) == [3] + + def test_completed_load_with_only_invalid_rows_reports_error() -> None: manager = ExampleManager() From cf3f68a1ce1a76e0c7398a76f1bb347fdc9bc275 Mon Sep 17 00:00:00 2001 From: MarcoMC Date: Sun, 13 Sep 2026 14:39:23 +0200 Subject: [PATCH 10/10] Detect descendants of migrated concrete entities Seed supported account and transaction entities during descendant discovery so incomplete migrated hierarchies report unknown source rows rather than false completeness. --- src/moneywiz_api/managers/record_manager.py | 5 +- tests/unit/test_entity_source_coverage.py | 57 +++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/moneywiz_api/managers/record_manager.py b/src/moneywiz_api/managers/record_manager.py index e378751..faf48a1 100644 --- a/src/moneywiz_api/managers/record_manager.py +++ b/src/moneywiz_api/managers/record_manager.py @@ -59,7 +59,10 @@ def _load_records(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: """Read rows while keeping the public report unloaded until publication.""" typenames = list(self.ents) if self.entity_roots: - discovered = db_accessor.descendant_typenames(self.entity_roots) + discovery_roots = tuple( + dict.fromkeys((*self.entity_roots, *self.ents.keys())) + ) + discovered = db_accessor.descendant_typenames(discovery_roots) typenames = list(dict.fromkeys([*typenames, *discovered])) records = db_accessor.query_objects(typenames) diff --git a/tests/unit/test_entity_source_coverage.py b/tests/unit/test_entity_source_coverage.py index dac97c0..da92ee9 100644 --- a/tests/unit/test_entity_source_coverage.py +++ b/tests/unit/test_entity_source_coverage.py @@ -128,6 +128,26 @@ def transaction_row() -> dict: ] +PARTIALLY_MIGRATED_HIERARCHIES = [ + pytest.param( + AccountManager, + "Account", + "CashAccount", + 10, + account_row, + id="account", + ), + pytest.param( + TransactionManager, + "Transaction", + "DepositTransaction", + 38, + transaction_row, + id="transaction", + ), +] + + def create_store(path: Path, metadata: list[tuple[int, str, int]], rows: list[dict]): connection = sqlite3.connect(path) connection.execute("PRAGMA journal_mode=WAL") @@ -206,6 +226,43 @@ def test_all_managers_count_unknown_descendants( assert report.skipped[0].error.value == "unknown_entity" +@pytest.mark.parametrize( + ("manager_type", "missing_root", "known_name", "known_id", "row_factory"), + PARTIALLY_MIGRATED_HIERARCHIES, +) +def test_unknown_descendant_is_counted_when_abstract_root_is_absent( + tmp_path, + manager_type, + missing_root, + known_name, + known_id, + row_factory, +) -> None: + path = tmp_path / f"missing-{missing_root}.sqlite" + future_name = f"Future{known_name}" + create_store( + path, + [ + (8, "SyncObject", 0), + (known_id, known_name, 8), + (91, future_name, known_id), + ], + [row_factory(), common_row(2, 91, f"future-{known_name.lower()}")], + ) + + with DatabaseAccessor(path) as accessor: + assert accessor.ent_for(missing_root) is None + report = manager_type().load(accessor) + + assert set(report.source_ids) == {1, 2} + assert report.parsed_ids == (1,) + assert len(report.skipped) == 1 + assert report.skipped[0].record_id == 2 + assert report.skipped[0].entity == future_name + assert report.skipped[0].error.value == "unknown_entity" + assert not report.complete + + def orphan_metadata(kind: str) -> tuple[list[tuple[int, str, int]], int | None]: metadata = [(8, "SyncObject", 0), (9, "Account", 8), (10, "CashAccount", 9)] if kind == "null":