diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5f49171..49e549c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,9 +4,9 @@ permissions: contents: read on: push: - branches: ["main"] + branches: ["main", "release/0.3.0"] pull_request: - branches: ["main"] + branches: ["main", "release/0.3.0"] jobs: build: diff --git a/README.md b/README.md index 7874bcf..f822c16 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,12 @@ Support the original author on A Python API to access MoneyWiz Sqlite database. +## Table of Contents + +- [Get Started](#get-started) +- [Bounded reads](#bounded-reads) +- [Contribution](#contribution) + ## Get Started ```bash @@ -15,7 +21,6 @@ pip install moneywiz-api ``` ```python - from moneywiz_api import MoneywizApi moneywizApi = MoneywizApi("") @@ -38,7 +43,6 @@ moneywizApi = MoneywizApi("") record = accessor.get_record(record_id) print(record) - ``` It also offers a interactive shell `moneywiz-cli`. @@ -58,6 +62,74 @@ Integration tests are opt-in and never read a CLI default database path. Point MONEYWIZ_TEST_DB_PATH=/absolute/path/to/test.sqlite uv run pytest tests ``` +## Bounded reads + +The default remains an eager load of every manager. Use an explicit manager list +to isolate a read from unrelated malformed records: + +```python +from pathlib import Path + +from moneywiz_api import MoneywizApi + +with MoneywizApi( + Path("/absolute/path/to/moneywiz.sqlite"), + managers=("accounts", "transactions"), +) as api: + completeness = api.completeness().as_dict() + snapshot = api.snapshot().as_dict() +``` + +Supported manager names are `accounts`, `payees`, `categories`, `transactions`, +`investment_holdings`, and `tags`. `completeness` contains ordered source and +parsed IDs plus identity-only diagnostics for every skipped row. Transaction +completeness also reports category, refund, and tag relationship storage as +`present`, `absent`, or `unknown`; unknown or malformed storage makes the read +partial. Category relationship counts cover transaction assignments only; rows +owned by budgets, scheduled transactions, or history items are outside that +map. `snapshot` contains JSON-safe records, completeness, and the selected +schema profile. Snapshot construction refuses unsupported leaves such as BLOBs, +arbitrary objects, non-finite built-in floats, unsupported mapping keys, and +keys that collide after normalization. It does not encode or stringify those +values. Legacy model `as_dict()` methods remain raw compatibility interfaces. + +Published completeness and snapshot values own immutable nested mappings and +sequences, so later caller or manager changes cannot rewrite earlier evidence. +Their `as_dict()` and `json_safe()` outputs are detached mutable JSON trees. + +A public manager that has not completed a load exposes a `load_report` with +status `unloaded` and `complete == False`. A successful zero-row load is instead +`complete`, so an empty result is distinguishable from an unobserved manager. + +Database paths must identify existing files. Connections are opened read-only, +and each multi-manager load uses one consistent SQLite read transaction. A +later scoped load atomically refreshes the union of requested and already loaded +managers so one snapshot never mixes database generations. If a reload fails, +the previously published records and completeness evidence remain available. +Direct `manager.load(accessor)` calls use the same boundary: records, +transaction relationships, and the published report come from one read +snapshot, while a failed load resets that manager to `unloaded`. +Nested accessor reads reuse the outer scope's successful schema and source +admission. Each independent public scope revalidates, including scopes inside a +caller-owned SQLite transaction. If SQLite ends an admitted transaction, later +nested reads refuse the lost snapshot until the outer scope unwinds. + +Core Data record global IDs must be nonempty strings. Identity and relationship +endpoint fields must arrive as uncoerced integers; nullable payee and +category-parent references remain supported. Account, payee, category, and tag +names must be strings while preserving existing empty-string behavior; account +info also permits `None`. +Transaction reconciliation accepts only raw integer `0` or `1` before exposing +the value as `bool`. Refused rows remain visible through identity-only +completeness diagnostics without embedding malformed payload values. + +Schema metadata and physical table definitions are bound when the accessor is +opened; if either changes, cache-dependent reads require closing and reopening +the API instead of combining current rows with stale schema information. +Each verified read also rejects physical rows with NULL, unmapped, missing-parent, +or cyclic entity ancestry. Unknown descendants of a requested manager root are +included as skipped source rows instead of disappearing from completeness. + ## Contribution This project is in very early stage, all contributions are welcomed! diff --git a/src/moneywiz_api/__init__.py b/src/moneywiz_api/__init__.py index 45e4455..28b35d9 100644 --- a/src/moneywiz_api/__init__.py +++ b/src/moneywiz_api/__init__.py @@ -1,3 +1,24 @@ +from moneywiz_api.database_accessor import DatabasePathError, DatabaseSchemaError from moneywiz_api.moneywiz_api import MoneywizApi +from moneywiz_api.read_result import ( + ApiCompleteness, + LoadErrorKind, + ManagerLoadReport, + RelationshipLoadReport, + RelationshipStorage, + ReadSnapshot, + SkippedRecord, +) -__all__ = ["MoneywizApi"] +__all__ = [ + "ApiCompleteness", + "DatabasePathError", + "DatabaseSchemaError", + "LoadErrorKind", + "ManagerLoadReport", + "MoneywizApi", + "ReadSnapshot", + "RelationshipLoadReport", + "RelationshipStorage", + "SkippedRecord", +] diff --git a/src/moneywiz_api/database_accessor.py b/src/moneywiz_api/database_accessor.py index 0125ae2..8e30f8d 100644 --- a/src/moneywiz_api/database_accessor.py +++ b/src/moneywiz_api/database_accessor.py @@ -1,8 +1,11 @@ import sqlite3 +import re from collections import defaultdict +from contextlib import contextmanager from pathlib import Path from typing import Dict, List, Any, Callable, Tuple from decimal import Decimal +from enum import Enum from moneywiz_api.model.record import Record from moneywiz_api.model.investment_holding import InvestmentHolding @@ -11,13 +14,53 @@ InvestmentSellTransaction, ) from moneywiz_api.model.raw_data_handler import RawDataHandler as RDH +from moneywiz_api.read_result import ( + LoadErrorKind, + RelationshipLoadReport, + RelationshipStorage, + SkippedRecord, +) from moneywiz_api.schema_profile import SchemaProfile, detect_schema_profile from moneywiz_api.types import ENT_ID, ID, GID +from moneywiz_api.validation import require_integer_identity, require_valid + + +TRANSACTION_TAG_TABLE_RE = re.compile(r"Z_\d+TAGS") +SUPPORTED_UNRELATED_TAG_TABLES = { + "Z_23TAGS": frozenset({"Z_23INFOCARDS5", "Z_35TAGS1"}), + "Z_31TAGS": frozenset({"Z_31SCHEDULEDTRANSACTIONS1", "Z_35TAGS2"}), +} + + +class _TagTableShape(Enum): + DIRECT = "direct" + UNRELATED = "unrelated" + AMBIGUOUS = "ambiguous" + + +class DatabasePathError(ValueError): + """Raised when an explicit database path cannot be opened safely.""" + + +class DatabaseSchemaError(ValueError): + """Raised when a readable SQLite file is not a supported MoneyWiz store.""" class DatabaseAccessor: def __init__(self, db_path: Path): - self._con = sqlite3.connect(db_path, uri=True) + self._db_path = Path(db_path).expanduser().resolve() + if not self._db_path.is_file(): + raise DatabasePathError("database path must identify an existing file") + + try: + self._con = sqlite3.connect( + f"{self._db_path.as_uri()}?mode=ro", + uri=True, + ) + except sqlite3.Error as exc: + raise DatabasePathError( + "database file could not be opened read-only" + ) from exc def dict_factory(cursor, row): record = {} @@ -25,25 +68,109 @@ def dict_factory(cursor, row): record[col[0]] = row[idx] return record - self._con.row_factory = dict_factory - self._schema_profile = detect_schema_profile(self._con) + try: + self._con.row_factory = dict_factory + self._initialize_schema_cache() + except BaseException as exc: + self._con.close() + if not isinstance(exc, Exception): + raise + if isinstance(exc, DatabaseSchemaError): + raise + raise DatabaseSchemaError("database schema could not be read") from exc - self._ent_to_typename: Dict[ENT_ID, str] = self._load_primarykey() - self._typename_to_ent: Dict[str, ENT_ID] = { - v: k for k, v in self._ent_to_typename.items() - } + def _initialize_schema_cache(self) -> None: + """Bind all schema-dependent caches from one SQLite snapshot.""" + self._admission_depth = 0 + self._admission_broken = False + with self._raw_read_transaction(): + if not self._table_exists("Z_PRIMARYKEY") or not self._table_exists( + "ZSYNCOBJECT" + ): + raise DatabaseSchemaError( + "database is missing required MoneyWiz schema tables" + ) + required_metadata_columns = {"Z_ENT", "Z_NAME", "Z_SUPER"} + if not required_metadata_columns.issubset( + self._table_columns("Z_PRIMARYKEY") + ): + raise DatabaseSchemaError( + "Z_PRIMARYKEY is missing required entity metadata columns" + ) + schema_identity = self._read_schema_identity() + schema_profile = detect_schema_profile(self._con) + ent_to_typename, ent_to_super, typename_to_ent = self._entity_maps( + schema_identity[1] + ) + + self._schema_identity = schema_identity + self._schema_profile = schema_profile + self._ent_to_typename = ent_to_typename + self._ent_to_super = ent_to_super + self._typename_to_ent = typename_to_ent - def _load_primarykey(self) -> Dict[int, str]: + def _read_entity_metadata(self) -> tuple[tuple[int, str, int], ...]: cur = self._con.cursor() res = cur.execute( """ - SELECT * FROM "Z_PRIMARYKEY" ORDER BY "Z_ENT" LIMIT 1000 OFFSET 0; + SELECT Z_ENT, Z_NAME, Z_SUPER + FROM "Z_PRIMARYKEY" + ORDER BY Z_ENT """ ) - ent_to_typename: Dict[int, str] = {} + rows: list[tuple[int, str, int]] = [] + ent_ids: set[int] = set() + typenames: set[str] = set() for row in res.fetchall(): - ent_to_typename[row["Z_ENT"]] = row["Z_NAME"] - return ent_to_typename + ent_id = row["Z_ENT"] + typename = row["Z_NAME"] + super_id = row["Z_SUPER"] + if ( + not isinstance(ent_id, int) + or not isinstance(typename, str) + or not typename + or not isinstance(super_id, int) + ): + raise DatabaseSchemaError("Z_PRIMARYKEY contains invalid metadata") + if ent_id in ent_ids: + raise DatabaseSchemaError("Z_PRIMARYKEY contains duplicate entity IDs") + if typename in typenames: + raise DatabaseSchemaError( + "Z_PRIMARYKEY contains duplicate entity names" + ) + ent_ids.add(ent_id) + typenames.add(typename) + rows.append((ent_id, typename, super_id)) + return tuple(rows) + + @staticmethod + def _entity_maps( + rows: tuple[tuple[int, str, int], ...], + ) -> tuple[Dict[int, str], Dict[int, int], Dict[str, int]]: + ent_to_typename = {ent_id: typename for ent_id, typename, _ in rows} + ent_to_super = {ent_id: super_id for ent_id, _, super_id in rows} + typename_to_ent = {typename: ent_id for ent_id, typename, _ in rows} + return ent_to_typename, ent_to_super, typename_to_ent + + def _read_schema_identity( + self, + ) -> tuple[ + tuple[tuple[str, str, str, str | None], ...], + tuple[tuple[int, str, int], ...], + ]: + schema_rows = self._con.execute( + """ + SELECT type, name, tbl_name, sql + FROM sqlite_schema + WHERE name NOT LIKE 'sqlite_%' + ORDER BY type, name, tbl_name + """ + ).fetchall() + physical_schema = tuple( + (row["type"], row["name"], row["tbl_name"], row["sql"]) + for row in schema_rows + ) + return physical_schema, self._read_entity_metadata() def __repr__(self): return "\n".join( @@ -61,6 +188,25 @@ def typename_for(self, ent_id: ENT_ID) -> str: def ent_for(self, typename: str) -> ENT_ID: return self._typename_to_ent.get(typename) + def descendant_typenames(self, roots: tuple[str, ...]) -> list[str]: + """Return physical entity names descended from the requested roots.""" + root_ids = { + ent_id + for root in roots + if (ent_id := self._typename_to_ent.get(root)) is not None + } + descendants: list[str] = [] + for ent_id, typename in self._ent_to_typename.items(): + current = ent_id + visited: set[int] = set() + while current and current not in visited: + if current in root_ids: + descendants.append(typename) + break + visited.add(current) + current = self._ent_to_super.get(current, 0) + return descendants + def _table_exists(self, table_name: str) -> bool: row = self._con.execute( "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", @@ -68,17 +214,237 @@ def _table_exists(self, table_name: str) -> bool: ).fetchone() return row is not None + def _table_columns(self, table_name: str) -> set[str]: + return { + str(row["name"]) + for row in self._con.execute( + f'PRAGMA table_info("{table_name}")' + ).fetchall() + } + + def _relationship_storage( + self, + *, + entity_names: tuple[str, ...], + table_name: str, + required_columns: tuple[str, ...], + ) -> RelationshipStorage: + metadata_present = all(self.ent_for(name) is not None for name in entity_names) + table_present = self._table_exists(table_name) + if not metadata_present and not table_present: + return RelationshipStorage.ABSENT + if not metadata_present or not table_present: + return RelationshipStorage.UNKNOWN + if not set(required_columns).issubset(self._table_columns(table_name)): + return RelationshipStorage.UNKNOWN + return RelationshipStorage.PRESENT + + def _classify_tag_table(self, table_name: str) -> _TagTableShape: + """Classify an exact evidenced shape; a matching name alone is insufficient.""" + columns = frozenset(self._table_columns(table_name)) + if columns == SUPPORTED_UNRELATED_TAG_TABLES.get(table_name): + return _TagTableShape.UNRELATED + transaction_ent = self.ent_for("Transaction") + tag_ent = self.ent_for("Tag") + if ( + transaction_ent is not None + and tag_ent is not None + and table_name == f"Z_{transaction_ent}TAGS" + and columns == {f"Z_{transaction_ent}TRANSACTIONS", f"Z_{tag_ent}TAGS"} + ): + return _TagTableShape.DIRECT + return _TagTableShape.AMBIGUOUS + + def _transaction_tag_candidates(self) -> tuple[str, ...]: + """Return bounded tables that could store direct transaction-tag links.""" + rows = self._con.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name" + ).fetchall() + candidates: list[str] = [] + for row in rows: + table_name = row["name"] + if not isinstance( + table_name, str + ) or not TRANSACTION_TAG_TABLE_RE.fullmatch(table_name): + continue + if self._classify_tag_table(table_name) == _TagTableShape.UNRELATED: + continue + candidates.append(table_name) + return tuple(candidates) + + def _transaction_tag_storage( + self, + ) -> tuple[RelationshipStorage, str | None, str | None, str | None]: + transaction_ent = self.ent_for("Transaction") + tag_ent = self.ent_for("Tag") + candidates = self._transaction_tag_candidates() + metadata_count = sum(ent is not None for ent in (transaction_ent, tag_ent)) + if metadata_count == 0: + storage = ( + RelationshipStorage.UNKNOWN + if candidates + else RelationshipStorage.ABSENT + ) + return storage, candidates[0] if len(candidates) == 1 else None, None, None + if metadata_count == 1: + return RelationshipStorage.UNKNOWN, None, None, None + + table_name = f"Z_{transaction_ent}TAGS" + transaction_column = f"Z_{transaction_ent}TRANSACTIONS" + tag_column = f"Z_{tag_ent}TAGS" + if ( + candidates == (table_name,) + and self._classify_tag_table(table_name) == _TagTableShape.DIRECT + ): + storage = RelationshipStorage.PRESENT + else: + storage = RelationshipStorage.UNKNOWN + return storage, table_name, transaction_column, tag_column + + @staticmethod + def _relationship_error(exc: Exception) -> LoadErrorKind: + if isinstance(exc, KeyError): + return LoadErrorKind.MISSING_FIELD + if isinstance(exc, AssertionError): + return LoadErrorKind.VALIDATION + if isinstance(exc, ValueError): + return LoadErrorKind.INVALID_VALUE + return LoadErrorKind.CONSTRUCTION + def query_objects(self, typenames: List[str]) -> List[Any]: + """Query live rows against the verified cached entity mapping.""" + with self.read_transaction(): + return self._query_objects(typenames) + + def _query_objects(self, typenames: List[str]) -> List[Any]: + ent_ids = [self.ent_for(name) for name in typenames] + ent_ids = [ent_id for ent_id in ent_ids if ent_id is not None] + if not ent_ids: + return [] cur = self._con.cursor() res = cur.execute( """ SELECT * FROM ZSYNCOBJECT WHERE Z_ENT in (%s) """ - % (",".join("?" * len(typenames))), - [self.ent_for(x) for x in typenames], + % (",".join("?" * len(ent_ids))), + ent_ids, ) return res.fetchall() + def close(self) -> None: + """Close the read-only SQLite connection.""" + self._con.close() + + @contextmanager + def _raw_read_transaction(self): + """Provide snapshot ownership without consulting schema caches.""" + owns_transaction = not self._con.in_transaction + if owns_transaction: + self._con.execute("BEGIN") + try: + yield + finally: + if owns_transaction and self._transaction_is_active(): + self._con.rollback() + + def _transaction_is_active(self) -> bool: + try: + return self._con.in_transaction + except sqlite3.Error: + return False + + def _verify_schema_identity(self) -> None: + try: + baseline = self._schema_identity + except AttributeError as exc: + raise DatabaseSchemaError( + "database schema cache is not initialized; close and reopen the accessor" + ) from exc + try: + current = self._read_schema_identity() + except DatabaseSchemaError: + raise + except sqlite3.Error as exc: + raise DatabaseSchemaError( + "database schema could not be verified; close and reopen the accessor" + ) from exc + if current != baseline: + raise DatabaseSchemaError( + "database schema changed; close and reopen the accessor" + ) + + def _verify_source_eligibility(self) -> None: + """Reject rows whose entity ancestry cannot be classified safely.""" + try: + rows = self._con.execute( + "SELECT DISTINCT Z_ENT FROM ZSYNCOBJECT ORDER BY Z_ENT" + ).fetchall() + except sqlite3.Error as exc: + raise DatabaseSchemaError( + "database entity sources could not be verified" + ) from exc + + for row in rows: + current = row["Z_ENT"] + if not isinstance(current, int) or current not in self._ent_to_super: + raise DatabaseSchemaError( + "database contains rows with unclassifiable entity ancestry" + ) + visited: set[int] = set() + while current != 0: + if current in visited or current not in self._ent_to_super: + raise DatabaseSchemaError( + "database contains rows with unclassifiable entity ancestry" + ) + visited.add(current) + current = self._ent_to_super[current] + + @contextmanager + def read_transaction(self): + """Keep cache-dependent reads on one verified SQLite snapshot.""" + admission_depth = getattr(self, "_admission_depth", 0) + admission_broken = getattr(self, "_admission_broken", False) + if admission_depth and (admission_broken or not self._transaction_is_active()): + self._admission_broken = True + raise DatabaseSchemaError( + "database read transaction was interrupted; close and reopen the accessor" + ) + + with self._raw_read_transaction(): + owns_admission = admission_depth == 0 + if owns_admission: + self._verify_schema_identity() + self._verify_source_eligibility() + if not self._transaction_is_active(): + raise DatabaseSchemaError( + "database read transaction was interrupted; " + "close and reopen the accessor" + ) + self._admission_depth += 1 + try: + yield + except BaseException: + if not self._transaction_is_active(): + self._admission_broken = True + raise + else: + if self._admission_broken or not self._transaction_is_active(): + self._admission_broken = True + raise DatabaseSchemaError( + "database read transaction was interrupted; " + "close and reopen the accessor" + ) + finally: + self._admission_depth -= 1 + if owns_admission: + self._admission_broken = False + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.close() + def _construct_record(self, row, constructor: Callable): investment_constructors = { InvestmentBuyTransaction, @@ -86,10 +452,19 @@ def _construct_record(self, row, constructor: Callable): InvestmentHolding, } if constructor in investment_constructors: - return constructor(row, schema_profile=self.schema_profile) - return constructor(row) + record = constructor(row, schema_profile=self.schema_profile) + else: + record = constructor(row) + validate = getattr(record, "validate", None) + if validate is not None: + validate() + return record def get_record(self, pk_id: ID, constructor: Callable = Record): + with self.read_transaction(): + return self._get_record(pk_id, constructor) + + def _get_record(self, pk_id: ID, constructor: Callable): cur = self._con.cursor() res = cur.execute( """ @@ -102,6 +477,10 @@ def get_record(self, pk_id: ID, constructor: Callable = Record): return self._construct_record(res.fetchone(), constructor) def get_record_by_gid(self, gid: GID, constructor: Callable = Record): + with self.read_transaction(): + return self._get_record_by_gid(gid, constructor) + + def _get_record_by_gid(self, gid: GID, constructor: Callable): cur = self._con.cursor() res = cur.execute( """ @@ -113,52 +492,257 @@ def get_record_by_gid(self, gid: GID, constructor: Callable = Record): return self._construct_record(res.fetchone(), constructor) - def get_category_assignment(self) -> Dict[ID, List[Tuple[ID, Decimal]]]: + def read_category_assignments( + self, + ) -> tuple[Dict[ID, List[Tuple[ID, Decimal]]], RelationshipLoadReport]: + with self.read_transaction(): + return self._read_category_assignments() + + def _read_category_assignments( + self, + ) -> tuple[Dict[ID, List[Tuple[ID, Decimal]]], RelationshipLoadReport]: transaction_map: Dict[ID, List[Tuple[ID, Decimal]]] = defaultdict(list) - if not self._table_exists("ZCATEGORYASSIGMENT"): - return transaction_map - cur = self._con.cursor() - res = cur.execute( - """ - SELECT ZCATEGORY, ZTRANSACTION, ZAMOUNT FROM ZCATEGORYASSIGMENT WHERE ZTRANSACTION IS NOT NULL - - """ + table_name = "ZCATEGORYASSIGMENT" + columns = ("Z_PK", "ZCATEGORY", "ZTRANSACTION", "ZAMOUNT") + storage = self._relationship_storage( + entity_names=("CategoryAssigment",), + table_name=table_name, + required_columns=columns, ) - for row in res.fetchall(): - transaction_map[row["ZTRANSACTION"]].append( - (row["ZCATEGORY"], RDH.get_decimal(row, "ZAMOUNT")) + if storage != RelationshipStorage.PRESENT: + return transaction_map, RelationshipLoadReport( + storage=storage, + storage_name=table_name, ) - return transaction_map - def get_refund_maps(self) -> Dict[ID, ID]: + source_ids: list[ID | None] = [] + parsed_ids: list[ID] = [] + skipped: list[SkippedRecord] = [] + rows = self._con.execute( + f'SELECT {", ".join(columns)} FROM "{table_name}" ' + "WHERE ZTRANSACTION IS NOT NULL" + ).fetchall() + for row in rows: + raw_source_id = row.get("Z_PK") + source_id = raw_source_id if type(raw_source_id) is int else None + source_ids.append(source_id) + try: + require_valid( + raw_source_id is not None, "category assignment ID is required" + ) + require_integer_identity( + raw_source_id, + "category assignment ID must be an uncoerced integer", + ) + category_id = row["ZCATEGORY"] + transaction_id = row["ZTRANSACTION"] + require_valid( + category_id is not None, + "category assignment category endpoint is required", + ) + require_valid( + transaction_id is not None, + "category assignment transaction endpoint is required", + ) + require_integer_identity( + category_id, + "category assignment category endpoint must be an uncoerced integer", + ) + require_integer_identity( + transaction_id, + "category assignment transaction endpoint must be an uncoerced integer", + ) + amount = RDH.get_decimal(row, "ZAMOUNT") + transaction_map[transaction_id].append((category_id, amount)) + except Exception as exc: + skipped.append( + SkippedRecord( + record_id=source_id, + entity="CategoryAssigment", + error=self._relationship_error(exc), + exception_type=type(exc).__name__, + ) + ) + continue + parsed_ids.append(source_id) + return transaction_map, RelationshipLoadReport( + storage=storage, + storage_name=table_name, + source_ids=tuple(source_ids), + parsed_ids=tuple(parsed_ids), + skipped=tuple(skipped), + ) + + def get_category_assignment(self) -> Dict[ID, List[Tuple[ID, Decimal]]]: + """Return category assignments without completeness metadata.""" + with self.read_transaction(): + assignments, _ = self._read_category_assignments() + return assignments + + def read_refund_maps( + self, + ) -> tuple[Dict[ID, ID], RelationshipLoadReport]: + with self.read_transaction(): + return self._read_refund_maps() + + def _read_refund_maps( + self, + ) -> tuple[Dict[ID, ID], RelationshipLoadReport]: refund_to_withdraw: Dict[ID, ID] = {} - if not self._table_exists("ZWITHDRAWREFUNDTRANSACTIONLINK"): - return refund_to_withdraw - cur = self._con.cursor() - res = cur.execute( - """ - SELECT ZREFUNDTRANSACTION, ZWITHDRAWTRANSACTION FROM ZWITHDRAWREFUNDTRANSACTIONLINK - - """ + table_name = "ZWITHDRAWREFUNDTRANSACTIONLINK" + columns = ("Z_PK", "ZREFUNDTRANSACTION", "ZWITHDRAWTRANSACTION") + storage = self._relationship_storage( + entity_names=("WithdrawRefundTransactionLink",), + table_name=table_name, + required_columns=columns, ) - for row in res.fetchall(): - refund_to_withdraw[row["ZREFUNDTRANSACTION"]] = row["ZWITHDRAWTRANSACTION"] - return refund_to_withdraw + if storage != RelationshipStorage.PRESENT: + return refund_to_withdraw, RelationshipLoadReport( + storage=storage, + storage_name=table_name, + ) - def get_tags_map(self) -> Dict[ID, List[ID]]: + source_ids: list[ID | None] = [] + parsed_ids: list[ID] = [] + skipped: list[SkippedRecord] = [] + rows = self._con.execute( + f'SELECT {", ".join(columns)} FROM "{table_name}"' + ).fetchall() + for row in rows: + raw_source_id = row.get("Z_PK") + source_id = raw_source_id if type(raw_source_id) is int else None + source_ids.append(source_id) + try: + require_valid(raw_source_id is not None, "refund link ID is required") + require_integer_identity( + raw_source_id, "refund link ID must be an uncoerced integer" + ) + refund_id = row["ZREFUNDTRANSACTION"] + withdraw_id = row["ZWITHDRAWTRANSACTION"] + require_valid( + refund_id is not None, "refund transaction endpoint is required" + ) + require_valid( + withdraw_id is not None, "withdraw transaction endpoint is required" + ) + require_integer_identity( + refund_id, + "refund transaction endpoint must be an uncoerced integer", + ) + require_integer_identity( + withdraw_id, + "withdraw transaction endpoint must be an uncoerced integer", + ) + if refund_id in refund_to_withdraw: + raise ValueError() + refund_to_withdraw[refund_id] = withdraw_id + except Exception as exc: + skipped.append( + SkippedRecord( + record_id=source_id, + entity="WithdrawRefundTransactionLink", + error=self._relationship_error(exc), + exception_type=type(exc).__name__, + ) + ) + continue + parsed_ids.append(source_id) + return refund_to_withdraw, RelationshipLoadReport( + storage=storage, + storage_name=table_name, + source_ids=tuple(source_ids), + parsed_ids=tuple(parsed_ids), + skipped=tuple(skipped), + ) + + def get_refund_maps(self) -> Dict[ID, ID]: + """Return refund links without completeness metadata.""" + with self.read_transaction(): + refund_maps, _ = self._read_refund_maps() + return refund_maps + + def read_tags_map( + self, + ) -> tuple[Dict[ID, List[ID]], RelationshipLoadReport]: + with self.read_transaction(): + return self._read_tags_map() + + def _read_tags_map( + self, + ) -> tuple[Dict[ID, List[ID]], RelationshipLoadReport]: transactions_to_tags: Dict[ID, List[ID]] = defaultdict(list) - if not self._table_exists("Z_36TAGS"): - return transactions_to_tags - cur = self._con.cursor() - res = cur.execute( - """ - SELECT Z_36TRANSACTIONS, Z_35TAGS FROM Z_36TAGS - - """ + storage, table_name, transaction_column, tag_column = ( + self._transaction_tag_storage() ) - for row in res.fetchall(): - transactions_to_tags[row["Z_36TRANSACTIONS"]].append(row["Z_35TAGS"]) - return transactions_to_tags + if storage != RelationshipStorage.PRESENT: + return transactions_to_tags, RelationshipLoadReport( + storage=storage, + storage_name=table_name, + ) + require_valid(table_name is not None, "transaction-tag table is required") + require_valid( + transaction_column is not None, + "transaction-tag transaction column is required", + ) + require_valid(tag_column is not None, "transaction-tag tag column is required") + columns = (transaction_column, tag_column) + + source_ids: list[str] = [] + parsed_ids: list[str] = [] + skipped: list[SkippedRecord] = [] + rows = self._con.execute( + f'SELECT {", ".join(columns)} FROM "{table_name}"' + ).fetchall() + for position, row in enumerate(rows): + transaction_id = row.get(transaction_column) + tag_id = row.get(tag_column) + source_id = ( + f"{transaction_id}:{tag_id}" + if type(transaction_id) is int and type(tag_id) is int + else f"row:{position}" + ) + source_ids.append(source_id) + try: + require_valid( + transaction_id is not None, + "transaction-tag transaction endpoint is required", + ) + require_valid( + tag_id is not None, "transaction-tag tag endpoint is required" + ) + require_integer_identity( + transaction_id, + "transaction-tag transaction endpoint must be an uncoerced integer", + ) + require_integer_identity( + tag_id, + "transaction-tag tag endpoint must be an uncoerced integer", + ) + transactions_to_tags[transaction_id].append(tag_id) + except Exception as exc: + skipped.append( + SkippedRecord( + record_id=source_id, + entity="TransactionTag", + error=self._relationship_error(exc), + exception_type=type(exc).__name__, + ) + ) + continue + parsed_ids.append(source_id) + return transactions_to_tags, RelationshipLoadReport( + storage=storage, + storage_name=table_name, + source_ids=tuple(source_ids), + parsed_ids=tuple(parsed_ids), + skipped=tuple(skipped), + ) + + def get_tags_map(self) -> Dict[ID, List[ID]]: + """Return transaction tags without completeness metadata.""" + with self.read_transaction(): + transactions_to_tags, _ = self._read_tags_map() + return transactions_to_tags def get_users(self) -> Dict[ID, str]: users_map: Dict[ID, str] = {} diff --git a/src/moneywiz_api/managers/account_manager.py b/src/moneywiz_api/managers/account_manager.py index 5a5ed48..2c53984 100644 --- a/src/moneywiz_api/managers/account_manager.py +++ b/src/moneywiz_api/managers/account_manager.py @@ -31,6 +31,10 @@ def ents(self) -> Dict[str, Callable]: "ForexAccount": ForexAccount, } + @property + def entity_roots(self) -> tuple[str, ...]: + return ("Account",) + def records(self) -> Dict[ID, Account]: return dict(sorted(super().records().items(), key=lambda x: x[1].display_order)) diff --git a/src/moneywiz_api/managers/category_manager.py b/src/moneywiz_api/managers/category_manager.py index e124e13..b149c95 100644 --- a/src/moneywiz_api/managers/category_manager.py +++ b/src/moneywiz_api/managers/category_manager.py @@ -15,6 +15,10 @@ def ents(self) -> Dict[str, Callable]: "Category": Category, } + @property + def entity_roots(self) -> tuple[str, ...]: + return ("Category",) + def get_name_chain(self, category_id: ID) -> List[str]: ret: List[str] = [] current = self.get(category_id) diff --git a/src/moneywiz_api/managers/investment_holding_manager.py b/src/moneywiz_api/managers/investment_holding_manager.py index 069d02c..2e1e68a 100644 --- a/src/moneywiz_api/managers/investment_holding_manager.py +++ b/src/moneywiz_api/managers/investment_holding_manager.py @@ -17,6 +17,10 @@ def ents(self) -> Dict[str, Callable]: "InvestmentHolding": InvestmentHolding, } + @property + def entity_roots(self) -> tuple[str, ...]: + return ("InvestmentHolding",) + def construct_record( self, constructor: Callable, record, db_accessor: DatabaseAccessor ): diff --git a/src/moneywiz_api/managers/payee_manager.py b/src/moneywiz_api/managers/payee_manager.py index 714d0a2..1c92748 100644 --- a/src/moneywiz_api/managers/payee_manager.py +++ b/src/moneywiz_api/managers/payee_manager.py @@ -13,3 +13,7 @@ def ents(self) -> Dict[str, Callable]: return { "Payee": Payee, } + + @property + def entity_roots(self) -> tuple[str, ...]: + return ("Payee",) diff --git a/src/moneywiz_api/managers/record_manager.py b/src/moneywiz_api/managers/record_manager.py index f158c43..faf48a1 100644 --- a/src/moneywiz_api/managers/record_manager.py +++ b/src/moneywiz_api/managers/record_manager.py @@ -1,9 +1,14 @@ from abc import ABC, abstractmethod import logging -from typing import Dict, Generic, TypeVar, Callable +from typing import Callable, Dict, Generic, TypeVar from moneywiz_api.database_accessor import DatabaseAccessor from moneywiz_api.model.record import Record +from moneywiz_api.read_result import ( + LoadErrorKind, + ManagerLoadReport, + SkippedRecord, +) from moneywiz_api.types import ID, GID @@ -11,39 +16,126 @@ logger = logging.getLogger(__name__) +class DuplicateRecordIdError(ValueError): + """Raised when two source rows use the same primary key.""" + + +class DuplicateRecordGidError(ValueError): + """Raised when two source rows use the same global identifier.""" + + class RecordManager(ABC, Generic[T]): def __init__(self): self._records: Dict[ID, T] = {} self._gid_to_id: Dict[GID, ID] = {} - self._load_errors: list[tuple[ID, str, str]] = [] + self._load_report = ManagerLoadReport.unloaded() @property @abstractmethod def ents(self) -> Dict[str, Callable]: raise NotImplementedError() - def load(self, db_accessor: DatabaseAccessor) -> None: - records = db_accessor.query_objects(self.ents.keys()) + @property + def entity_roots(self) -> tuple[str, ...]: + """Return schema roots whose unknown descendants must be reported.""" + return () + + def load(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: + self._discard_incomplete_load() + try: + with db_accessor.read_transaction(): + report = self._load_in_transaction(db_accessor) + except BaseException: + self._discard_incomplete_load() + raise + self._load_report = report + return report + + def _load_in_transaction(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: + """Load all manager state inside the caller-owned read snapshot.""" + return self._load_records(db_accessor) + + def _load_records(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: + """Read rows while keeping the public report unloaded until publication.""" + typenames = list(self.ents) + if self.entity_roots: + discovery_roots = tuple( + dict.fromkeys((*self.entity_roots, *self.ents.keys())) + ) + discovered = db_accessor.descendant_typenames(discovery_roots) + typenames = list(dict.fromkeys([*typenames, *discovered])) + records = db_accessor.query_objects(typenames) + + source_ids: list[ID | None] = [] + parsed_ids: list[ID] = [] + skipped: list[SkippedRecord] = [] for record in records: - typename = db_accessor.typename_for(record["Z_ENT"]) - if typename in self.ents: - try: - obj = self.construct_record( - self.ents[typename], record, db_accessor - ) - except (AssertionError, KeyError, ValueError) as exc: - record_id = record.get("Z_PK") - detail = type(exc).__name__ - self._load_errors.append((record_id, typename, detail)) - logger.debug( - "Skipping unreadable %s record %s: %s", - typename, - record_id, - detail, - ) - continue + raw_record_id = record.get("Z_PK") + record_id = raw_record_id if type(raw_record_id) is int else None + source_ids.append(record_id) + typename = None + try: + typename = db_accessor.typename_for(record["Z_ENT"]) + if typename not in self.ents: + raise UnknownEntityError() + obj = self.construct_record(self.ents[typename], record, db_accessor) + validate = getattr(obj, "validate", None) + if validate is not None: + validate() self.add(obj) + except Exception as exc: + error = self._error_kind(exc) + skipped.append( + SkippedRecord( + record_id=record_id, + entity=typename, + error=error, + exception_type=type(exc).__name__, + ) + ) + logger.debug( + "Skipping unreadable %s record %s: %s", + typename, + record_id, + error.value, + ) + continue + parsed_ids.append(obj.id) + + return ManagerLoadReport( + source_ids=tuple(source_ids), + parsed_ids=tuple(parsed_ids), + skipped=tuple(skipped), + ) + + def _discard_incomplete_load(self) -> None: + """Clear state when a direct manager load did not finish.""" + self._records = {} + self._gid_to_id = {} + self._load_report = ManagerLoadReport.unloaded() + + def _adopt_loaded_state(self, staged: "RecordManager[T]") -> None: + """Publish a successfully staged load without replacing this manager.""" + self._records = staged._records + self._gid_to_id = staged._gid_to_id + self._load_report = staged._load_report + + @staticmethod + def _error_kind(exc: Exception) -> LoadErrorKind: + if isinstance(exc, UnknownEntityError): + return LoadErrorKind.UNKNOWN_ENTITY + if isinstance(exc, DuplicateRecordIdError): + return LoadErrorKind.DUPLICATE_ID + if isinstance(exc, DuplicateRecordGidError): + return LoadErrorKind.DUPLICATE_GID + if isinstance(exc, KeyError): + return LoadErrorKind.MISSING_FIELD + if isinstance(exc, AssertionError): + return LoadErrorKind.VALIDATION + if isinstance(exc, ValueError): + return LoadErrorKind.INVALID_VALUE + return LoadErrorKind.CONSTRUCTION def construct_record( self, constructor: Callable, record, db_accessor: DatabaseAccessor @@ -52,12 +144,12 @@ def construct_record( return constructor(record) def add(self, record: T) -> None: - self._records[record.id] = record + if record.id in self._records: + raise DuplicateRecordIdError() if record.gid in self._gid_to_id: - raise RuntimeError( - f"Duplicate gid for {record}, existing record Id {self._gid_to_id[record.gid]}" - ) + raise DuplicateRecordGidError() + self._records[record.id] = record self._gid_to_id[record.gid] = record.id def get(self, record_id: ID) -> T | None: @@ -70,9 +162,21 @@ def records(self) -> Dict[ID, T]: return self._records @property - def load_errors(self) -> list[tuple[ID, str, str]]: + def load_errors(self) -> list[tuple[ID | None, str | None, str]]: """Return records skipped during best-effort read parsing.""" - return list(self._load_errors) + return [ + (item.record_id, item.entity, item.exception_type) + for item in self._load_report.skipped + ] + + @property + def load_report(self) -> ManagerLoadReport: + """Return structured completeness evidence from the latest load.""" + return self._load_report def __repr__(self): return "\n".join(f"{key}: {value}" for key, value in self.records().items()) + + +class UnknownEntityError(ValueError): + """Raised when a source row belongs to an unsupported subtype.""" diff --git a/src/moneywiz_api/managers/tag_manager.py b/src/moneywiz_api/managers/tag_manager.py index 57de588..8a4bb32 100644 --- a/src/moneywiz_api/managers/tag_manager.py +++ b/src/moneywiz_api/managers/tag_manager.py @@ -13,3 +13,7 @@ def ents(self) -> Dict[str, Callable]: return { "Tag": Tag, } + + @property + def entity_roots(self) -> tuple[str, ...]: + return ("Tag",) diff --git a/src/moneywiz_api/managers/transaction_manager.py b/src/moneywiz_api/managers/transaction_manager.py index 03f6f36..26b7af1 100644 --- a/src/moneywiz_api/managers/transaction_manager.py +++ b/src/moneywiz_api/managers/transaction_manager.py @@ -1,4 +1,5 @@ from datetime import datetime +from dataclasses import replace from typing import Dict, Callable, List, Tuple from decimal import Decimal @@ -17,6 +18,7 @@ WithdrawTransaction, ) from moneywiz_api.managers.record_manager import RecordManager +from moneywiz_api.read_result import ManagerLoadReport from moneywiz_api.types import ID @@ -25,7 +27,7 @@ def __init__(self): super().__init__() self.category_assignment: Dict[ID, List[Tuple[ID, Decimal]]] = {} self.refund_maps: Dict[ID, ID] = {} - self.tags_map: Dict[ID, ID] = {} + self.tags_map: Dict[ID, List[ID]] = {} @property def ents(self) -> Dict[str, Callable]: @@ -42,6 +44,10 @@ def ents(self) -> Dict[str, Callable]: "WithdrawTransaction": WithdrawTransaction, } + @property + def entity_roots(self) -> tuple[str, ...]: + return ("Transaction",) + def construct_record( self, constructor: Callable, record, db_accessor: DatabaseAccessor ): @@ -53,13 +59,36 @@ def construct_record( return constructor(record, schema_profile=db_accessor.schema_profile) return super().construct_record(constructor, record, db_accessor) - def load(self, db_accessor: DatabaseAccessor) -> None: - super().load(db_accessor) - self.category_assignment: Dict[ID, List[Tuple[ID, Decimal]]] = ( - db_accessor.get_category_assignment() + def _load_in_transaction(self, db_accessor: DatabaseAccessor) -> ManagerLoadReport: + report = super()._load_in_transaction(db_accessor) + category_assignment, category_report = db_accessor.read_category_assignments() + refund_maps, refund_report = db_accessor.read_refund_maps() + tags_map, tags_report = db_accessor.read_tags_map() + self.category_assignment = category_assignment + self.refund_maps = refund_maps + self.tags_map = tags_map + return replace( + report, + relationships={ + "category_assignments": category_report, + "refund_links": refund_report, + "transaction_tags": tags_report, + }, ) - self.refund_maps: Dict[ID, ID] = db_accessor.get_refund_maps() - self.tags_map: Dict[ID, ID] = db_accessor.get_tags_map() + + def _discard_incomplete_load(self) -> None: + """Clear records and relationships after an incomplete load.""" + super()._discard_incomplete_load() + self.category_assignment = {} + self.refund_maps = {} + self.tags_map = {} + + def _adopt_loaded_state(self, staged: "TransactionManager") -> None: + """Publish records and relationship state from one completed load.""" + super()._adopt_loaded_state(staged) + self.category_assignment = staged.category_assignment + self.refund_maps = staged.refund_maps + self.tags_map = staged.tags_map def category_for_transaction( self, transaction_id: ID diff --git a/src/moneywiz_api/model/account.py b/src/moneywiz_api/model/account.py index d46d97b..accd1c3 100644 --- a/src/moneywiz_api/model/account.py +++ b/src/moneywiz_api/model/account.py @@ -1,10 +1,16 @@ from abc import ABC from dataclasses import dataclass, field from decimal import Decimal +from typing import Optional from moneywiz_api.types import ID from moneywiz_api.model.raw_data_handler import RawDataHandler as RDH from moneywiz_api.model.record import Record +from moneywiz_api.validation import ( + require_integer_identity, + require_text, + require_valid, +) @dataclass @@ -19,7 +25,7 @@ class Account(Record, ABC): name: str currency: str opening_balance: Decimal # might be a tiny number - info: str + info: Optional[str] user: ID def __init__(self, row): @@ -30,20 +36,37 @@ def __init__(self, row): self.name = row["ZNAME"] self.currency = row["ZCURRENCYNAME"] self.opening_balance = RDH.get_decimal(row, "ZOPENINGBALANCE") - self.info = row["ZINFO"] + self.info = row.get("ZINFO") self.user = row["ZUSER"] - - # Fixes - - # Validate - assert self.display_order is not None, self.as_dict() - assert self.group_id is not None, self.as_dict() - assert self.name is not None, self.as_dict() - assert self.currency is not None, self.as_dict() - assert self.opening_balance is not None, self.as_dict() - assert self.info is not None, self.as_dict() - assert self.user is not None, self.as_dict() + Account.validate(self) + + def validate(self) -> None: + require_valid( + self.display_order is not None, "account display order is required" + ) + require_integer_identity( + self.display_order, + "account display order must be an uncoerced integer", + ) + require_valid(self.group_id is not None, "account group identity is required") + require_integer_identity( + self.group_id, "account group identity must be an uncoerced integer" + ) + require_valid(self.name is not None, "account name is required") + require_text(self.name, "account name must be an uncoerced string") + require_valid(self.currency is not None, "account currency is required") + require_valid( + self.opening_balance is not None, "account opening balance is required" + ) + # info is nullable in valid MoneyWiz stores. + require_text( + self.info, "account info must be an uncoerced string", optional=True + ) + require_valid(self.user is not None, "account owner is required") + require_integer_identity( + self.user, "account owner must be an uncoerced integer" + ) @dataclass @@ -87,9 +110,13 @@ class CreditCardAccount(Account): def __init__(self, row): super().__init__(row) self.statement_day = row["ZSTATEMENTENDDAY"] + CreditCardAccount.validate(self) - # Validate - assert self.statement_day is not None + def validate(self) -> None: + super().validate() + require_valid( + self.statement_day is not None, "account statement day is required" + ) @dataclass diff --git a/src/moneywiz_api/model/category.py b/src/moneywiz_api/model/category.py index 10d9405..71c5886 100644 --- a/src/moneywiz_api/model/category.py +++ b/src/moneywiz_api/model/category.py @@ -3,6 +3,11 @@ from moneywiz_api.model.record import Record from moneywiz_api.types import CategoryType, ID +from moneywiz_api.validation import ( + require_integer_identity, + require_text, + require_valid, +) @dataclass @@ -26,12 +31,21 @@ def __init__(self, row): # Fixes # Validate - assert self.name is not None, self.as_dict() - assert self.type is not None, self.as_dict() - assert self.user is not None, self.as_dict() + require_valid(self.name is not None, "category name is required") + require_text(self.name, "category name must be an uncoerced string") + require_integer_identity( + self.parent_id, + "category parent must be an uncoerced integer", + optional=True, + ) + require_valid(self.type is not None, "category type is required") + require_valid(self.user is not None, "category owner is required") + require_integer_identity( + self.user, "category owner must be an uncoerced integer" + ) @staticmethod def _convert_type(type_: Optional[int]) -> CategoryType: if type_ and type_ in [1, 2]: return "Expenses" if type_ == 1 else "Income" - raise RuntimeError(f"Invalid type {type_}") + raise ValueError("unsupported category type") diff --git a/src/moneywiz_api/model/investment_holding.py b/src/moneywiz_api/model/investment_holding.py index 3b1dd8a..39c4792 100644 --- a/src/moneywiz_api/model/investment_holding.py +++ b/src/moneywiz_api/model/investment_holding.py @@ -6,6 +6,7 @@ from moneywiz_api.model.record import Record from moneywiz_api.schema_profile import SchemaProfile from moneywiz_api.types import ID +from moneywiz_api.validation import require_integer_identity, require_valid @dataclass @@ -18,7 +19,7 @@ class InvestmentHolding(Record): opening_number_of_shares: Optional[Decimal] number_of_shares: Decimal - # price_per_share: Decimal + price_per_share: Optional[Decimal] symbol: str holding_type: Optional[str] description: str @@ -56,12 +57,26 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): "ZNUMBEROFSHARES", "ZNUMBEROFSHARES1", ) - # self.price_per_share = row["ZPRICEPERSHARE"] + self.price_per_share = RDH.get_profile_nullable_decimal( + row, + schema_profile.holding_price_per_share_column if schema_profile else None, + "ZPRICEPERSHARE", + "ZPRICEPERSHARE1", + ) self.symbol = row["ZSYMBOL"] self.holding_type = row["ZHOLDINGTYPE"] self.description = row["ZDESC"] + raw_price_per_share_available_online = row["ZISPRICEPERSHAREAVAILABLEONLINE"] + require_integer_identity( + raw_price_per_share_available_online, + "holding online-price state must be an uncoerced integer", + ) + require_valid( + raw_price_per_share_available_online in (0, 1), + "holding online-price state must be zero or one", + ) self.price_per_share_available_online = ( - row["ZISPRICEPERSHAREAVAILABLEONLINE"] == 1 + raw_price_per_share_available_online == 1 ) self._investment_object_type = row["ZINVESTMENTOBJECTTYPE"] @@ -69,21 +84,27 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): row, "ZCOSTBASISOFMISSINGOBSHARES" ) - # Fixes - self.number_of_shares = self.number_of_shares or Decimal(0) - # Validate self.validate() def validate(self): - assert self.account is not None, self.as_dict() - assert self.number_of_shares is not None, self.as_dict() - # assert self.price_per_share is not None - assert self.symbol is not None, self.as_dict() - assert self.description is not None, self.as_dict() + require_valid(self.account is not None, "holding account is required") + require_integer_identity( + self.account, "holding account must be an uncoerced integer" + ) + require_valid(self.number_of_shares is not None, "holding quantity is required") + # price_per_share can be None when no current quote is stored. + require_valid(self.symbol is not None, "holding symbol is required") + require_valid(self.description is not None, "holding description is required") - assert self._investment_object_type is not None, self.as_dict() - assert self._cost_basis_of_missing_ob_shares is not None, self.as_dict() + require_valid( + self._investment_object_type is not None, + "holding investment object type is required", + ) + require_valid( + self._cost_basis_of_missing_ob_shares is not None, + "holding missing-share cost basis is required", + ) def as_dict(self) -> Dict[str, Any]: original = super().as_dict() diff --git a/src/moneywiz_api/model/payee.py b/src/moneywiz_api/model/payee.py index 836e303..b71be40 100644 --- a/src/moneywiz_api/model/payee.py +++ b/src/moneywiz_api/model/payee.py @@ -2,6 +2,11 @@ from moneywiz_api.model.record import Record from moneywiz_api.types import ID +from moneywiz_api.validation import ( + require_integer_identity, + require_text, + require_valid, +) @dataclass @@ -21,5 +26,7 @@ def __init__(self, row): # Fixes # Validate - assert self.name is not None, self.as_dict() - assert self.user is not None, self.as_dict() + require_valid(self.name is not None, "payee name is required") + require_text(self.name, "payee name must be an uncoerced string") + require_valid(self.user is not None, "payee owner is required") + require_integer_identity(self.user, "payee owner must be an uncoerced integer") diff --git a/src/moneywiz_api/model/raw_data_handler.py b/src/moneywiz_api/model/raw_data_handler.py index 5ad237b..0d98d15 100644 --- a/src/moneywiz_api/model/raw_data_handler.py +++ b/src/moneywiz_api/model/raw_data_handler.py @@ -3,15 +3,16 @@ from decimal import Decimal from moneywiz_api.utils import get_datetime +from moneywiz_api.validation import require_valid class RawDataHandler: @staticmethod def get_datetime(row: Dict[str, Any], key: str) -> datetime: raw_value = row[key] - assert isinstance(raw_value, float) or isinstance(raw_value, int), ( - f"row['{key}'] = {row[key]}, is not a float or int, where row is: " - + str(RawDataHandler.filter_row(row)) + require_valid( + isinstance(raw_value, float) or isinstance(raw_value, int), + "datetime field must be numeric", ) return get_datetime(raw_value) @@ -26,9 +27,9 @@ def get_nullable_decimal(row: Dict[str, Any], key: str) -> Optional[Decimal]: @staticmethod def get_decimal(row: Dict[str, Any], key: str) -> Decimal: raw_value = row[key] - assert isinstance(raw_value, float) or isinstance(raw_value, int), ( - f"row['{key}'] = {row[key]}, is not a float or int, where row is: " - + str(RawDataHandler.filter_row(row)) + require_valid( + isinstance(raw_value, float) or isinstance(raw_value, int), + "decimal field must be numeric", ) return Decimal(str(raw_value)) diff --git a/src/moneywiz_api/model/record.py b/src/moneywiz_api/model/record.py index d35b99b..c11a188 100644 --- a/src/moneywiz_api/model/record.py +++ b/src/moneywiz_api/model/record.py @@ -4,6 +4,11 @@ from moneywiz_api.types import ID, ENT_ID from moneywiz_api.model.raw_data_handler import RawDataHandler as RDH +from moneywiz_api.validation import ( + require_integer_identity, + require_text, + require_valid, +) @dataclass @@ -24,11 +29,18 @@ def __init__(self, row): # Fixes # Validate - assert self._raw - assert self._ent - assert self._created_at - assert self.gid - assert self.id + require_valid(self._raw, "record source must not be empty") + require_valid(self._ent, "record entity must be set") + require_integer_identity( + self._ent, "record entity must be an uncoerced integer" + ) + require_valid(self._created_at, "record creation date must be set") + require_valid(self.gid, "record global identity must be set") + require_text(self.gid, "record global identity must be an uncoerced string") + require_valid(self.id, "record identity must be set") + require_integer_identity( + self.id, "record identity must be an uncoerced integer" + ) def ent(self) -> ENT_ID: return self._ent diff --git a/src/moneywiz_api/model/tag.py b/src/moneywiz_api/model/tag.py index 11b61d5..b1e2cd0 100644 --- a/src/moneywiz_api/model/tag.py +++ b/src/moneywiz_api/model/tag.py @@ -2,6 +2,11 @@ from moneywiz_api.model.record import Record from moneywiz_api.types import ID +from moneywiz_api.validation import ( + require_integer_identity, + require_text, + require_valid, +) @dataclass @@ -21,5 +26,7 @@ def __init__(self, row): # Fixes # Validate - assert self.name is not None, self.as_dict() - assert self.user is not None, self.as_dict() + require_valid(self.name is not None, "tag name is required") + require_text(self.name, "tag name must be an uncoerced string") + require_valid(self.user is not None, "tag owner is required") + require_integer_identity(self.user, "tag owner must be an uncoerced integer") diff --git a/src/moneywiz_api/model/transaction.py b/src/moneywiz_api/model/transaction.py index 1b4342b..c73553e 100644 --- a/src/moneywiz_api/model/transaction.py +++ b/src/moneywiz_api/model/transaction.py @@ -10,6 +10,7 @@ from moneywiz_api.model.record import Record from moneywiz_api.schema_profile import SchemaProfile from moneywiz_api.types import ID +from moneywiz_api.validation import require_integer_identity, require_valid ABS_TOLERANCE = 0.001 @@ -29,7 +30,16 @@ class Transaction(Record, ABC): def __init__(self, row): super().__init__(row) - self.reconciled = row["ZRECONCILED"] == 1 + raw_reconciled = row["ZRECONCILED"] + require_integer_identity( + raw_reconciled, + "transaction reconciled state must be an uncoerced integer", + ) + require_valid( + raw_reconciled in (0, 1), + "transaction reconciled state must be zero or one", + ) + self.reconciled = raw_reconciled == 1 self.amount = RDH.get_decimal(row, "ZAMOUNT1") self.description = row["ZDESC2"] self.datetime = RDH.get_datetime(row, "ZDATE1") @@ -38,10 +48,12 @@ def __init__(self, row): # Fixes # Validate - assert self.reconciled is not None, self.as_dict() - assert self.amount is not None, self.as_dict() - assert self.description is not None, self.as_dict() - assert self.datetime is not None, self.as_dict() + require_valid(self.reconciled is not None, "transaction state is required") + require_valid(self.amount is not None, "transaction amount is required") + require_valid( + self.description is not None, "transaction description is required" + ) + require_valid(self.datetime is not None, "transaction date is required") # self.notes can be None @@ -79,17 +91,37 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None, self.as_dict() - assert self.amount is not None, self.as_dict() + require_valid(self.account is not None, "deposit account is required") + require_integer_identity( + self.account, "deposit account must be an uncoerced integer" + ) + require_valid(self.amount is not None, "deposit amount is required") # self.payee can be None - assert self.original_currency is not None, self.as_dict() - assert self.original_amount is not None, self.as_dict() + require_integer_identity( + self.payee, + "deposit payee must be an uncoerced integer", + optional=True, + ) + require_valid( + self.original_currency is not None, "deposit original currency is required" + ) + require_valid( + self.original_amount is not None, "deposit original amount is required" + ) - assert self.amount * self.original_amount > 0, self.as_dict() # Same sign + require_valid( + self.amount * self.original_amount > 0, + "deposit amounts must have the same sign", + ) if self.original_exchange_rate is not None: - assert self.amount == pytest.approx( - self.original_amount * self.original_exchange_rate, abs=ABS_TOLERANCE - ), self.as_dict() + require_valid( + self.amount + == pytest.approx( + self.original_amount * self.original_exchange_rate, + abs=ABS_TOLERANCE, + ), + "deposit exchange rate must match the amounts", + ) @dataclass @@ -134,15 +166,41 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None - assert self.from_investment_holding is not None - assert self.from_symbol - assert self.to_investment_holding is not None - assert self.to_symbol - assert self.from_number_of_shares <= 0 - assert self.to_number_of_shares >= 0 - assert self.original_fee is not None - assert self.original_fee_currency in [self.from_symbol, self.to_symbol] + require_valid(self.account is not None, "exchange account is required") + require_integer_identity( + self.account, "exchange account must be an uncoerced integer" + ) + require_valid( + self.from_investment_holding is not None, + "exchange source holding is required", + ) + require_integer_identity( + self.from_investment_holding, + "exchange source holding must be an uncoerced integer", + ) + require_valid(self.from_symbol, "exchange source symbol is required") + require_valid( + self.to_investment_holding is not None, + "exchange destination holding is required", + ) + require_integer_identity( + self.to_investment_holding, + "exchange destination holding must be an uncoerced integer", + ) + require_valid(self.to_symbol, "exchange destination symbol is required") + require_valid( + self.from_number_of_shares <= 0, + "exchange source quantity must not be positive", + ) + require_valid( + self.to_number_of_shares >= 0, + "exchange destination quantity must not be negative", + ) + require_valid(self.original_fee is not None, "exchange fee is required") + require_valid( + self.original_fee_currency in [self.from_symbol, self.to_symbol], + "exchange fee currency must match one holding", + ) @dataclass @@ -190,7 +248,9 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): ) self.price_per_share = RDH.get_profile_decimal( row, - schema_profile.price_per_share_column if schema_profile else None, + schema_profile.transaction_price_per_share_column + if schema_profile + else None, "ZPRICEPERSHARE1", "ZPRICEPERSHARE", ) @@ -202,24 +262,48 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): self.validate() def validate(self): - assert self.account is not None - assert self.amount is not None - assert self.amount <= 0 - assert self.fee is not None - assert self.fee >= 0 + require_valid(self.account is not None, "investment buy account is required") + require_integer_identity( + self.account, "investment buy account must be an uncoerced integer" + ) + require_valid(self.amount is not None, "investment buy amount is required") + require_valid(self.amount <= 0, "investment buy amount must not be positive") + require_valid(self.fee is not None, "investment buy fee is required") + require_valid(self.fee >= 0, "investment buy fee must not be negative") # Either tiny (close to 0) or positive - assert ( + require_valid( abs(self.fee) == pytest.approx(0, abs=ABS_TOLERANCE) - or self.fee > ABS_TOLERANCE + or self.fee > ABS_TOLERANCE, + "investment buy fee must be tiny or positive", + ) + require_valid( + self.investment_holding is not None, + "investment buy holding is required", + ) + require_integer_identity( + self.investment_holding, + "investment buy holding must be an uncoerced integer", + ) + require_valid( + self.number_of_shares is not None, + "investment buy quantity is required", + ) + require_valid( + self.number_of_shares > 0, "investment buy quantity must be positive" + ) + require_valid( + self.price_per_share is not None, + "investment buy share price is required", + ) + require_valid( + self.price_per_share >= 0, + "investment buy share price must not be negative", + ) + require_valid( + -(self.number_of_shares * self.price_per_share + self.fee) + == pytest.approx(self.amount, abs=ABS_TOLERANCE), + "investment buy total must match amount", ) - assert self.investment_holding is not None - assert self.number_of_shares is not None - assert self.number_of_shares > 0 - assert self.price_per_share is not None - assert self.price_per_share >= 0 - assert -( - self.number_of_shares * self.price_per_share + self.fee - ) == pytest.approx(self.amount, abs=ABS_TOLERANCE) @dataclass @@ -257,7 +341,9 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): ) self.price_per_share = RDH.get_profile_decimal( row, - schema_profile.price_per_share_column if schema_profile else None, + schema_profile.transaction_price_per_share_column + if schema_profile + else None, "ZPRICEPERSHARE1", "ZPRICEPERSHARE", ) @@ -269,25 +355,49 @@ def __init__(self, row, schema_profile: SchemaProfile | None = None): self.validate() def validate(self): - assert self.account is not None - assert self.amount is not None + require_valid(self.account is not None, "investment sell account is required") + require_integer_identity( + self.account, "investment sell account must be an uncoerced integer" + ) + require_valid(self.amount is not None, "investment sell amount is required") - assert self.fee is not None - assert self.fee >= 0 + require_valid(self.fee is not None, "investment sell fee is required") + require_valid(self.fee >= 0, "investment sell fee must not be negative") # Either tiny (close to 0) or positive - assert ( + require_valid( abs(self.fee) == pytest.approx(0, abs=ABS_TOLERANCE) - or self.fee > ABS_TOLERANCE + or self.fee > ABS_TOLERANCE, + "investment sell fee must be tiny or positive", ) - assert self.investment_holding is not None - assert self.number_of_shares is not None - assert self.number_of_shares > 0 - assert self.price_per_share is not None - assert self.price_per_share >= 0 - assert ( + require_valid( + self.investment_holding is not None, + "investment sell holding is required", + ) + require_integer_identity( + self.investment_holding, + "investment sell holding must be an uncoerced integer", + ) + require_valid( + self.number_of_shares is not None, + "investment sell quantity is required", + ) + require_valid( + self.number_of_shares > 0, "investment sell quantity must be positive" + ) + require_valid( + self.price_per_share is not None, + "investment sell share price is required", + ) + require_valid( + self.price_per_share >= 0, + "investment sell share price must not be negative", + ) + require_valid( self.number_of_shares * self.price_per_share - self.fee - ) == pytest.approx(self.amount, abs=ABS_TOLERANCE) + == pytest.approx(self.amount, abs=ABS_TOLERANCE), + "investment sell total must match amount", + ) @dataclass @@ -313,10 +423,14 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None - assert ( + require_valid(self.account is not None, "reconcile account is required") + require_integer_identity( + self.account, "reconcile account must be an uncoerced integer" + ) + require_valid( self.reconcile_amount is not None - or self.reconcile_number_of_shares is not None + or self.reconcile_number_of_shares is not None, + "reconcile amount or quantity is required", ) @@ -355,17 +469,36 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None - assert self.amount is not None - assert self.amount > 0 + require_valid(self.account is not None, "refund account is required") + require_integer_identity( + self.account, "refund account must be an uncoerced integer" + ) + require_integer_identity( + self.payee, + "refund payee must be an uncoerced integer", + optional=True, + ) + require_valid(self.amount is not None, "refund amount is required") + require_valid(self.amount > 0, "refund amount must be positive") - assert self.original_currency is not None - assert self.original_amount is not None - assert self.original_amount > 0 + require_valid( + self.original_currency is not None, "refund original currency is required" + ) + require_valid( + self.original_amount is not None, "refund original amount is required" + ) + require_valid( + self.original_amount > 0, "refund original amount must be positive" + ) if self.original_exchange_rate is not None: - assert self.amount == pytest.approx( - self.original_amount * self.original_exchange_rate, abs=ABS_TOLERANCE + require_valid( + self.amount + == pytest.approx( + self.original_amount * self.original_exchange_rate, + abs=ABS_TOLERANCE, + ), + "refund exchange rate must match the amounts", ) @@ -433,28 +566,73 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None - assert self.amount is not None - assert self.amount > 0 - assert self.sender_account is not None - assert self.sender_transaction is not None - assert self.original_amount is not None - assert self.original_amount > 0 - assert self.original_currency is not None - assert self.sender_amount is not None - assert self.sender_amount <= 0 - assert self.sender_currency is not None + require_valid(self.account is not None, "transfer deposit account is required") + require_integer_identity( + self.account, "transfer deposit account must be an uncoerced integer" + ) + require_valid(self.amount is not None, "transfer deposit amount is required") + require_valid(self.amount > 0, "transfer deposit amount must be positive") + require_valid( + self.sender_account is not None, + "transfer deposit sender account is required", + ) + require_integer_identity( + self.sender_account, + "transfer deposit sender account must be an uncoerced integer", + ) + require_valid( + self.sender_transaction is not None, + "transfer deposit sender transaction is required", + ) + require_integer_identity( + self.sender_transaction, + "transfer deposit sender transaction must be an uncoerced integer", + ) + require_valid( + self.original_amount is not None, + "transfer deposit original amount is required", + ) + require_valid( + self.original_amount > 0, + "transfer deposit original amount must be positive", + ) + require_valid( + self.original_currency is not None, + "transfer deposit original currency is required", + ) + require_valid( + self.sender_amount is not None, + "transfer deposit sender amount is required", + ) + require_valid( + self.sender_amount <= 0, + "transfer deposit sender amount must not be positive", + ) + require_valid( + self.sender_currency is not None, + "transfer deposit sender currency is required", + ) if self.original_fee is not None and self.original_fee != 0: - assert self.original_fee_currency is not None + require_valid( + self.original_fee_currency is not None, + "transfer deposit fee currency is required", + ) - assert self.original_exchange_rate is not None + require_valid( + self.original_exchange_rate is not None, + "transfer deposit exchange rate is required", + ) # assert self.amount == self.original_amount # original_amount could be different with amount ZCURRENCYEXCHANGERATE is playing up - assert self.original_amount == pytest.approx( - -self.sender_amount * self.original_exchange_rate - - (self.original_fee or 0), - abs=ABS_TOLERANCE, + require_valid( + self.original_amount + == pytest.approx( + -self.sender_amount * self.original_exchange_rate + - (self.original_fee or 0), + abs=ABS_TOLERANCE, + ), + "transfer deposit exchange rate must match the amounts", ) @@ -516,27 +694,77 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None - assert self.amount is not None - assert self.amount < 0 - assert self.recipient_account is not None - assert self.recipient_transaction is not None - assert self.original_amount is not None - assert self.original_amount < 0 - assert self.original_currency is not None - assert self.recipient_amount is not None - assert self.recipient_amount > 0 - assert self.recipient_currency is not None + require_valid( + self.account is not None, "transfer withdrawal account is required" + ) + require_integer_identity( + self.account, "transfer withdrawal account must be an uncoerced integer" + ) + require_valid(self.amount is not None, "transfer withdrawal amount is required") + require_valid(self.amount < 0, "transfer withdrawal amount must be negative") + require_valid( + self.recipient_account is not None, + "transfer withdrawal recipient account is required", + ) + require_integer_identity( + self.recipient_account, + "transfer withdrawal recipient account must be an uncoerced integer", + ) + require_valid( + self.recipient_transaction is not None, + "transfer withdrawal recipient transaction is required", + ) + require_integer_identity( + self.recipient_transaction, + "transfer withdrawal recipient transaction must be an uncoerced integer", + ) + require_valid( + self.original_amount is not None, + "transfer withdrawal original amount is required", + ) + require_valid( + self.original_amount < 0, + "transfer withdrawal original amount must be negative", + ) + require_valid( + self.original_currency is not None, + "transfer withdrawal original currency is required", + ) + require_valid( + self.recipient_amount is not None, + "transfer withdrawal recipient amount is required", + ) + require_valid( + self.recipient_amount > 0, + "transfer withdrawal recipient amount must be positive", + ) + require_valid( + self.recipient_currency is not None, + "transfer withdrawal recipient currency is required", + ) if self.original_fee is not None and self.original_fee != 0: - assert self.original_fee_currency is not None + require_valid( + self.original_fee_currency is not None, + "transfer withdrawal fee currency is required", + ) - assert self.original_exchange_rate is not None + require_valid( + self.original_exchange_rate is not None, + "transfer withdrawal exchange rate is required", + ) - assert self.amount == self.original_amount - assert self.amount == pytest.approx( - -self.recipient_amount / self.original_exchange_rate, - abs=ABS_TOLERANCE, + require_valid( + self.amount == self.original_amount, + "transfer withdrawal amount must match original amount", + ) + require_valid( + self.amount + == pytest.approx( + -self.recipient_amount / self.original_exchange_rate, + abs=ABS_TOLERANCE, + ), + "transfer withdrawal exchange rate must match the amounts", ) @@ -578,15 +806,36 @@ def __init__(self, row): self.validate() def validate(self): - assert self.account is not None - assert self.amount is not None + require_valid(self.account is not None, "withdrawal account is required") + require_integer_identity( + self.account, "withdrawal account must be an uncoerced integer" + ) + require_valid(self.amount is not None, "withdrawal amount is required") # self.payee can be None - assert self.original_currency is not None - assert self.original_amount is not None + require_integer_identity( + self.payee, + "withdrawal payee must be an uncoerced integer", + optional=True, + ) + require_valid( + self.original_currency is not None, + "withdrawal original currency is required", + ) + require_valid( + self.original_amount is not None, "withdrawal original amount is required" + ) - assert self.amount * self.original_amount > 0 + require_valid( + self.amount * self.original_amount > 0, + "withdrawal amounts must have the same sign", + ) if self.original_exchange_rate is not None: - assert self.amount == pytest.approx( - self.original_amount * self.original_exchange_rate, abs=ABS_TOLERANCE + require_valid( + self.amount + == pytest.approx( + self.original_amount * self.original_exchange_rate, + abs=ABS_TOLERANCE, + ), + "withdrawal exchange rate must match the amounts", ) diff --git a/src/moneywiz_api/moneywiz_api.py b/src/moneywiz_api/moneywiz_api.py index 3195eb0..41fb65d 100644 --- a/src/moneywiz_api/moneywiz_api.py +++ b/src/moneywiz_api/moneywiz_api.py @@ -1,5 +1,8 @@ +"""Public MoneyWiz read API.""" + +from dataclasses import asdict from pathlib import Path -import logging +from typing import Iterable from moneywiz_api.database_accessor import DatabaseAccessor from moneywiz_api.managers.account_manager import AccountManager @@ -10,12 +13,26 @@ from moneywiz_api.managers.payee_manager import PayeeManager from moneywiz_api.managers.transaction_manager import TransactionManager from moneywiz_api.managers.tag_manager import TagManager - -logger = logging.getLogger(__name__) +from moneywiz_api.read_result import ApiCompleteness, ReadSnapshot, json_safe class MoneywizApi: - def __init__(self, db_file: Path): + """Read a MoneyWiz database with explicit completeness evidence.""" + + MANAGER_NAMES = ( + "accounts", + "payees", + "categories", + "transactions", + "investment_holdings", + "tags", + ) + + def __init__( + self, + db_file: Path, + managers: Iterable[str] | None = None, + ): self.accessor = DatabaseAccessor(db_file) self.account_manager = AccountManager() self.payee_manager = PayeeManager() @@ -23,13 +40,91 @@ def __init__(self, db_file: Path): self.transaction_manager = TransactionManager() self.investment_holding_manager = InvestmentHoldingManager() self.tag_manager = TagManager() + self._managers = { + "accounts": self.account_manager, + "payees": self.payee_manager, + "categories": self.category_manager, + "transactions": self.transaction_manager, + "investment_holdings": self.investment_holding_manager, + "tags": self.tag_manager, + } + self._loaded_managers: set[str] = set() + + try: + self.load(managers) + except BaseException: + self.close() + raise + + def _manager_names(self, managers: Iterable[str] | None) -> tuple[str, ...]: + if managers is None: + return self.MANAGER_NAMES + if isinstance(managers, str): + raise TypeError("managers must be an iterable of manager names") + names = tuple(dict.fromkeys(managers)) + unknown = [name for name in names if name not in self._managers] + if unknown: + raise ValueError(f"unknown manager names: {', '.join(unknown)}") + return names + + def load(self, managers: Iterable[str] | None = None) -> ApiCompleteness: + """Atomically reload requested managers and every retained manager.""" + requested = self._manager_names(managers) + names = tuple( + name + for name in self.MANAGER_NAMES + if name in self._loaded_managers or name in requested + ) + staged = {name: type(self._managers[name])() for name in names} + with self.accessor.read_transaction(): + for name in names: + staged[name].load(self.accessor) + for name in names: + self._managers[name]._adopt_loaded_state(staged[name]) + self._loaded_managers.update(names) + return self.completeness(names) + + def completeness(self, managers: Iterable[str] | None = None) -> ApiCompleteness: + """Return completeness evidence without reloading the database.""" + names = ( + tuple(name for name in self.MANAGER_NAMES if name in self._loaded_managers) + if managers is None + else self._manager_names(managers) + ) + unloaded = [name for name in names if name not in self._loaded_managers] + if unloaded: + raise ValueError(f"manager not loaded: {', '.join(unloaded)}") + return ApiCompleteness( + managers={name: self._managers[name].load_report for name in names} + ) + + def snapshot(self, managers: Iterable[str] | None = None) -> ReadSnapshot: + """Return loaded records and diagnostics in a JSON-safe structure.""" + names = ( + tuple(name for name in self.MANAGER_NAMES if name in self._loaded_managers) + if managers is None + else self._manager_names(managers) + ) + completeness = self.completeness(names) + records = { + name: tuple( + json_safe(record.as_dict()) + for record in self._managers[name].records().values() + ) + for name in names + } + return ReadSnapshot( + records=records, + completeness=completeness, + schema_profile=json_safe(asdict(self.accessor.schema_profile)), + ) + + def close(self) -> None: + """Close the underlying read-only database connection.""" + self.accessor.close() - self.load() + def __enter__(self): + return self - def load(self): - self.account_manager.load(self.accessor) - self.payee_manager.load(self.accessor) - self.category_manager.load(self.accessor) - self.transaction_manager.load(self.accessor) - self.investment_holding_manager.load(self.accessor) - self.tag_manager.load(self.accessor) + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.close() diff --git a/src/moneywiz_api/read_result.py b/src/moneywiz_api/read_result.py new file mode 100644 index 0000000..a85513c --- /dev/null +++ b/src/moneywiz_api/read_result.py @@ -0,0 +1,313 @@ +"""Structured results for bounded MoneyWiz reads.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import asdict, dataclass, field, is_dataclass +from datetime import date, datetime +from decimal import Decimal +from enum import Enum +import math +from pathlib import Path +from typing import Any, Mapping + +from moneywiz_api.types import ID + + +class _FrozenMapping(dict): + """Recursively own published mapping values and refuse ordinary mutation.""" + + def __init__(self, values=()): + if getattr(self, "_initialized", False): + raise TypeError("published result mappings are immutable") + dict.__init__( + self, + ((key, _freeze_value(value)) for key, value in dict(values).items()), + ) + self._initialized = True + + @staticmethod + def _immutable(*_args, **_kwargs): + raise TypeError("published result mappings are immutable") + + __setitem__ = _immutable + __delitem__ = _immutable + __ior__ = _immutable + clear = _immutable + pop = _immutable + popitem = _immutable + setdefault = _immutable + update = _immutable + + def __deepcopy__(self, memo): + return type(self)( + (deepcopy(key, memo), deepcopy(value, memo)) for key, value in self.items() + ) + + +def _freeze_value(value: Any) -> Any: + """Recursively freeze mappings and sequences at a result boundary.""" + if isinstance(value, _FrozenMapping): + return value + if isinstance(value, Mapping): + return _FrozenMapping(value) + if isinstance(value, (tuple, list)): + return tuple(_freeze_value(item) for item in value) + return value + + +class LoadErrorKind(str, Enum): + """Bounded classifications for records omitted from a read.""" + + MISSING_FIELD = "missing_field" + INVALID_VALUE = "invalid_value" + VALIDATION = "validation" + DUPLICATE_ID = "duplicate_id" + DUPLICATE_GID = "duplicate_gid" + UNKNOWN_ENTITY = "unknown_entity" + CONSTRUCTION = "construction" + + +class RelationshipStorage(str, Enum): + """Whether relationship storage is available and understood.""" + + PRESENT = "present" + ABSENT = "absent" + UNKNOWN = "unknown" + + +@dataclass(frozen=True) +class SkippedRecord: + """Identity-only diagnostic for one source row that was not parsed.""" + + record_id: ID | str | None + entity: str | None + error: LoadErrorKind + exception_type: str + + def as_dict(self) -> dict[str, Any]: + return { + "record_id": self.record_id, + "entity": self.entity, + "error": self.error.value, + "exception_type": self.exception_type, + } + + +@dataclass(frozen=True) +class RelationshipLoadReport: + """Completeness evidence for one relationship storage layout.""" + + storage: RelationshipStorage + storage_name: str | None = None + source_ids: tuple[ID | str | None, ...] = () + parsed_ids: tuple[ID | str, ...] = () + skipped: tuple[SkippedRecord, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "source_ids", tuple(self.source_ids)) + object.__setattr__(self, "parsed_ids", tuple(self.parsed_ids)) + object.__setattr__(self, "skipped", tuple(self.skipped)) + + @property + def source_count(self) -> int: + return len(self.source_ids) + + @property + def parsed_count(self) -> int: + return len(self.parsed_ids) + + @property + def complete(self) -> bool: + return ( + self.storage != RelationshipStorage.UNKNOWN + and self.source_count == self.parsed_count + and not self.skipped + ) + + @property + def status(self) -> str: + if self.storage == RelationshipStorage.ABSENT: + return "absent" + if self.complete: + return "complete" + if self.parsed_count: + return "partial" + return "error" + + def as_dict(self) -> dict[str, Any]: + return { + "status": self.status, + "complete": self.complete, + "storage": self.storage.value, + "storage_name": self.storage_name, + "source_count": self.source_count, + "parsed_count": self.parsed_count, + "source_ids": list(self.source_ids), + "parsed_ids": list(self.parsed_ids), + "skipped": [item.as_dict() for item in self.skipped], + } + + +@dataclass(frozen=True) +class ManagerLoadReport: + """Completeness evidence for one manager load.""" + + source_ids: tuple[ID | None, ...] = () + parsed_ids: tuple[ID, ...] = () + skipped: tuple[SkippedRecord, ...] = () + relationships: Mapping[str, RelationshipLoadReport] = field(default_factory=dict) + observed: bool = True + + def __post_init__(self) -> None: + object.__setattr__(self, "source_ids", tuple(self.source_ids)) + object.__setattr__(self, "parsed_ids", tuple(self.parsed_ids)) + object.__setattr__(self, "skipped", tuple(self.skipped)) + object.__setattr__(self, "relationships", _FrozenMapping(self.relationships)) + + @classmethod + def unloaded(cls) -> "ManagerLoadReport": + """Return evidence that no complete manager observation was published.""" + return cls(observed=False) + + @property + def source_count(self) -> int: + return len(self.source_ids) + + @property + def parsed_count(self) -> int: + return len(self.parsed_ids) + + @property + def complete(self) -> bool: + return ( + self.observed + and self.source_count == self.parsed_count + and not self.skipped + and all(report.complete for report in self.relationships.values()) + ) + + @property + def status(self) -> str: + if not self.observed: + return "unloaded" + if self.complete: + return "complete" + if self.parsed_count or any( + report.parsed_count for report in self.relationships.values() + ): + return "partial" + return "error" + + def as_dict(self) -> dict[str, Any]: + return { + "status": self.status, + "complete": self.complete, + "source_count": self.source_count, + "parsed_count": self.parsed_count, + "source_ids": list(self.source_ids), + "parsed_ids": list(self.parsed_ids), + "skipped": [item.as_dict() for item in self.skipped], + "relationships": { + name: report.as_dict() for name, report in self.relationships.items() + }, + } + + +@dataclass(frozen=True) +class ApiCompleteness: + """Aggregate completeness for the managers included in a read.""" + + managers: Mapping[str, ManagerLoadReport] + + def __post_init__(self) -> None: + object.__setattr__(self, "managers", _FrozenMapping(self.managers)) + + @property + def complete(self) -> bool: + return bool(self.managers) and all( + report.complete for report in self.managers.values() + ) + + @property + def status(self) -> str: + if self.complete: + return "complete" + if self.managers and all( + report.status == "error" for report in self.managers.values() + ): + return "error" + return "partial" + + def as_dict(self) -> dict[str, Any]: + return { + "status": self.status, + "complete": self.complete, + "managers": { + name: report.as_dict() for name, report in self.managers.items() + }, + } + + +@dataclass(frozen=True) +class ReadSnapshot: + """JSON-safe records and completeness metadata from one API instance.""" + + records: Mapping[str, tuple[Mapping[str, Any], ...]] + completeness: ApiCompleteness + schema_profile: Mapping[str, Any] + + def __post_init__(self) -> None: + records = json_safe(self.records) + schema_profile = json_safe(self.schema_profile) + json_safe(self.completeness) + object.__setattr__(self, "records", _FrozenMapping(records)) + object.__setattr__(self, "schema_profile", _FrozenMapping(schema_profile)) + + def as_dict(self) -> dict[str, Any]: + return { + "schema_profile": json_safe(self.schema_profile), + "completeness": self.completeness.as_dict(), + "records": json_safe(self.records), + } + + +def json_safe(value: Any) -> Any: + """Convert public model values into deterministic JSON-compatible values.""" + if value is None or type(value) in (bool, int, str): + return value + if type(value) is float: + if not math.isfinite(value): + raise TypeError("snapshot float values must be finite") + return value + if isinstance(value, Decimal): + return str(value) + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, Path): + return str(value) + if isinstance(value, Enum): + return json_safe(value.value) + if is_dataclass(value) and not isinstance(value, type): + return json_safe(asdict(value)) + if isinstance(value, Mapping): + converted = {} + for key, item in value.items(): + normalized_key = _json_safe_key(key) + if normalized_key in converted: + raise TypeError("snapshot mapping keys collide after normalization") + converted[normalized_key] = json_safe(item) + return converted + if isinstance(value, (tuple, list)): + return [json_safe(item) for item in value] + raise TypeError("snapshot contains an unsupported value") + + +def _json_safe_key(value: Any) -> str: + if value is None or type(value) in (bool, int, str): + return str(value) + if type(value) is float: + if not math.isfinite(value): + raise TypeError("snapshot mapping keys must be finite") + return str(value) + raise TypeError("snapshot contains an unsupported mapping key") diff --git a/src/moneywiz_api/schema_profile.py b/src/moneywiz_api/schema_profile.py index fcfb40c..c27145f 100644 --- a/src/moneywiz_api/schema_profile.py +++ b/src/moneywiz_api/schema_profile.py @@ -13,7 +13,8 @@ class SchemaProfile: profile_id: str holding_number_of_shares_column: str | None transaction_number_of_shares_column: str | None - price_per_share_column: str | None + holding_price_per_share_column: str | None + transaction_price_per_share_column: str | None @property def is_known(self) -> bool: @@ -30,37 +31,42 @@ def detect_schema_profile(connection: sqlite3.Connection) -> SchemaProfile: has_suffixed_price = "ZPRICEPERSHARE1" in columns has_unsuffixed_price = "ZPRICEPERSHARE" in columns - if ( - has_suffixed_shares - and has_unsuffixed_shares - and has_suffixed_price - and has_unsuffixed_price + if not (has_suffixed_shares or has_unsuffixed_shares) or not ( + has_suffixed_price or has_unsuffixed_price ): profile_id = "unknown" holding_number_of_shares_column = None transaction_number_of_shares_column = None - price_per_share_column = None - elif ( - has_suffixed_shares - and has_unsuffixed_shares - and (has_suffixed_price or has_unsuffixed_price) + holding_price_per_share_column = None + transaction_price_per_share_column = None + elif (has_suffixed_shares and has_unsuffixed_shares) or ( + has_suffixed_price and has_unsuffixed_price ): profile_id = "mixed-investment-columns" - holding_number_of_shares_column = "ZNUMBEROFSHARES" - transaction_number_of_shares_column = "ZNUMBEROFSHARES1" - price_per_share_column = ( + holding_number_of_shares_column = ( + "ZNUMBEROFSHARES" if has_unsuffixed_shares else "ZNUMBEROFSHARES1" + ) + transaction_number_of_shares_column = ( + "ZNUMBEROFSHARES1" if has_suffixed_shares else "ZNUMBEROFSHARES" + ) + holding_price_per_share_column = ( + "ZPRICEPERSHARE" if has_unsuffixed_price else "ZPRICEPERSHARE1" + ) + transaction_price_per_share_column = ( "ZPRICEPERSHARE1" if has_suffixed_price else "ZPRICEPERSHARE" ) elif has_suffixed_shares and has_suffixed_price: profile_id = "suffixed-investment-columns" holding_number_of_shares_column = "ZNUMBEROFSHARES1" transaction_number_of_shares_column = "ZNUMBEROFSHARES1" - price_per_share_column = "ZPRICEPERSHARE1" + holding_price_per_share_column = "ZPRICEPERSHARE1" + transaction_price_per_share_column = "ZPRICEPERSHARE1" elif has_unsuffixed_shares and has_unsuffixed_price: profile_id = "unsuffixed-investment-columns" holding_number_of_shares_column = "ZNUMBEROFSHARES" transaction_number_of_shares_column = "ZNUMBEROFSHARES" - price_per_share_column = "ZPRICEPERSHARE" + holding_price_per_share_column = "ZPRICEPERSHARE" + transaction_price_per_share_column = "ZPRICEPERSHARE" elif (has_suffixed_shares or has_unsuffixed_shares) and ( has_suffixed_price or has_unsuffixed_price ): @@ -71,18 +77,19 @@ def detect_schema_profile(connection: sqlite3.Connection) -> SchemaProfile: transaction_number_of_shares_column = ( "ZNUMBEROFSHARES1" if has_suffixed_shares else "ZNUMBEROFSHARES" ) - price_per_share_column = ( + holding_price_per_share_column = ( + "ZPRICEPERSHARE" if has_unsuffixed_price else "ZPRICEPERSHARE1" + ) + transaction_price_per_share_column = ( "ZPRICEPERSHARE1" if has_suffixed_price else "ZPRICEPERSHARE" ) - else: - profile_id = "unknown" - holding_number_of_shares_column = None - transaction_number_of_shares_column = None - price_per_share_column = None + else: # pragma: no cover - exhaustive boolean cases above + raise AssertionError("unreachable investment schema profile") return SchemaProfile( profile_id=profile_id, holding_number_of_shares_column=holding_number_of_shares_column, transaction_number_of_shares_column=transaction_number_of_shares_column, - price_per_share_column=price_per_share_column, + holding_price_per_share_column=holding_price_per_share_column, + transaction_price_per_share_column=transaction_price_per_share_column, ) diff --git a/src/moneywiz_api/utils.py b/src/moneywiz_api/utils.py index 1d1d8ed..294e2eb 100644 --- a/src/moneywiz_api/utils.py +++ b/src/moneywiz_api/utils.py @@ -1,10 +1,11 @@ -from datetime import datetime +from datetime import datetime, timezone -_CUTOFF = datetime(2001, 1, 1, 0, 0, 0).timestamp() +_APPLE_EPOCH_TIMESTAMP = datetime(2001, 1, 1, tzinfo=timezone.utc).timestamp() def get_datetime(date: float) -> datetime: - return datetime.fromtimestamp(date + _CUTOFF) + """Convert an absolute Core Data timestamp to a naive local datetime.""" + return datetime.fromtimestamp(date + _APPLE_EPOCH_TIMESTAMP) def get_date_iso(date: float) -> str: @@ -12,4 +13,5 @@ def get_date_iso(date: float) -> str: def get_date(dt: datetime) -> float: - return dt.timestamp() - _CUTOFF + """Convert an aware instant or naive local datetime to Core Data seconds.""" + return dt.timestamp() - _APPLE_EPOCH_TIMESTAMP diff --git a/src/moneywiz_api/validation.py b/src/moneywiz_api/validation.py new file mode 100644 index 0000000..3705ae1 --- /dev/null +++ b/src/moneywiz_api/validation.py @@ -0,0 +1,22 @@ +def require_valid(condition: object, message: str) -> None: + """Raise the historical validation exception even under optimized Python.""" + if not condition: + raise AssertionError(message) + + +def require_integer_identity( + value: object, message: str, *, optional: bool = False +) -> None: + """Require an uncoerced integer identity, optionally allowing ``None``.""" + if optional and value is None: + return + if type(value) is not int: + raise AssertionError(message) + + +def require_text(value: object, message: str, *, optional: bool = False) -> None: + """Require an uncoerced string, optionally allowing None.""" + if optional and value is None: + return + if type(value) is not str: + raise AssertionError(message) diff --git a/tests/integration/test_read_completeness_integration.py b/tests/integration/test_read_completeness_integration.py new file mode 100644 index 0000000..4ca1e8c --- /dev/null +++ b/tests/integration/test_read_completeness_integration.py @@ -0,0 +1,10 @@ +from conftest import moneywizApi + + +def test_loaded_managers_expose_accounted_source_rows() -> None: + completeness = moneywizApi.completeness() + + for report in completeness.managers.values(): + assert report.source_count == report.parsed_count + len(report.skipped) + assert len(report.source_ids) == report.source_count + assert len(report.parsed_ids) == report.parsed_count diff --git a/tests/unit/accessor_test_support.py b/tests/unit/accessor_test_support.py new file mode 100644 index 0000000..f68cd36 --- /dev/null +++ b/tests/unit/accessor_test_support.py @@ -0,0 +1,37 @@ +import sqlite3 + +from moneywiz_api.database_accessor import DatabaseAccessor + + +def initialized_memory_accessor(rows, entity_metadata): + """Build a guarded accessor after assembling its complete synthetic schema.""" + connection = sqlite3.connect(":memory:") + + def dict_factory(cursor, values): + return { + description[0]: values[index] + for index, description in enumerate(cursor.description) + } + + connection.row_factory = dict_factory + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, 0)", entity_metadata + ) + columns = tuple(dict.fromkeys(key for row in rows for key in row)) + definitions = ", ".join(f'"{column}"' for column in columns) + connection.execute(f"CREATE TABLE ZSYNCOBJECT ({definitions})") + placeholders = ", ".join("?" for _ in columns) + connection.executemany( + f"INSERT INTO ZSYNCOBJECT VALUES ({placeholders})", + [tuple(row.get(column) for column in columns) for row in rows], + ) + connection.commit() + + accessor = DatabaseAccessor.__new__(DatabaseAccessor) + accessor._con = connection + accessor._initialize_schema_cache() + return accessor diff --git a/tests/unit/optimized_validation_probe.py b/tests/unit/optimized_validation_probe.py new file mode 100644 index 0000000..8ab4774 --- /dev/null +++ b/tests/unit/optimized_validation_probe.py @@ -0,0 +1,625 @@ +"""Emit runtime-validation behavior for normal and optimized Python processes.""" + +from __future__ import annotations + +import json +import sqlite3 +import sys +import tempfile +from contextlib import contextmanager +from decimal import Decimal +from pathlib import Path + +from moneywiz_api.database_accessor import DatabaseAccessor +from moneywiz_api.managers.category_manager import CategoryManager +from moneywiz_api.managers.payee_manager import PayeeManager +from moneywiz_api.managers.transaction_manager import TransactionManager +from moneywiz_api.model.account import ( + Account, + BankChequeAccount, + BankSavingAccount, + CashAccount, + CreditCardAccount, + ForexAccount, + InvestmentAccount, + LoanAccount, +) +from moneywiz_api.model.category import Category +from moneywiz_api.model.investment_holding import InvestmentHolding +from moneywiz_api.model.payee import Payee +from moneywiz_api.model.raw_data_handler import RawDataHandler +from moneywiz_api.model.record import Record +from moneywiz_api.model.tag import Tag +from moneywiz_api.model.transaction import ( + DepositTransaction, + InvestmentBuyTransaction, + TransferWithdrawTransaction, + WithdrawTransaction, +) +from moneywiz_api.schema_profile import SchemaProfile +from accessor_test_support import initialized_memory_accessor + + +PROFILE = SchemaProfile( + "unsuffixed-investment-columns", + "ZNUMBEROFSHARES", + "ZNUMBEROFSHARES", + "ZPRICEPERSHARE", + "ZPRICEPERSHARE", +) + + +def normalize(value): + if isinstance(value, Decimal): + return str(value) + if hasattr(value, "isoformat"): + return value.isoformat() + if isinstance(value, dict): + return {str(key): normalize(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [normalize(item) for item in value] + return value + + +def capture(operation): + try: + value = operation() + except Exception as exc: + return { + "status": "error", + "type": type(exc).__name__, + "message": str(exc), + } + return {"status": "ok", "value": normalize(value)} + + +def common_row(**overrides): + row = { + "Z_ENT": 1, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": "record-1", + "Z_PK": 1, + } + row.update(overrides) + return row + + +def account_row(**overrides): + row = { + **common_row(Z_ENT=13, ZGID="account-1"), + "ZDISPLAYORDER": 1, + "ZGROUPID": 1, + "ZNAME": "Card", + "ZCURRENCYNAME": "EUR", + "ZOPENINGBALANCE": 0.0, + "ZINFO": None, + "ZUSER": 1, + "ZSTATEMENTENDDAY": 15, + } + row.update(overrides) + return row + + +def transaction_row(**overrides): + row = { + **common_row(Z_ENT=37, ZGID="transaction-1"), + "ZRECONCILED": 0, + "ZAMOUNT1": 20.0, + "ZDESC2": "Transaction", + "ZDATE1": 0.0, + "ZNOTES1": None, + "ZACCOUNT2": 1, + "ZPAYEE2": None, + "ZORIGINALCURRENCY": "EUR", + "ZORIGINALAMOUNT": 10.0, + "ZORIGINALEXCHANGERATE": 2.0, + } + row.update(overrides) + return row + + +def holding_row(**overrides): + row = { + **common_row(Z_ENT=24, ZGID="holding-1"), + "ZINVESTMENTACCOUNT": 1, + "ZOPENNINGNUMBEROFSHARES": None, + "ZNUMBEROFSHARES": 2.0, + "ZPRICEPERSHARE": 10.0, + "ZSYMBOL": "ACME", + "ZHOLDINGTYPE": None, + "ZDESC": "Acme", + "ZISPRICEPERSHAREAVAILABLEONLINE": 0, + "ZINVESTMENTOBJECTTYPE": 0, + "ZCOSTBASISOFMISSINGOBSHARES": 0.0, + } + row.update(overrides) + return row + + +def buy_row(**overrides): + row = { + **transaction_row(Z_ENT=40, ZGID="buy-1", ZAMOUNT1=-21.0), + "ZFEE2": 1.0, + "ZINVESTMENTHOLDING": 2, + "ZNUMBEROFSHARES": 2.0, + "ZPRICEPERSHARE": 10.0, + } + row.update(overrides) + return row + + +def transfer_withdraw_row(**overrides): + row = { + **transaction_row(Z_ENT=46, ZGID="transfer-withdraw-1", ZAMOUNT1=-10.0), + "ZRECIPIENTACCOUNT1": 2, + "ZRECIPIENTTRANSACTION": 3, + "ZORIGINALAMOUNT": 0.0, + "ZORIGINALRECIPIENTAMOUNT": 10.0, + "ZORIGINALRECIPIENTCURRENCY": "USD", + "ZORIGINALFEE": None, + "ZORIGINALFEECURRENCY": None, + "ZORIGINALEXCHANGERATE": 0.0, + } + row.update(overrides) + return row + + +def accessor_for(row): + return initialized_memory_accessor([row], [(13, "CreditCardAccount", 0)]) + + +class PayeeAccessor: + @contextmanager + def read_transaction(self): + yield + + def descendant_typenames(self, _roots): + return ["Payee"] + + def query_objects(self, _typenames): + return [ + { + **common_row(Z_ENT=28, ZGID="payee-valid", Z_PK=10), + "ZNAME5": "A", + "ZUSER7": 1, + }, + { + **common_row(Z_ENT=28, ZGID="payee-invalid", Z_PK=11), + "ZNAME5": "B", + "ZUSER7": None, + }, + { + **common_row( + Z_ENT=28, + ZGID="payee-invalid-identity", + Z_PK="PRIVATE_PAYLOAD", + ), + "ZNAME5": "C", + "ZUSER7": 1, + }, + ] + + def typename_for(self, _ent_id): + return "Payee" + + +class CategoryAccessor: + def __init__(self, rows): + self.rows = rows + + @contextmanager + def read_transaction(self): + yield + + def descendant_typenames(self, _roots): + return ["Category"] + + def query_objects(self, _typenames): + return self.rows + + def typename_for(self, _ent_id): + return "Category" + + +def direct_account_probe(): + constructors = ( + Account, + BankChequeAccount, + BankSavingAccount, + CashAccount, + CreditCardAccount, + LoanAccount, + InvestmentAccount, + ForexAccount, + ) + return { + constructor.__name__: { + "valid": capture( + lambda constructor=constructor: constructor(account_row()).info + ), + "invalid_name": capture( + lambda constructor=constructor: constructor(account_row(ZNAME=None)) + ), + "invalid_statement_day": ( + capture( + lambda constructor=constructor: constructor( + account_row(ZSTATEMENTENDDAY=None) + ) + ) + if issubclass(constructor, CreditCardAccount) + else None + ), + } + for constructor in constructors + } + + +def category_type_probe(): + def row(value=1, **overrides): + category = { + **common_row(Z_ENT=19), + "ZNAME2": "Category", + "ZPARENTCATEGORY": None, + "ZTYPE2": value, + "ZUSER3": 1, + } + category.update(overrides) + return category + + manager_reports = {} + for label, value in ( + ("null", None), + ("zero", 0), + ("integer", 3), + ("text", "PRIVATE_PAYLOAD"), + ): + report = CategoryManager().load(CategoryAccessor([row(value)])) + manager_reports[label] = report.as_dict() + missing = row() + del missing["ZTYPE2"] + missing_report = CategoryManager().load(CategoryAccessor([missing])) + owner_report = CategoryManager().load(CategoryAccessor([row(ZUSER3=None)])) + + def unexpected_constructor(_row): + raise RuntimeError("synthetic unexpected constructor failure") + + class UnexpectedCategoryManager(CategoryManager): + @property + def ents(self): + return {"Category": unexpected_constructor} + + unexpected_report = UnexpectedCategoryManager().load(CategoryAccessor([row()])) + return { + "supported": { + "1": capture(lambda: Category(row(1)).type), + "2": capture(lambda: Category(row(2)).type), + }, + "unsupported": manager_reports, + "missing": missing_report.as_dict(), + "invalid_owner": owner_report.as_dict(), + "unexpected": unexpected_report.as_dict(), + } + + +def named_entities_probe(): + rows = { + "payee_valid": ( + Payee, + {**common_row(Z_ENT=28), "ZNAME5": "Payee", "ZUSER7": 1}, + ), + "payee_invalid": ( + Payee, + {**common_row(Z_ENT=28), "ZNAME5": "Payee", "ZUSER7": None}, + ), + "payee_blob_name": ( + Payee, + {**common_row(Z_ENT=28), "ZNAME5": b"PRIVATE_PAYLOAD", "ZUSER7": 1}, + ), + "tag_valid": ( + Tag, + {**common_row(Z_ENT=35), "ZNAME6": "Tag", "ZUSER8": 1}, + ), + "tag_invalid": ( + Tag, + {**common_row(Z_ENT=35), "ZNAME6": "Tag", "ZUSER8": None}, + ), + "tag_blob_name": ( + Tag, + {**common_row(Z_ENT=35), "ZNAME6": b"PRIVATE_PAYLOAD", "ZUSER8": 1}, + ), + "category_valid": ( + Category, + { + **common_row(Z_ENT=19), + "ZNAME2": "Category", + "ZPARENTCATEGORY": None, + "ZTYPE2": 1, + "ZUSER3": 1, + }, + ), + "category_invalid": ( + Category, + { + **common_row(Z_ENT=19), + "ZNAME2": "Category", + "ZPARENTCATEGORY": None, + "ZTYPE2": 1, + "ZUSER3": None, + }, + ), + "category_blob_name": ( + Category, + { + **common_row(Z_ENT=19), + "ZNAME2": b"PRIVATE_PAYLOAD", + "ZPARENTCATEGORY": None, + "ZTYPE2": 1, + "ZUSER3": 1, + }, + ), + } + return { + name: capture(lambda constructor=constructor, row=row: constructor(row).name) + for name, (constructor, row) in rows.items() + } + + +def relationship_probe(): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "relationships.sqlite" + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.execute("CREATE TABLE ZSYNCOBJECT (Z_PK INTEGER, Z_ENT INTEGER)") + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", + [ + (2, "CategoryAssigment", 0, 0), + (36, "Tag", 0, 0), + (37, "Transaction", 0, 0), + (50, "WithdrawRefundTransactionLink", 0, 0), + ], + ) + connection.execute( + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT)" + ) + connection.executemany( + "INSERT INTO ZCATEGORYASSIGMENT VALUES (?, ?, ?, ?)", + [ + (1, 10, 100, 5.0), + (2, None, 101, 2.0), + (3, 12, 102, "PRIVATE_PAYLOAD"), + (4, 13, None, 4.0), + (5, "PRIVATE_PAYLOAD", 103, 1.0), + ("PRIVATE_PAYLOAD", 14, 104, 1.0), + ], + ) + connection.execute( + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, ZWITHDRAWTRANSACTION INTEGER)" + ) + connection.executemany( + "INSERT INTO ZWITHDRAWREFUNDTRANSACTIONLINK VALUES (?, ?, ?)", + [ + (5, 103, 104), + (6, 105, None), + (7, "PRIVATE_PAYLOAD", 106), + ("PRIVATE_PAYLOAD", 107, 108), + ], + ) + connection.execute( + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_37TAGS VALUES (?, ?)", + [(100, 200), (101, None), ("PRIVATE_PAYLOAD", 201)], + ) + connection.commit() + connection.close() + + with DatabaseAccessor(path) as accessor: + categories, category_report = accessor.read_category_assignments() + refunds, refund_report = accessor.read_refund_maps() + tags, tag_report = accessor.read_tags_map() + aggregate_report = TransactionManager().load(accessor) + return normalize( + { + "categories": categories, + "category_report": category_report.as_dict(), + "refunds": refunds, + "refund_report": refund_report.as_dict(), + "tags": tags, + "tag_report": tag_report.as_dict(), + "aggregate": aggregate_report.as_dict(), + } + ) + + +def sentinel_probe(): + original_payee_as_dict = Payee.as_dict + original_deposit_as_dict = DepositTransaction.as_dict + original_account_as_dict = CashAccount.as_dict + + def reject_serialization(_self): + raise RuntimeError("PRIVATE_PAYLOAD_SERIALIZED") + + Payee.as_dict = reject_serialization + DepositTransaction.as_dict = reject_serialization + CashAccount.as_dict = reject_serialization + try: + payee = capture( + lambda: Payee( + { + **common_row(Z_ENT=28), + "ZNAME5": "Payee", + "ZUSER7": None, + } + ) + ) + deposit = capture(lambda: DepositTransaction(transaction_row(ZACCOUNT2=None))) + account = capture(lambda: CashAccount(account_row(ZNAME=None))) + finally: + Payee.as_dict = original_payee_as_dict + DepositTransaction.as_dict = original_deposit_as_dict + CashAccount.as_dict = original_account_as_dict + return {"payee": payee, "deposit": deposit, "account": account} + + +def main(): + manager = PayeeManager() + manager_report = manager.load(PayeeAccessor()) + valid_account = accessor_for(account_row()).get_record(1, CreditCardAccount) + valid_deposit = DepositTransaction(transaction_row()) + zero_rate_deposit = DepositTransaction( + transaction_row(ZAMOUNT1=10.0, ZORIGINALAMOUNT=10.0, ZORIGINALEXCHANGERATE=0.0) + ) + corrected_withdrawal = WithdrawTransaction( + transaction_row( + Z_ENT=47, + ZGID="withdraw-1", + ZAMOUNT1=-10.0, + ZORIGINALAMOUNT=10.0, + ZORIGINALEXCHANGERATE=0.0, + ) + ) + output = { + "optimize": sys.flags.optimize, + "cases": { + "decimal_valid": capture( + lambda: RawDataHandler.get_decimal({"value": 1.25}, "value") + ), + "decimal_string": capture( + lambda: RawDataHandler.get_decimal( + {"value": "PRIVATE_PAYLOAD"}, "value" + ) + ), + "date_valid": capture( + lambda: RawDataHandler.get_datetime({"value": 0.0}, "value") + ), + "date_string": capture( + lambda: RawDataHandler.get_datetime( + {"value": "PRIVATE_PAYLOAD"}, "value" + ) + ), + "record_valid": capture(lambda: Record(common_row()).gid), + "record_invalid_identity": capture(lambda: Record(common_row(ZGID=""))), + "record_coerced_gid": capture( + lambda: Record(common_row(ZGID=b"PRIVATE_PAYLOAD")) + ), + "record_coerced_identity": capture(lambda: Record(common_row(Z_PK=True))), + "account_nullable_info": capture( + lambda: { + "info": valid_account.info, + "statement_day": valid_account.statement_day, + } + ), + "account_invalid_name_public": capture( + lambda: accessor_for(account_row(ZNAME=None)).get_record_by_gid( + "account-1", CreditCardAccount + ) + ), + "account_blob_name": capture( + lambda: CashAccount(account_row(ZNAME=b"PRIVATE_PAYLOAD")) + ), + "account_blob_info": capture( + lambda: CashAccount(account_row(ZINFO=b"PRIVATE_PAYLOAD")) + ), + "account_coerced_display_order": capture( + lambda: CashAccount(account_row(ZDISPLAYORDER="PRIVATE_PAYLOAD")) + ), + "direct_accounts": direct_account_probe(), + "category_types": category_type_probe(), + "named_entities": named_entities_probe(), + "manager": manager_report.as_dict(), + "holding_valid": capture( + lambda: InvestmentHolding(holding_row(), PROFILE).number_of_shares + ), + "holding_missing_quantity": capture( + lambda: InvestmentHolding(holding_row(ZNUMBEROFSHARES=None), PROFILE) + ), + "holding_online_price_state": { + label: capture( + lambda value=value: ( + InvestmentHolding( + holding_row(ZISPRICEPERSHAREAVAILABLEONLINE=value), PROFILE + ).price_per_share_available_online + ) + ) + for label, value in ( + ("none", None), + ("false_boolean", False), + ("true_boolean", True), + ("zero", 0), + ("one", 1), + ("two", 2), + ("float", 1.0), + ("text", "PRIVATE_PAYLOAD"), + ) + }, + "deposit_valid": capture( + lambda: { + "amount": valid_deposit.amount, + "rate": valid_deposit.original_exchange_rate, + } + ), + "deposit_invalid_sign": capture( + lambda: DepositTransaction(transaction_row(ZORIGINALAMOUNT=-10.0)) + ), + "deposit_coerced_account": capture( + lambda: DepositTransaction(transaction_row(ZACCOUNT2="PRIVATE_PAYLOAD")) + ), + "reconciled": { + label: capture( + lambda value=value: ( + DepositTransaction( + transaction_row(ZRECONCILED=value) + ).reconciled + ) + ) + for label, value in ( + ("none", None), + ("false_boolean", False), + ("true_boolean", True), + ("zero", 0), + ("one", 1), + ("two", 2), + ("float", 1.0), + ("text", "PRIVATE_PAYLOAD"), + ) + }, + "deposit_invalid_fx": capture( + lambda: DepositTransaction(transaction_row(ZAMOUNT1=20.01)) + ), + "deposit_zero_rate": capture( + lambda: zero_rate_deposit.original_exchange_rate + ), + "investment_buy_valid_fee": capture( + lambda: { + "amount": InvestmentBuyTransaction(buy_row(), PROFILE).amount, + "fee": InvestmentBuyTransaction(buy_row(), PROFILE).fee, + } + ), + "investment_buy_invalid_quantity": capture( + lambda: InvestmentBuyTransaction(buy_row(ZNUMBEROFSHARES=0.0), PROFILE) + ), + "investment_buy_invalid_total": capture( + lambda: InvestmentBuyTransaction(buy_row(ZAMOUNT1=-20.0), PROFILE) + ), + "withdrawal_sign_fix": capture( + lambda: corrected_withdrawal.original_amount + ), + "transfer_zero_rate": capture( + lambda: TransferWithdrawTransaction(transfer_withdraw_row()) + ), + "relationships": relationship_probe(), + "sentinel": sentinel_probe(), + }, + } + print(json.dumps(normalize(output), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_accessor_admission.py b/tests/unit/test_accessor_admission.py new file mode 100644 index 0000000..706f3ae --- /dev/null +++ b/tests/unit/test_accessor_admission.py @@ -0,0 +1,251 @@ +from contextlib import contextmanager +import sqlite3 + +import pytest + +from moneywiz_api import MoneywizApi +from moneywiz_api.database_accessor import DatabaseAccessor, DatabaseSchemaError +from moneywiz_api.managers.account_manager import AccountManager +from tests.unit.test_schema_cache_binding import ( + create_account_store, + create_relationship_store, + open_writer, +) + + +def count_admissions(monkeypatch): + counts = {"schema": 0, "source": 0} + original_schema = DatabaseAccessor._verify_schema_identity + original_source = DatabaseAccessor._verify_source_eligibility + + def verify_schema(accessor): + counts["schema"] += 1 + return original_schema(accessor) + + def verify_source(accessor): + counts["source"] += 1 + return original_source(accessor) + + monkeypatch.setattr(DatabaseAccessor, "_verify_schema_identity", verify_schema) + monkeypatch.setattr(DatabaseAccessor, "_verify_source_eligibility", verify_source) + return counts + + +def assert_admission_reset(accessor) -> None: + assert accessor._admission_depth == 0 + assert accessor._admission_broken is False + assert not accessor._con.in_transaction + + +def test_eager_api_re_load_uses_one_admission(tmp_path, monkeypatch) -> None: + path = tmp_path / "eager.sqlite" + create_relationship_store(path) + counts = count_admissions(monkeypatch) + + with MoneywizApi(path) as api: + assert api.completeness().complete + assert counts == {"schema": 1, "source": 1} + + +def test_direct_manager_and_independent_getters_admit_per_outer_scope( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "direct.sqlite" + create_account_store(path) + counts = count_admissions(monkeypatch) + + with DatabaseAccessor(path) as accessor: + assert AccountManager().load(accessor).complete + assert counts == {"schema": 1, "source": 1} + assert accessor.get_record(1).id == 1 + assert accessor.get_record_by_gid("account-1").id == 1 + assert counts == {"schema": 3, "source": 3} + + +def test_explicit_nested_scope_reuses_one_admission(tmp_path, monkeypatch) -> None: + path = tmp_path / "nested.sqlite" + create_account_store(path) + counts = count_admissions(monkeypatch) + + with DatabaseAccessor(path) as accessor: + with accessor.read_transaction(): + assert accessor.query_objects(["CashAccount"]) + assert accessor.get_record(1).id == 1 + assert accessor.get_record_by_gid("account-1").id == 1 + assert counts == {"schema": 1, "source": 1} + assert_admission_reset(accessor) + + +@pytest.mark.parametrize("failure", ["schema", "source"]) +def test_failed_admission_resets_and_allows_retry( + tmp_path, monkeypatch, failure +) -> None: + path = tmp_path / f"failed-{failure}.sqlite" + create_account_store(path) + accessor = DatabaseAccessor(path) + method_name = ( + f"_verify_{failure}_eligibility" + if failure == "source" + else ("_verify_schema_identity") + ) + original = getattr(accessor, method_name) + + def refuse(): + raise DatabaseSchemaError("synthetic admission refusal") + + monkeypatch.setattr(accessor, method_name, refuse) + with pytest.raises(DatabaseSchemaError, match="synthetic admission refusal"): + accessor.query_objects(["CashAccount"]) + assert_admission_reset(accessor) + + monkeypatch.setattr(accessor, method_name, original) + assert len(accessor.query_objects(["CashAccount"])) == 1 + accessor.close() + + +def test_nested_body_failure_does_not_poison_active_or_later_scope( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "body-error.sqlite" + create_account_store(path) + counts = count_admissions(monkeypatch) + + with DatabaseAccessor(path) as accessor: + with accessor.read_transaction(): + with pytest.raises(RuntimeError, match="synthetic body failure"): + with accessor.read_transaction(): + raise RuntimeError("synthetic body failure") + assert accessor.get_record(1).id == 1 + assert counts == {"schema": 1, "source": 1} + assert accessor.query_objects(["CashAccount"]) + assert counts == {"schema": 2, "source": 2} + + +def test_outer_exit_failure_resets_admission_and_allows_retry( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "exit-error.sqlite" + create_account_store(path) + accessor = DatabaseAccessor(path) + original = accessor._raw_read_transaction + + @contextmanager + def fail_after_exit(): + with original(): + yield + raise DatabaseSchemaError("synthetic outer exit failure") + + monkeypatch.setattr(accessor, "_raw_read_transaction", fail_after_exit) + with pytest.raises(DatabaseSchemaError, match="synthetic outer exit failure"): + accessor.query_objects(["CashAccount"]) + assert_admission_reset(accessor) + + monkeypatch.setattr(accessor, "_raw_read_transaction", original) + assert accessor.query_objects(["CashAccount"]) + accessor.close() + + +def test_public_close_inside_scope_refuses_interrupted_snapshot_and_cleans_depth( + tmp_path, +) -> None: + path = tmp_path / "closed.sqlite" + create_account_store(path) + accessor = DatabaseAccessor(path) + + with pytest.raises(DatabaseSchemaError, match="read transaction was interrupted"): + with accessor.read_transaction(): + accessor.close() + + assert accessor._admission_depth == 0 + assert accessor._admission_broken is False + + +def test_simulated_transaction_loss_latches_until_outer_scope_unwinds(tmp_path) -> None: + path = tmp_path / "simulated-loss.sqlite" + create_account_store(path) + + with DatabaseAccessor(path) as accessor: + with pytest.raises( + DatabaseSchemaError, match="read transaction was interrupted" + ): + with accessor.read_transaction(): + # Simulate a SQLite statement aborting the transaction. This does not + # assert that an interrupted SELECT itself performs a rollback. + accessor._con.rollback() + with accessor.read_transaction(): + pytest.fail("lost outer snapshot must not admit a nested read") + assert_admission_reset(accessor) + assert accessor.query_objects(["CashAccount"]) + + +def test_caller_owned_transaction_is_verified_per_public_outer_scope( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "caller-owned.sqlite" + create_account_store(path) + counts = count_admissions(monkeypatch) + + with DatabaseAccessor(path) as accessor: + accessor._con.execute("BEGIN") + with accessor.read_transaction(): + assert accessor.get_record(1).id == 1 + assert counts == {"schema": 1, "source": 1} + assert accessor._con.in_transaction + assert accessor.query_objects(["CashAccount"]) + assert counts == {"schema": 2, "source": 2} + assert accessor._con.in_transaction + accessor._con.rollback() + + +def test_wal_orphan_remains_outside_pinned_generation_then_next_scope_refuses( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "wal-orphan.sqlite" + create_account_store(path) + counts = count_admissions(monkeypatch) + + with DatabaseAccessor(path) as accessor: + with accessor.read_transaction(): + assert len(accessor.query_objects(["CashAccount"])) == 1 + writer = open_writer(path) + writer.execute( + "INSERT INTO ZSYNCOBJECT VALUES " + "(2, 999, 0, 'orphan', 2, 1, 'Orphan', 'EUR', 0, NULL, 1)" + ) + writer.commit() + writer.close() + assert len(accessor.query_objects(["CashAccount"])) == 1 + assert counts == {"schema": 1, "source": 1} + + with pytest.raises(DatabaseSchemaError, match="unclassifiable entity ancestry"): + accessor.query_objects(["CashAccount"]) + assert counts == {"schema": 2, "source": 2} + + writer = open_writer(path) + writer.execute("DELETE FROM ZSYNCOBJECT WHERE Z_PK = 2") + writer.commit() + writer.close() + assert len(accessor.query_objects(["CashAccount"])) == 1 + assert counts == {"schema": 3, "source": 3} + + +def test_nested_query_error_with_intact_transaction_reuses_admission( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "query-error.sqlite" + create_account_store(path) + counts = count_admissions(monkeypatch) + + with DatabaseAccessor(path) as accessor: + original = accessor._query_objects + + def fail_query(_typenames): + raise sqlite3.OperationalError("synthetic query error") + + with accessor.read_transaction(): + monkeypatch.setattr(accessor, "_query_objects", fail_query) + with pytest.raises(sqlite3.OperationalError, match="synthetic query error"): + accessor.query_objects(["CashAccount"]) + monkeypatch.setattr(accessor, "_query_objects", original) + assert accessor.query_objects(["CashAccount"]) + assert counts == {"schema": 1, "source": 1} diff --git a/tests/unit/test_account_diagnostics.py b/tests/unit/test_account_diagnostics.py new file mode 100644 index 0000000..12be241 --- /dev/null +++ b/tests/unit/test_account_diagnostics.py @@ -0,0 +1,232 @@ +from contextlib import contextmanager + +import pytest + +from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.model.account import ( + Account, + BankChequeAccount, + BankSavingAccount, + CashAccount, + CreditCardAccount, + ForexAccount, + InvestmentAccount, + LoanAccount, +) +from tests.unit.accessor_test_support import initialized_memory_accessor + + +ACCOUNT_CONSTRUCTORS = ( + Account, + BankChequeAccount, + BankSavingAccount, + CashAccount, + CreditCardAccount, + LoanAccount, + InvestmentAccount, + ForexAccount, +) + + +def account_row(**overrides): + row = { + "Z_ENT": 13, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": "account-1", + "Z_PK": 1, + "ZDISPLAYORDER": 1, + "ZGROUPID": 1, + "ZNAME": "Card", + "ZCURRENCYNAME": "EUR", + "ZOPENINGBALANCE": 0.0, + "ZINFO": None, + "ZUSER": 1, + "ZSTATEMENTENDDAY": 15, + } + row.update(overrides) + return row + + +class AccountAccessor: + def __init__(self, rows): + self.rows = rows + + @contextmanager + def read_transaction(self): + yield + + def descendant_typenames(self, _roots): + return ["CreditCardAccount"] + + def query_objects(self, _typenames): + return self.rows + + def typename_for(self, _ent_id): + return "CreditCardAccount" + + +def test_nullable_info_does_not_mask_complete_credit_card() -> None: + manager = AccountManager() + + report = manager.load(AccountAccessor([account_row()])) + + assert report.complete + assert manager.get(1).info is None + assert manager.get(1).statement_day == 15 + + +def test_missing_optional_info_column_is_nullable() -> None: + manager = AccountManager() + row = account_row() + del row["ZINFO"] + + report = manager.load(AccountAccessor([row])) + + assert report.complete + assert manager.get(1).info is None + + +def test_required_account_identity_remains_required() -> None: + manager = AccountManager() + + report = manager.load(AccountAccessor([account_row(ZNAME=None)])) + + assert not report.complete + assert report.skipped[0].error.value == "validation" + assert report.skipped[0].exception_type == "AssertionError" + + +@pytest.mark.parametrize( + "raw_value", + [None, False, True, 1.0, "PRIVATE_PAYLOAD"], +) +def test_direct_accounts_reject_non_integer_display_order(raw_value) -> None: + with pytest.raises(AssertionError) as error: + CashAccount(account_row(ZDISPLAYORDER=raw_value)) + + assert "PRIVATE_PAYLOAD" not in str(error.value) + + +def test_account_manager_skips_invalid_display_order_before_sorting() -> None: + invalid = account_row(ZDISPLAYORDER="PRIVATE_PAYLOAD") + valid = account_row(Z_PK=2, ZGID="account-2", ZDISPLAYORDER=2) + manager = AccountManager() + + report = manager.load(AccountAccessor([invalid, valid])) + + assert not report.complete + assert report.parsed_ids == (2,) + assert report.skipped[0].error.value == "validation" + assert list(manager.records()) == [2] + + +@pytest.mark.parametrize("constructor", ACCOUNT_CONSTRUCTORS) +@pytest.mark.parametrize("info_present", [True, False]) +def test_direct_account_constructors_accept_nullable_or_absent_info( + constructor, info_present +) -> None: + row = account_row() + if not info_present: + del row["ZINFO"] + + account = constructor(row) + + assert account.info is None + assert account.name == "Card" + + +@pytest.mark.parametrize("constructor", ACCOUNT_CONSTRUCTORS) +def test_direct_account_constructors_reject_missing_required_name(constructor) -> None: + with pytest.raises(AssertionError, match="account name is required"): + constructor(account_row(ZNAME=None)) + + +@pytest.mark.parametrize("constructor", [CreditCardAccount, LoanAccount]) +def test_direct_credit_constructors_reject_null_statement_day(constructor) -> None: + with pytest.raises(AssertionError, match="account statement day is required"): + constructor(account_row(ZSTATEMENTENDDAY=None)) + + +@pytest.mark.parametrize("constructor", [CreditCardAccount, LoanAccount]) +def test_direct_credit_constructors_preserve_missing_statement_key(constructor) -> None: + row = account_row() + del row["ZSTATEMENTENDDAY"] + + with pytest.raises(KeyError, match="ZSTATEMENTENDDAY"): + constructor(row) + + +@pytest.mark.parametrize( + ("field", "value", "exception"), + [ + ("ZGID", "", AssertionError), + ("Z_PK", 0, AssertionError), + ("ZGID", None, AssertionError), + ], +) +def test_direct_account_preserves_record_identity_validation( + field, value, exception +) -> None: + with pytest.raises(exception): + CashAccount(account_row(**{field: value})) + + +def test_direct_account_preserves_missing_record_identity_field() -> None: + row = account_row() + del row["ZGID"] + + with pytest.raises(KeyError, match="ZGID"): + CashAccount(row) + + +def test_direct_account_validation_does_not_serialize_source(monkeypatch) -> None: + def reject_serialization(_self): + raise RuntimeError("PRIVATE_PAYLOAD_SERIALIZED") + + monkeypatch.setattr(CashAccount, "as_dict", reject_serialization) + row = account_row(ZNAME=None, ZINFO="PRIVATE_PAYLOAD") + + with pytest.raises(AssertionError, match="account name is required") as error: + CashAccount(row) + + assert "PRIVATE_PAYLOAD" not in str(error.value) + + +def accessor_for(row): + return initialized_memory_accessor([row], [(13, "CreditCardAccount", 0)]) + + +def test_public_accessor_validates_nullable_info_after_construction() -> None: + account = accessor_for(account_row()).get_record(1, CreditCardAccount) + account_by_gid = accessor_for(account_row()).get_record_by_gid( + "account-1", CreditCardAccount + ) + + assert account.info is None + assert account_by_gid.statement_day == 15 + + +@pytest.mark.parametrize( + "row", + [ + account_row(ZNAME=None), + account_row(ZSTATEMENTENDDAY=None), + ], +) +@pytest.mark.parametrize( + ("method_name", "lookup"), + [("get_record", 1), ("get_record_by_gid", "account-1")], +) +def test_public_accessor_rejects_invalid_account_metadata( + row, method_name, lookup +) -> None: + with pytest.raises(AssertionError): + getattr(accessor_for(row), method_name)(lookup, CreditCardAccount) + + +def test_public_accessor_missing_subclass_field_is_not_masked() -> None: + row = account_row() + del row["ZSTATEMENTENDDAY"] + + with pytest.raises(KeyError, match="ZSTATEMENTENDDAY"): + accessor_for(row).get_record_by_gid("account-1", CreditCardAccount) diff --git a/tests/unit/test_api_atomic_reload.py b/tests/unit/test_api_atomic_reload.py new file mode 100644 index 0000000..4cdb1e6 --- /dev/null +++ b/tests/unit/test_api_atomic_reload.py @@ -0,0 +1,402 @@ +from contextlib import contextmanager +from decimal import Decimal + +import pytest + +from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.managers.transaction_manager import TransactionManager +from moneywiz_api.read_result import RelationshipLoadReport, RelationshipStorage +from moneywiz_api.schema_profile import SchemaProfile + + +def account_row(record_id, name): + return { + "Z_ENT": 10, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": f"account-{record_id}", + "Z_PK": record_id, + "ZDISPLAYORDER": record_id, + "ZGROUPID": 1, + "ZNAME": name, + "ZCURRENCYNAME": "EUR", + "ZOPENINGBALANCE": 0.0, + "ZINFO": None, + "ZUSER": 1, + } + + +def transaction_row(record_id, account_id): + return { + "Z_ENT": 37, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": f"transaction-{record_id}", + "Z_PK": record_id, + "ZRECONCILED": 0, + "ZAMOUNT1": 5.0, + "ZDESC2": f"Transaction {record_id}", + "ZDATE1": 0.0, + "ZNOTES1": None, + "ZACCOUNT2": account_id, + "ZPAYEE2": None, + "ZORIGINALCURRENCY": "EUR", + "ZORIGINALAMOUNT": 5.0, + "ZORIGINALEXCHANGERATE": 1.0, + } + + +class SyntheticAccessor: + def __init__(self): + self.schema_profile = SchemaProfile( + "unsuffixed-investment-columns", + "ZNUMBEROFSHARES", + "ZNUMBEROFSHARES", + "ZPRICEPERSHARE", + "ZPRICEPERSHARE", + ) + self.generation = "old" + self.fail_at = None + self.extra_invalid_account = False + self.transaction_depth = 0 + self.transaction_entries = 0 + self.transaction_exits = 0 + self.exit_failure_depths = [] + + @property + def in_transaction(self): + return self.transaction_depth > 0 + + @contextmanager + def read_transaction(self): + entered_depth = self.transaction_depth + 1 + if self.fail_at == "enter" and entered_depth == 1: + raise RuntimeError("synthetic transaction enter failure") + self.transaction_entries += 1 + self.transaction_depth = entered_depth + try: + yield + finally: + self.transaction_depth -= 1 + self.transaction_exits += 1 + if self.fail_at == "exit" and entered_depth == 1: + self.exit_failure_depths.append(entered_depth) + raise RuntimeError("synthetic transaction exit failure") + + def descendant_typenames(self, _roots): + return [] + + def query_objects(self, typenames): + if self.fail_at == "transaction_query" and "DepositTransaction" in typenames: + raise RuntimeError("synthetic transaction query failure") + if "CashAccount" in typenames: + if self.generation == "old": + return [account_row(1, "Old")] + rows = [account_row(2, "New")] + if self.extra_invalid_account: + rows.append(account_row(3, None)) + return rows + if "DepositTransaction" in typenames: + record_id, account_id = (10, 1) if self.generation == "old" else (20, 2) + return [transaction_row(record_id, account_id)] + return [] + + def typename_for(self, ent_id): + return {10: "CashAccount", 37: "DepositTransaction"}.get(ent_id) + + def _relationship(self, name): + if self.fail_at == name: + raise RuntimeError(f"synthetic {name} failure") + transaction_id = 10 if self.generation == "old" else 20 + if name == "category": + value = {transaction_id: [(30, Decimal("5"))]} + elif name == "refund": + value = {transaction_id: 40} + else: + value = {transaction_id: [50]} + report = RelationshipLoadReport( + storage=RelationshipStorage.PRESENT, + storage_name=name, + source_ids=(f"{name}-{transaction_id}",), + parsed_ids=(f"{name}-{transaction_id}",), + ) + return value, report + + def read_category_assignments(self): + return self._relationship("category") + + def read_refund_maps(self): + return self._relationship("refund") + + def read_tags_map(self): + return self._relationship("tags") + + def close(self): + pass + + +def api_for(monkeypatch, accessor, managers): + import moneywiz_api.moneywiz_api as api_module + + monkeypatch.setattr(api_module, "DatabaseAccessor", lambda _path: accessor) + return api_module.MoneywizApi("unused.sqlite", managers=managers) + + +@pytest.mark.parametrize("interruption_type", [KeyboardInterrupt, SystemExit]) +def test_constructor_interruption_closes_accessor_and_reraises( + monkeypatch, interruption_type +) -> None: + import moneywiz_api.moneywiz_api as api_module + + accessor = SyntheticAccessor() + accessor.closed = False + interruption = interruption_type("synthetic API construction interruption") + + def interrupt_load(_self, _managers=None): + raise interruption + + def close(): + accessor.closed = True + + accessor.close = close + monkeypatch.setattr(api_module, "DatabaseAccessor", lambda _path: accessor) + monkeypatch.setattr(api_module.MoneywizApi, "load", interrupt_load) + + with pytest.raises(interruption_type) as error: + api_module.MoneywizApi("unused.sqlite") + + assert error.value is interruption + assert accessor.closed + + +@pytest.mark.parametrize( + "failure", + ["transaction_query", "category", "refund", "tags"], +) +def test_failed_later_load_keeps_published_union_and_relationships( + monkeypatch, failure +) -> None: + accessor = SyntheticAccessor() + api = api_for(monkeypatch, accessor, ("accounts", "transactions")) + account_manager = api.account_manager + transaction_manager = api.transaction_manager + old_state = { + "account_records": account_manager._records, + "account_gid_index": account_manager._gid_to_id, + "account_report": account_manager.load_report, + "account": account_manager.get(1), + "transaction_records": transaction_manager._records, + "transaction_gid_index": transaction_manager._gid_to_id, + "transaction_report": transaction_manager.load_report, + "transaction": transaction_manager.get(10), + "categories": transaction_manager.category_assignment, + "refunds": transaction_manager.refund_maps, + "tags": transaction_manager.tags_map, + } + accessor.generation = "new" + accessor.fail_at = failure + + with pytest.raises(RuntimeError, match="synthetic"): + api.load(("transactions",)) + + assert api.account_manager is account_manager + assert api.transaction_manager is transaction_manager + assert api._loaded_managers == {"accounts", "transactions"} + assert account_manager._records is old_state["account_records"] + assert account_manager._gid_to_id is old_state["account_gid_index"] + assert account_manager.load_report is old_state["account_report"] + assert account_manager.load_report.status == "complete" + assert account_manager.get(1) is old_state["account"] + assert account_manager.get_by_gid("account-1") is old_state["account"] + assert transaction_manager._records is old_state["transaction_records"] + assert transaction_manager._gid_to_id is old_state["transaction_gid_index"] + assert transaction_manager.load_report is old_state["transaction_report"] + assert transaction_manager.load_report.status == "complete" + assert transaction_manager.get(10) is old_state["transaction"] + assert transaction_manager.get_by_gid("transaction-10") is old_state["transaction"] + assert transaction_manager.category_assignment is old_state["categories"] + assert transaction_manager.refund_maps is old_state["refunds"] + assert transaction_manager.tags_map is old_state["tags"] + assert api.completeness().managers["accounts"] is old_state["account_report"] + assert ( + api.completeness().managers["transactions"] is old_state["transaction_report"] + ) + assert not accessor.in_transaction + + +@pytest.mark.parametrize("failure", ["enter", "exit"]) +def test_transaction_context_failure_does_not_publish(monkeypatch, failure) -> None: + accessor = SyntheticAccessor() + api = api_for(monkeypatch, accessor, ("accounts",)) + records = api.account_manager._records + report = api.account_manager.load_report + account = api.account_manager.get(1) + accessor.generation = "new" + accessor.fail_at = failure + + with pytest.raises(RuntimeError, match="transaction .* failure"): + api.load(("accounts",)) + + assert api.account_manager._records is records + assert api.account_manager.load_report is report + assert api.account_manager.get(1) is account + assert api._loaded_managers == {"accounts"} + assert not accessor.in_transaction + + +def test_outer_exit_failure_retains_all_published_state_and_can_retry( + monkeypatch, +) -> None: + accessor = SyntheticAccessor() + api = api_for(monkeypatch, accessor, ("accounts", "transactions")) + account_manager = api.account_manager + transaction_manager = api.transaction_manager + loaded_managers = api._loaded_managers + old_state = { + "account_records": account_manager._records, + "account_gid_index": account_manager._gid_to_id, + "account_report": account_manager.load_report, + "account": account_manager.get(1), + "transaction_records": transaction_manager._records, + "transaction_gid_index": transaction_manager._gid_to_id, + "transaction_report": transaction_manager.load_report, + "transaction": transaction_manager.get(10), + "categories": transaction_manager.category_assignment, + "refunds": transaction_manager.refund_maps, + "tags": transaction_manager.tags_map, + } + accessor.generation = "new" + accessor.fail_at = "exit" + + with pytest.raises(RuntimeError, match="synthetic transaction exit failure"): + api.load(("transactions",)) + + assert accessor.exit_failure_depths == [1] + assert accessor.transaction_depth == 0 + assert api.account_manager is account_manager + assert api.transaction_manager is transaction_manager + assert api._loaded_managers is loaded_managers + assert api._loaded_managers == {"accounts", "transactions"} + assert account_manager._records is old_state["account_records"] + assert account_manager._gid_to_id is old_state["account_gid_index"] + assert account_manager.load_report is old_state["account_report"] + assert account_manager.get(1) is old_state["account"] + assert transaction_manager._records is old_state["transaction_records"] + assert transaction_manager._gid_to_id is old_state["transaction_gid_index"] + assert transaction_manager.load_report is old_state["transaction_report"] + assert transaction_manager.get(10) is old_state["transaction"] + assert transaction_manager.category_assignment is old_state["categories"] + assert transaction_manager.refund_maps is old_state["refunds"] + assert transaction_manager.tags_map is old_state["tags"] + + accessor.fail_at = None + report = api.load(("transactions",)) + + assert report.complete + assert api.account_manager.get(2).name == "New" + assert api.transaction_manager.get(20).account == 2 + + +def test_failed_new_selection_remains_unloaded_then_successful_retry_publishes( + monkeypatch, +) -> None: + accessor = SyntheticAccessor() + api = api_for(monkeypatch, accessor, ("accounts",)) + account_manager = api.account_manager + transaction_manager = api.transaction_manager + old_account = account_manager.get(1) + accessor.generation = "new" + accessor.fail_at = "transaction_query" + + with pytest.raises(RuntimeError, match="transaction query"): + api.load(("transactions",)) + + assert api._loaded_managers == {"accounts"} + assert account_manager.get(1) is old_account + assert transaction_manager.records() == {} + assert transaction_manager.load_report.status == "unloaded" + with pytest.raises(ValueError, match="manager not loaded: transactions"): + api.completeness(("transactions",)) + + accessor.fail_at = None + report = api.load(("transactions",)) + + assert api.account_manager is account_manager + assert api.transaction_manager is transaction_manager + assert api._loaded_managers == {"accounts", "transactions"} + assert account_manager.get(1) is None + assert account_manager.get(2).name == "New" + assert account_manager.get_by_gid("account-2") is account_manager.get(2) + assert transaction_manager.get(20).description == "Transaction 20" + assert transaction_manager.category_assignment == {20: [(30, Decimal("5"))]} + assert transaction_manager.refund_maps == {20: 40} + assert transaction_manager.tags_map == {20: [50]} + assert report.complete + + +@pytest.mark.parametrize("failure", ["category", "refund", "tags"]) +def test_direct_transaction_relationship_failure_stays_unloaded_then_retries( + failure, +) -> None: + accessor = SyntheticAccessor() + accessor.fail_at = failure + manager = TransactionManager() + + with pytest.raises(RuntimeError, match=f"synthetic {failure} failure"): + manager.load(accessor) + + assert manager.load_report.status == "unloaded" + assert not manager.load_report.complete + assert manager.records() == {} + assert manager.category_assignment == {} + assert manager.refund_maps == {} + assert manager.tags_map == {} + + accessor.fail_at = None + report = manager.load(accessor) + + assert report.status == "complete" + assert report.complete + assert manager.get(10) is not None + + accessor.fail_at = failure + with pytest.raises(RuntimeError, match=f"synthetic {failure} failure"): + manager.load(accessor) + + assert manager.load_report.status == "unloaded" + assert not manager.load_report.complete + assert manager.records() == {} + + +def test_successful_publication_occurs_after_transaction_exit(monkeypatch) -> None: + accessor = SyntheticAccessor() + adoption_depths = [] + original_adopt = AccountManager._adopt_loaded_state + + def observe_adoption(self, staged): + assert accessor.transaction_depth == 0 + adoption_depths.append(accessor.transaction_depth) + original_adopt(self, staged) + + monkeypatch.setattr(AccountManager, "_adopt_loaded_state", observe_adoption) + api = api_for(monkeypatch, accessor, ("accounts",)) + accessor.generation = "new" + + api.load(("accounts",)) + + assert adoption_depths == [0, 0] + assert accessor.transaction_entries == accessor.transaction_exits == 4 + + +def test_partial_staged_generation_is_published(monkeypatch) -> None: + accessor = SyntheticAccessor() + api = api_for(monkeypatch, accessor, ("accounts",)) + accessor.generation = "new" + accessor.extra_invalid_account = True + + report = api.load(("accounts",)).managers["accounts"] + + assert report.source_ids == (2, 3) + assert report.parsed_ids == (2,) + assert report.skipped[0].record_id == 3 + assert report.skipped[0].error.value == "validation" + assert api.account_manager.get(2).name == "New" + assert api.account_manager.get(3) is None diff --git a/tests/unit/test_api_snapshot_generation.py b/tests/unit/test_api_snapshot_generation.py new file mode 100644 index 0000000..c778d9e --- /dev/null +++ b/tests/unit/test_api_snapshot_generation.py @@ -0,0 +1,83 @@ +import sqlite3 + +import pytest + +from moneywiz_api import MoneywizApi + + +def create_read_schema(path) -> None: + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", + [ + (8, "SyncObject", 0, 0), + (9, "Account", 8, 0), + (10, "CashAccount", 9, 0), + (36, "Transaction", 8, 0), + (37, "DepositTransaction", 36, 0), + ], + ) + connection.execute( + "CREATE TABLE ZSYNCOBJECT (" + "Z_PK INTEGER PRIMARY KEY, Z_ENT INTEGER, ZOBJECTCREATIONDATE FLOAT, " + "ZGID TEXT, ZDISPLAYORDER INTEGER, ZGROUPID INTEGER, ZNAME TEXT, " + "ZCURRENCYNAME TEXT, ZOPENINGBALANCE FLOAT, ZINFO TEXT, ZUSER INTEGER, " + "ZRECONCILED INTEGER, ZAMOUNT1 FLOAT, ZDESC2 TEXT, ZDATE1 FLOAT, " + "ZNOTES1 TEXT, ZACCOUNT2 INTEGER, ZPAYEE2 INTEGER, " + "ZORIGINALCURRENCY TEXT, ZORIGINALAMOUNT FLOAT, " + "ZORIGINALEXCHANGERATE FLOAT)" + ) + connection.execute( + "INSERT INTO ZSYNCOBJECT " + "(Z_PK, Z_ENT, ZOBJECTCREATIONDATE, ZGID, ZDISPLAYORDER, ZGROUPID, " + "ZNAME, ZCURRENCYNAME, ZOPENINGBALANCE, ZINFO, ZUSER) " + "VALUES (1, 10, 0, 'account-1', 1, 1, 'Before', 'EUR', 0, NULL, 1)" + ) + connection.commit() + connection.close() + + +def mutate_between_loads(path) -> None: + connection = sqlite3.connect(path) + connection.execute("UPDATE ZSYNCOBJECT SET ZNAME = 'After' WHERE Z_PK = 1") + connection.execute( + "INSERT INTO ZSYNCOBJECT " + "(Z_PK, Z_ENT, ZOBJECTCREATIONDATE, ZGID, ZRECONCILED, ZAMOUNT1, " + "ZDESC2, ZDATE1, ZNOTES1, ZACCOUNT2, ZPAYEE2, ZORIGINALCURRENCY, " + "ZORIGINALAMOUNT, ZORIGINALEXCHANGERATE) " + "VALUES (2, 37, 0, 'transaction-2', 0, 5, 'Income', 0, NULL, 1, " + "NULL, 'EUR', 5, 1)" + ) + connection.commit() + connection.close() + + +@pytest.mark.parametrize( + ("initial_managers", "expected_managers"), + [ + (("accounts",), ("accounts", "transactions")), + (None, MoneywizApi.MANAGER_NAMES), + ], +) +def test_scoped_reload_refreshes_loaded_union_atomically( + tmp_path, initial_managers, expected_managers +) -> None: + path = tmp_path / "generation.sqlite" + create_read_schema(path) + + with MoneywizApi(path, managers=initial_managers) as api: + assert api.account_manager.get(1).name == "Before" + mutate_between_loads(path) + + completeness = api.load(("transactions",)) + snapshot = api.snapshot().as_dict() + + assert tuple(completeness.managers) == expected_managers + assert api.account_manager.get(1).name == "After" + assert api.transaction_manager.get(2).description == "Income" + assert tuple(snapshot["completeness"]["managers"]) == expected_managers + assert snapshot["records"]["accounts"][0]["name"] == "After" diff --git a/tests/unit/test_category_diagnostics.py b/tests/unit/test_category_diagnostics.py new file mode 100644 index 0000000..26e6781 --- /dev/null +++ b/tests/unit/test_category_diagnostics.py @@ -0,0 +1,124 @@ +from contextlib import contextmanager +import json + +import pytest + +from moneywiz_api.managers.category_manager import CategoryManager +from moneywiz_api.model.category import Category +from moneywiz_api.moneywiz_api import MoneywizApi + + +def category_row(**overrides): + row = { + "Z_ENT": 19, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": "category-1", + "Z_PK": 1, + "ZNAME2": "Category", + "ZPARENTCATEGORY": None, + "ZTYPE2": 1, + "ZUSER3": 1, + } + row.update(overrides) + return row + + +class CategoryAccessor: + def __init__(self, rows): + self.rows = rows + + @contextmanager + def read_transaction(self): + yield + + def descendant_typenames(self, _roots): + return ["Category"] + + def query_objects(self, _typenames): + return self.rows + + def typename_for(self, _ent_id): + return "Category" + + +@pytest.mark.parametrize( + ("raw_type", "expected"), + [(1, "Expenses"), (2, "Income")], +) +def test_supported_category_types_are_unchanged(raw_type, expected) -> None: + assert Category(category_row(ZTYPE2=raw_type)).type == expected + + +@pytest.mark.parametrize("raw_type", [None, 0, 3, "PRIVATE_PAYLOAD"]) +def test_unsupported_category_type_is_invalid_value(raw_type, caplog) -> None: + manager = CategoryManager() + caplog.set_level("DEBUG") + + report = manager.load(CategoryAccessor([category_row(ZTYPE2=raw_type)])) + + assert report.source_ids == (1,) + assert report.parsed_ids == () + assert len(report.skipped) == 1 + assert report.skipped[0].record_id == 1 + assert report.skipped[0].error.value == "invalid_value" + assert report.skipped[0].exception_type == "ValueError" + serialized = json.dumps(report.as_dict(), sort_keys=True) + assert "PRIVATE_PAYLOAD" not in serialized + assert "PRIVATE_PAYLOAD" not in caplog.text + + +def test_unsupported_category_direct_error_has_static_message() -> None: + with pytest.raises(ValueError, match="^unsupported category type$") as error: + Category(category_row(ZTYPE2="PRIVATE_PAYLOAD")) + + assert "PRIVATE_PAYLOAD" not in str(error.value) + + +def test_missing_category_type_remains_missing_field() -> None: + row = category_row() + del row["ZTYPE2"] + + report = CategoryManager().load(CategoryAccessor([row])) + + assert report.skipped[0].error.value == "missing_field" + assert report.skipped[0].exception_type == "KeyError" + + +def test_invalid_category_owner_remains_validation() -> None: + report = CategoryManager().load(CategoryAccessor([category_row(ZUSER3=None)])) + + assert report.skipped[0].error.value == "validation" + assert report.skipped[0].exception_type == "AssertionError" + + +def test_unexpected_runtime_error_remains_construction() -> None: + def unexpected_constructor(_row): + raise RuntimeError("synthetic unexpected constructor failure") + + class UnexpectedCategoryManager(CategoryManager): + @property + def ents(self): + return {"Category": unexpected_constructor} + + report = UnexpectedCategoryManager().load(CategoryAccessor([category_row()])) + + assert report.skipped[0].error.value == "construction" + assert report.skipped[0].exception_type == "RuntimeError" + + +def test_category_diagnostic_is_exposed_by_api_completeness() -> None: + manager = CategoryManager() + manager.load(CategoryAccessor([category_row(ZTYPE2=3)])) + api = MoneywizApi.__new__(MoneywizApi) + api.category_manager = manager + api._managers = {"categories": manager} + api._loaded_managers = {"categories"} + + report = api.completeness(("categories",)).as_dict() + + category_report = report["managers"]["categories"] + assert report["status"] == "error" + assert not report["complete"] + assert category_report["source_count"] == 1 + assert category_report["parsed_count"] == 0 + assert category_report["skipped"][0]["error"] == "invalid_value" diff --git a/tests/unit/test_database_accessor.py b/tests/unit/test_database_accessor.py new file mode 100644 index 0000000..f71328e --- /dev/null +++ b/tests/unit/test_database_accessor.py @@ -0,0 +1,131 @@ +import sqlite3 + +import pytest + +from moneywiz_api.database_accessor import ( + DatabaseAccessor, + DatabasePathError, + DatabaseSchemaError, +) + + +def create_schema(path) -> None: + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.execute("CREATE TABLE ZSYNCOBJECT (Z_PK INTEGER, Z_ENT INTEGER)") + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", + [ + (8, "SyncObject", 0, 0), + (9, "Account", 8, 0), + (10, "CashAccount", 9, 0), + (11, "FutureAccount", 9, 0), + (12, "OnlineAccount", 8, 0), + ], + ) + connection.commit() + connection.close() + + +def test_accessor_requires_an_existing_database(tmp_path) -> None: + with pytest.raises(DatabasePathError): + DatabaseAccessor(tmp_path / "missing.sqlite") + + +def test_accessor_rejects_non_moneywiz_sqlite(tmp_path) -> None: + path = tmp_path / "other.sqlite" + sqlite3.connect(path).close() + + with pytest.raises(DatabaseSchemaError): + DatabaseAccessor(path) + + +def test_accessor_is_read_only_and_discovers_true_descendants(tmp_path) -> None: + path = tmp_path / "moneywiz.sqlite" + create_schema(path) + + with DatabaseAccessor(path) as accessor: + assert accessor.descendant_typenames(("Account",)) == [ + "Account", + "CashAccount", + "FutureAccount", + ] + with pytest.raises(sqlite3.OperationalError, match="readonly"): + accessor._con.execute("CREATE TABLE mutation (id INTEGER)") + + +class TrackingConnection(sqlite3.Connection): + was_closed = False + + def close(self): + self.was_closed = True + super().close() + + +@pytest.mark.parametrize("super_definition", ["", ", Z_SUPER INTEGER"]) +def test_metadata_failures_close_connection_and_raise_schema_error( + tmp_path, monkeypatch, super_definition +) -> None: + path = tmp_path / "malformed.sqlite" + original_connect = sqlite3.connect + setup_connection = original_connect(path) + setup_connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + f"(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT{super_definition})" + ) + setup_connection.execute("CREATE TABLE ZSYNCOBJECT (Z_PK INTEGER, Z_ENT INTEGER)") + if super_definition: + setup_connection.execute( + "INSERT INTO Z_PRIMARYKEY (Z_ENT, Z_NAME, Z_SUPER) " + "VALUES (8, 'SyncObject', NULL)" + ) + setup_connection.commit() + setup_connection.close() + opened = [] + + def tracked_connect(*args, **kwargs): + connection = original_connect(*args, factory=TrackingConnection, **kwargs) + opened.append(connection) + return connection + + monkeypatch.setattr(sqlite3, "connect", tracked_connect) + + with pytest.raises(DatabaseSchemaError): + DatabaseAccessor(path) + + assert len(opened) == 1 + assert opened[0].was_closed + + +@pytest.mark.parametrize("interruption_type", [KeyboardInterrupt, SystemExit]) +def test_schema_initialization_interruption_closes_connection_and_reraises( + tmp_path, monkeypatch, interruption_type +) -> None: + path = tmp_path / "interrupted.sqlite" + sqlite3.connect(path).close() + original_connect = sqlite3.connect + opened = [] + interruption = interruption_type("synthetic schema initialization interruption") + + def tracked_connect(*args, **kwargs): + connection = original_connect(*args, factory=TrackingConnection, **kwargs) + opened.append(connection) + return connection + + def interrupt_initialization(_self): + raise interruption + + monkeypatch.setattr(sqlite3, "connect", tracked_connect) + monkeypatch.setattr( + DatabaseAccessor, "_initialize_schema_cache", interrupt_initialization + ) + + with pytest.raises(interruption_type) as error: + DatabaseAccessor(path) + + assert error.value is interruption + assert len(opened) == 1 + assert opened[0].was_closed diff --git a/tests/unit/test_entity_source_coverage.py b/tests/unit/test_entity_source_coverage.py new file mode 100644 index 0000000..da92ee9 --- /dev/null +++ b/tests/unit/test_entity_source_coverage.py @@ -0,0 +1,423 @@ +from pathlib import Path +import sqlite3 + +import pytest + +from moneywiz_api import MoneywizApi +from moneywiz_api.database_accessor import DatabaseAccessor, DatabaseSchemaError +from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.managers.category_manager import CategoryManager +from moneywiz_api.managers.investment_holding_manager import ( + InvestmentHoldingManager, +) +from moneywiz_api.managers.payee_manager import PayeeManager +from moneywiz_api.managers.tag_manager import TagManager +from moneywiz_api.managers.transaction_manager import TransactionManager + + +UNCLASSIFIABLE = "database contains rows with unclassifiable entity ancestry" + + +def common_row(record_id: int, ent_id: int | None, gid: str) -> dict: + return { + "Z_PK": record_id, + "Z_ENT": ent_id, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": gid, + } + + +def account_row() -> dict: + return { + **common_row(1, 10, "account-1"), + "ZDISPLAYORDER": 1, + "ZGROUPID": 1, + "ZNAME": "Account", + "ZCURRENCYNAME": "EUR", + "ZOPENINGBALANCE": 0.0, + "ZINFO": None, + "ZUSER": 1, + } + + +def payee_row() -> dict: + return { + **common_row(1, 28, "payee-1"), + "ZNAME5": "Payee", + "ZUSER7": 1, + } + + +def category_row() -> dict: + return { + **common_row(1, 19, "category-1"), + "ZNAME2": "Category", + "ZPARENTCATEGORY": None, + "ZTYPE2": 1, + "ZUSER3": 1, + } + + +def holding_row() -> dict: + return { + **common_row(1, 24, "holding-1"), + "ZINVESTMENTACCOUNT": 1, + "ZOPENNINGNUMBEROFSHARES": None, + "ZNUMBEROFSHARES": 2.0, + "ZPRICEPERSHARE": 10.0, + "ZSYMBOL": "ACME", + "ZHOLDINGTYPE": None, + "ZDESC": "Acme", + "ZISPRICEPERSHAREAVAILABLEONLINE": 0, + "ZINVESTMENTOBJECTTYPE": 0, + "ZCOSTBASISOFMISSINGOBSHARES": 0.0, + } + + +def tag_row() -> dict: + return { + **common_row(1, 35, "tag-1"), + "ZNAME6": "Tag", + "ZUSER8": 1, + } + + +def transaction_row() -> dict: + return { + **common_row(1, 38, "transaction-1"), + "ZRECONCILED": 0, + "ZAMOUNT1": 5.0, + "ZDESC2": "Deposit", + "ZDATE1": 0.0, + "ZNOTES1": None, + "ZACCOUNT2": 1, + "ZPAYEE2": None, + "ZORIGINALCURRENCY": "EUR", + "ZORIGINALAMOUNT": 5.0, + "ZORIGINALEXCHANGERATE": 1.0, + } + + +MANAGER_CASES = [ + pytest.param( + AccountManager, "Account", 9, "CashAccount", 10, account_row, id="account" + ), + pytest.param(PayeeManager, "Payee", 28, "Payee", 28, payee_row, id="payee"), + pytest.param( + CategoryManager, "Category", 19, "Category", 19, category_row, id="category" + ), + pytest.param( + InvestmentHoldingManager, + "InvestmentHolding", + 24, + "InvestmentHolding", + 24, + holding_row, + id="investment-holding", + ), + pytest.param(TagManager, "Tag", 35, "Tag", 35, tag_row, id="tag"), + pytest.param( + TransactionManager, + "Transaction", + 37, + "DepositTransaction", + 38, + transaction_row, + id="transaction", + ), +] + + +PARTIALLY_MIGRATED_HIERARCHIES = [ + pytest.param( + AccountManager, + "Account", + "CashAccount", + 10, + account_row, + id="account", + ), + pytest.param( + TransactionManager, + "Transaction", + "DepositTransaction", + 38, + transaction_row, + id="transaction", + ), +] + + +def create_store(path: Path, metadata: list[tuple[int, str, int]], rows: list[dict]): + connection = sqlite3.connect(path) + connection.execute("PRAGMA journal_mode=WAL") + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.executemany("INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, 0)", metadata) + columns = list( + dict.fromkeys( + ["Z_PK", "Z_ENT", "ZOBJECTCREATIONDATE", "ZGID"] + + [column for row in rows for column in row] + + ["ZNUMBEROFSHARES", "ZPRICEPERSHARE"] + ) + ) + definitions = ", ".join( + f'"{column}" INTEGER' if column in {"Z_PK", "Z_ENT"} else f'"{column}"' + for column in columns + ) + connection.execute(f"CREATE TABLE ZSYNCOBJECT ({definitions})") + for row in rows: + insert_row(connection, row) + connection.commit() + connection.close() + + +def insert_row(connection: sqlite3.Connection, row: dict) -> None: + columns = tuple(row) + names = ", ".join(f'"{column}"' for column in columns) + placeholders = ", ".join("?" for _ in columns) + connection.execute( + f"INSERT INTO ZSYNCOBJECT ({names}) VALUES ({placeholders})", + tuple(row[column] for column in columns), + ) + + +@pytest.mark.parametrize( + ("manager_type", "root_name", "root_id", "known_name", "known_id", "row_factory"), + MANAGER_CASES, +) +@pytest.mark.parametrize("transitive", [False, True], ids=["direct", "transitive"]) +def test_all_managers_count_unknown_descendants( + tmp_path, + manager_type, + root_name, + root_id, + known_name, + known_id, + row_factory, + transitive, +) -> None: + path = tmp_path / f"{root_name}-{transitive}.sqlite" + metadata = [(8, "SyncObject", 0), (root_id, root_name, 8)] + if known_id != root_id: + metadata.append((known_id, known_name, root_id)) + unknown_parent = root_id + if transitive: + metadata.append((90, f"Future{root_name}Parent", root_id)) + unknown_parent = 90 + unknown_name = f"Future{root_name}" + metadata.append((91, unknown_name, unknown_parent)) + create_store( + path, + metadata, + [row_factory(), common_row(2, 91, f"future-{root_name.lower()}")], + ) + + with DatabaseAccessor(path) as accessor: + report = manager_type().load(accessor) + + assert set(report.source_ids) == {1, 2} + assert report.parsed_ids == (1,) + assert len(report.skipped) == 1 + assert report.skipped[0].record_id == 2 + assert report.skipped[0].entity == unknown_name + assert report.skipped[0].error.value == "unknown_entity" + + +@pytest.mark.parametrize( + ("manager_type", "missing_root", "known_name", "known_id", "row_factory"), + PARTIALLY_MIGRATED_HIERARCHIES, +) +def test_unknown_descendant_is_counted_when_abstract_root_is_absent( + tmp_path, + manager_type, + missing_root, + known_name, + known_id, + row_factory, +) -> None: + path = tmp_path / f"missing-{missing_root}.sqlite" + future_name = f"Future{known_name}" + create_store( + path, + [ + (8, "SyncObject", 0), + (known_id, known_name, 8), + (91, future_name, known_id), + ], + [row_factory(), common_row(2, 91, f"future-{known_name.lower()}")], + ) + + with DatabaseAccessor(path) as accessor: + assert accessor.ent_for(missing_root) is None + report = manager_type().load(accessor) + + assert set(report.source_ids) == {1, 2} + assert report.parsed_ids == (1,) + assert len(report.skipped) == 1 + assert report.skipped[0].record_id == 2 + assert report.skipped[0].entity == future_name + assert report.skipped[0].error.value == "unknown_entity" + assert not report.complete + + +def orphan_metadata(kind: str) -> tuple[list[tuple[int, str, int]], int | None]: + metadata = [(8, "SyncObject", 0), (9, "Account", 8), (10, "CashAccount", 9)] + if kind == "null": + return metadata, None + if kind == "unmapped": + return metadata, 99 + if kind == "missing-parent": + return [*metadata, (90, "FuturePayee", 99)], 90 + if kind == "cycle": + return [*metadata, (90, "FuturePayee", 91), (91, "FuturePayeeParent", 90)], 90 + raise AssertionError("unsupported synthetic orphan kind") + + +@pytest.mark.parametrize("kind", ["null", "unmapped", "missing-parent", "cycle"]) +@pytest.mark.parametrize("include_valid", [False, True], ids=["orphan-only", "mixed"]) +def test_unclassifiable_observed_entity_refuses_source_read( + tmp_path, kind, include_valid +) -> None: + path = tmp_path / f"{kind}-{include_valid}.sqlite" + metadata, ent_id = orphan_metadata(kind) + rows = [common_row(2, ent_id, "orphan")] + if include_valid: + rows.insert(0, account_row()) + create_store(path, metadata, rows) + + with DatabaseAccessor(path) as accessor: + with pytest.raises(DatabaseSchemaError, match=f"^{UNCLASSIFIABLE}$"): + accessor.query_objects(["CashAccount"]) + + +@pytest.mark.parametrize( + "operation", + [ + lambda accessor: accessor.query_objects(["CashAccount"]), + lambda accessor: accessor.get_record(1), + lambda accessor: accessor.get_record_by_gid("account-1"), + lambda accessor: accessor.read_category_assignments(), + lambda accessor: accessor.get_category_assignment(), + lambda accessor: accessor.read_refund_maps(), + lambda accessor: accessor.get_refund_maps(), + lambda accessor: accessor.read_tags_map(), + lambda accessor: accessor.get_tags_map(), + ], +) +def test_every_public_cache_dependent_reader_refuses_orphan( + tmp_path, operation +) -> None: + path = tmp_path / "reader-orphan.sqlite" + metadata, ent_id = orphan_metadata("unmapped") + create_store(path, metadata, [account_row(), common_row(2, ent_id, "orphan")]) + + with DatabaseAccessor(path) as accessor: + with pytest.raises(DatabaseSchemaError, match=f"^{UNCLASSIFIABLE}$"): + operation(accessor) + + +def test_unobserved_broken_metadata_and_absent_optional_roots_are_allowed( + tmp_path, +) -> None: + path = tmp_path / "unused-broken-metadata.sqlite" + create_store(path, [(8, "SyncObject", 0), (90, "FutureUnused", 99)], []) + + with DatabaseAccessor(path) as accessor: + reports = [ + manager_type().load(accessor) + for manager_type in ( + AccountManager, + PayeeManager, + CategoryManager, + TransactionManager, + InvestmentHoldingManager, + TagManager, + ) + ] + + assert all(report.source_count == 0 and report.complete for report in reports) + + +def test_known_unrelated_malformed_row_does_not_block_account_scope(tmp_path) -> None: + path = tmp_path / "known-unrelated.sqlite" + metadata = [ + (8, "SyncObject", 0), + (9, "Account", 8), + (10, "CashAccount", 9), + (80, "KnownUnrelated", 8), + ] + create_store(path, metadata, [account_row(), common_row(2, 80, "unrelated")]) + + with DatabaseAccessor(path) as accessor: + report = AccountManager().load(accessor) + + assert report.complete + assert report.source_ids == (1,) + assert report.parsed_ids == (1,) + + +def test_api_initial_orphan_failure_closes_accessor(tmp_path, monkeypatch) -> None: + import moneywiz_api.moneywiz_api as api_module + + path = tmp_path / "initial-orphan.sqlite" + metadata, ent_id = orphan_metadata("unmapped") + create_store(path, metadata, [common_row(2, ent_id, "orphan")]) + accessor = DatabaseAccessor(path) + monkeypatch.setattr(api_module, "DatabaseAccessor", lambda _path: accessor) + + with pytest.raises(DatabaseSchemaError, match=f"^{UNCLASSIFIABLE}$"): + api_module.MoneywizApi(path, managers=("accounts",)) + + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + accessor._con.execute("SELECT 1") + + +def test_same_schema_orphan_reload_preserves_generation_and_can_retry(tmp_path) -> None: + path = tmp_path / "reload-orphan.sqlite" + metadata, _ = orphan_metadata("unmapped") + create_store(path, metadata, [account_row()]) + + with MoneywizApi(path, managers=("accounts",)) as api: + manager = api.account_manager + record = manager.get(1) + report = manager.load_report + writer = sqlite3.connect(path) + insert_row(writer, common_row(2, 99, "orphan")) + writer.commit() + writer.close() + + with pytest.raises(DatabaseSchemaError, match=f"^{UNCLASSIFIABLE}$"): + api.load(("accounts",)) + + assert api.account_manager is manager + assert manager.get(1) is record + assert manager.load_report is report + + writer = sqlite3.connect(path) + writer.execute("DELETE FROM ZSYNCOBJECT WHERE Z_PK = 2") + writer.commit() + writer.close() + + retry = api.load(("accounts",)).managers["accounts"] + + assert retry.complete + assert retry.source_ids == (1,) + + +def test_explicit_query_names_do_not_expand_to_other_known_scopes(tmp_path) -> None: + path = tmp_path / "exact-query.sqlite" + metadata = [ + (8, "SyncObject", 0), + (9, "Account", 8), + (10, "CashAccount", 9), + (28, "Payee", 8), + ] + create_store(path, metadata, [account_row(), payee_row()]) + + with DatabaseAccessor(path) as accessor: + rows = accessor.query_objects(["CashAccount"]) + + assert [row["Z_PK"] for row in rows] == [1] diff --git a/tests/unit/test_identity_domains.py b/tests/unit/test_identity_domains.py new file mode 100644 index 0000000..c294118 --- /dev/null +++ b/tests/unit/test_identity_domains.py @@ -0,0 +1,545 @@ +from contextlib import contextmanager +from decimal import Decimal +import json + +import pytest + +from moneywiz_api.managers.transaction_manager import TransactionManager +from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.managers.category_manager import CategoryManager +from moneywiz_api.managers.investment_holding_manager import InvestmentHoldingManager +from moneywiz_api.managers.payee_manager import PayeeManager +from moneywiz_api.managers.tag_manager import TagManager +from moneywiz_api.model.account import CashAccount +from moneywiz_api.model.category import Category +from moneywiz_api.model.investment_holding import InvestmentHolding +from moneywiz_api.model.payee import Payee +from moneywiz_api.model.record import Record +from moneywiz_api.model.tag import Tag +from moneywiz_api.model.transaction import ( + DepositTransaction, + InvestmentBuyTransaction, + InvestmentExchangeTransaction, + InvestmentSellTransaction, + ReconcileTransaction, + RefundTransaction, + TransferBudgetTransaction, + TransferDepositTransaction, + TransferWithdrawTransaction, + WithdrawTransaction, +) +from moneywiz_api.read_result import RelationshipLoadReport, RelationshipStorage +from moneywiz_api.schema_profile import SchemaProfile +from tests.unit.accessor_test_support import initialized_memory_accessor + + +INVALID_IDENTITIES = (True, 1.0, "PRIVATE_PAYLOAD", Decimal("1")) +PROFILE = SchemaProfile( + "unsuffixed-investment-columns", + "ZNUMBEROFSHARES", + "ZNUMBEROFSHARES", + "ZPRICEPERSHARE", + "ZPRICEPERSHARE", +) + + +def record_row(**overrides): + row = { + "Z_ENT": 37, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": "record-1", + "Z_PK": 1, + } + row.update(overrides) + return row + + +def account_row(**overrides): + row = { + **record_row(Z_ENT=10, ZGID="account-1"), + "ZDISPLAYORDER": 0, + "ZGROUPID": 1, + "ZNAME": "Cash", + "ZCURRENCYNAME": "EUR", + "ZOPENINGBALANCE": 0.0, + "ZINFO": None, + "ZUSER": 2, + } + row.update(overrides) + return row + + +def payee_row(**overrides): + row = { + **record_row(Z_ENT=28, ZGID="payee-1"), + "ZNAME5": "Payee", + "ZUSER7": 2, + } + row.update(overrides) + return row + + +def category_row(**overrides): + row = { + **record_row(Z_ENT=19, ZGID="category-1"), + "ZNAME2": "Category", + "ZPARENTCATEGORY": None, + "ZTYPE2": 1, + "ZUSER3": 2, + } + row.update(overrides) + return row + + +def tag_row(**overrides): + row = { + **record_row(Z_ENT=35, ZGID="tag-1"), + "ZNAME6": "Tag", + "ZUSER8": 2, + } + row.update(overrides) + return row + + +def holding_row(**overrides): + row = { + **record_row(Z_ENT=24, ZGID="holding-1"), + "ZINVESTMENTACCOUNT": 2, + "ZOPENNINGNUMBEROFSHARES": None, + "ZNUMBEROFSHARES": 2.0, + "ZPRICEPERSHARE": 10.0, + "ZSYMBOL": "ACME", + "ZHOLDINGTYPE": None, + "ZDESC": "Acme", + "ZISPRICEPERSHAREAVAILABLEONLINE": 0, + "ZINVESTMENTOBJECTTYPE": 0, + "ZCOSTBASISOFMISSINGOBSHARES": 0.0, + } + row.update(overrides) + return row + + +def transaction_row(**overrides): + row = { + **record_row(Z_ENT=37, ZGID="transaction-1"), + "ZRECONCILED": 0, + "ZAMOUNT1": 10.0, + "ZDESC2": "Transaction", + "ZDATE1": 0.0, + "ZNOTES1": None, + "ZACCOUNT2": 2, + "ZPAYEE2": None, + "ZORIGINALCURRENCY": "EUR", + "ZORIGINALAMOUNT": 10.0, + "ZORIGINALEXCHANGERATE": 1.0, + } + row.update(overrides) + return row + + +def exchange_row(**overrides): + row = { + **transaction_row(Z_ENT=38, ZAMOUNT1=0.0), + "ZFROMINVESTMENTHOLDING": 3, + "ZFROMSYMBOL": "AAA", + "ZTOINVESTMENTHOLDING": 4, + "ZTOSYMBOL": "BBB", + "ZFROMNUMBEROFSHARES": -2.0, + "ZTONUMBEROFSHARES": 1.0, + "ZORIGINALFEE": 0.0, + "ZORIGINALFEECURRENCY": "AAA", + } + row.update(overrides) + return row + + +def buy_row(**overrides): + row = { + **transaction_row(Z_ENT=40, ZAMOUNT1=-20.0), + "ZFEE2": 0.0, + "ZINVESTMENTHOLDING": 3, + "ZNUMBEROFSHARES1": 2.0, + "ZPRICEPERSHARE1": 10.0, + } + row.update(overrides) + return row + + +def sell_row(**overrides): + row = { + **buy_row(Z_ENT=41, ZAMOUNT1=20.0), + } + row.update(overrides) + return row + + +def reconcile_row(**overrides): + row = { + **transaction_row(Z_ENT=42, ZAMOUNT1=0.0), + "ZRECONCILEAMOUNT": 0.0, + "ZRECONCILENUMBEROFSHARES": None, + } + row.update(overrides) + return row + + +def refund_row(**overrides): + row = {**transaction_row(Z_ENT=43), "ZPAYEE2": 5} + row.update(overrides) + return row + + +def transfer_budget_row(**overrides): + row = transaction_row(Z_ENT=44) + row.update(overrides) + return row + + +def transfer_deposit_row(**overrides): + row = { + **transaction_row(Z_ENT=45), + "ZSENDERACCOUNT": 3, + "ZSENDERTRANSACTION": 4, + "ZORIGINALSENDERAMOUNT": -10.0, + "ZORIGINALSENDERCURRENCY": "EUR", + "ZORIGINALFEE": None, + "ZORIGINALFEECURRENCY": None, + } + row.update(overrides) + return row + + +def transfer_withdraw_row(**overrides): + row = { + **transaction_row(Z_ENT=46, ZAMOUNT1=-10.0, ZORIGINALAMOUNT=-10.0), + "ZRECIPIENTACCOUNT1": 3, + "ZRECIPIENTTRANSACTION": 4, + "ZORIGINALRECIPIENTAMOUNT": 10.0, + "ZORIGINALRECIPIENTCURRENCY": "EUR", + "ZORIGINALFEE": None, + "ZORIGINALFEECURRENCY": None, + } + row.update(overrides) + return row + + +def withdraw_row(**overrides): + row = transaction_row(Z_ENT=47, ZAMOUNT1=-10.0, ZORIGINALAMOUNT=-10.0) + row.update(overrides) + return row + + +IDENTITY_FIELDS = ( + (Record, record_row, "Z_PK"), + (Record, record_row, "Z_ENT"), + (CashAccount, account_row, "ZGROUPID"), + (CashAccount, account_row, "ZUSER"), + (Payee, payee_row, "ZUSER7"), + (Category, category_row, "ZPARENTCATEGORY"), + (Category, category_row, "ZUSER3"), + (Tag, tag_row, "ZUSER8"), + (InvestmentHolding, holding_row, "ZINVESTMENTACCOUNT"), + (DepositTransaction, transaction_row, "ZACCOUNT2"), + (DepositTransaction, transaction_row, "ZPAYEE2"), + (InvestmentExchangeTransaction, exchange_row, "ZACCOUNT2"), + (InvestmentExchangeTransaction, exchange_row, "ZFROMINVESTMENTHOLDING"), + (InvestmentExchangeTransaction, exchange_row, "ZTOINVESTMENTHOLDING"), + (InvestmentBuyTransaction, buy_row, "ZACCOUNT2"), + (InvestmentBuyTransaction, buy_row, "ZINVESTMENTHOLDING"), + (InvestmentSellTransaction, sell_row, "ZACCOUNT2"), + (InvestmentSellTransaction, sell_row, "ZINVESTMENTHOLDING"), + (ReconcileTransaction, reconcile_row, "ZACCOUNT2"), + (RefundTransaction, refund_row, "ZACCOUNT2"), + (RefundTransaction, refund_row, "ZPAYEE2"), + (TransferDepositTransaction, transfer_deposit_row, "ZACCOUNT2"), + (TransferDepositTransaction, transfer_deposit_row, "ZSENDERACCOUNT"), + (TransferDepositTransaction, transfer_deposit_row, "ZSENDERTRANSACTION"), + (TransferWithdrawTransaction, transfer_withdraw_row, "ZACCOUNT2"), + (TransferWithdrawTransaction, transfer_withdraw_row, "ZRECIPIENTACCOUNT1"), + (TransferWithdrawTransaction, transfer_withdraw_row, "ZRECIPIENTTRANSACTION"), + (WithdrawTransaction, withdraw_row, "ZACCOUNT2"), + (WithdrawTransaction, withdraw_row, "ZPAYEE2"), +) + + +@pytest.mark.parametrize(("constructor", "row_factory", "field"), IDENTITY_FIELDS) +@pytest.mark.parametrize("invalid_identity", INVALID_IDENTITIES) +def test_model_identity_fields_reject_coerced_domains( + constructor, row_factory, field, invalid_identity +) -> None: + with pytest.raises(AssertionError, match="uncoerced integer"): + constructor(row_factory(**{field: invalid_identity})) + + +@pytest.mark.parametrize(("constructor", "row_factory", "field"), IDENTITY_FIELDS) +def test_model_identity_fields_preserve_valid_integers( + constructor, row_factory, field +) -> None: + constructor(row_factory(**{field: 7})) + + +@pytest.mark.parametrize("gid", ["record-1", "Δοκιμή", " ", "id:/?!"]) +def test_record_gid_preserves_nonempty_exact_strings(gid) -> None: + assert Record(record_row(ZGID=gid)).gid == gid + + +@pytest.mark.parametrize("gid", [None, "", b"PRIVATE_PAYLOAD", 1, True, object()]) +def test_record_gid_refuses_empty_or_coerced_values_without_payload(gid) -> None: + with pytest.raises(AssertionError) as error: + Record(record_row(ZGID=gid)) + + assert "PRIVATE_PAYLOAD" not in str(error.value) + + +@pytest.mark.parametrize( + ("constructor", "row_factory", "field"), + [ + (CashAccount, account_row, "ZNAME"), + (Payee, payee_row, "ZNAME5"), + (Category, category_row, "ZNAME2"), + (Tag, tag_row, "ZNAME6"), + ], +) +def test_native_named_entities_preserve_empty_strings_and_reject_blobs( + constructor, row_factory, field +) -> None: + assert constructor(row_factory(**{field: ""})).name == "" + with pytest.raises(AssertionError, match="uncoerced string"): + constructor(row_factory(**{field: b"PRIVATE_PAYLOAD"})) + + +def test_account_info_preserves_nullable_and_empty_text_contract() -> None: + assert CashAccount(account_row(ZINFO=None)).info is None + assert CashAccount(account_row(ZINFO="")).info == "" + with pytest.raises(AssertionError, match="uncoerced string"): + CashAccount(account_row(ZINFO=b"PRIVATE_PAYLOAD")) + + +@pytest.mark.parametrize( + ("constructor", "row_factory", "field"), + [ + (CashAccount, account_row, "ZGROUPID"), + (CashAccount, account_row, "ZUSER"), + (Payee, payee_row, "ZUSER7"), + (Category, category_row, "ZPARENTCATEGORY"), + (Tag, tag_row, "ZUSER8"), + (InvestmentHolding, holding_row, "ZINVESTMENTACCOUNT"), + (DepositTransaction, transaction_row, "ZACCOUNT2"), + (DepositTransaction, transaction_row, "ZPAYEE2"), + ], +) +def test_non_record_identity_fields_preserve_integer_zero( + constructor, row_factory, field +) -> None: + constructor(row_factory(**{field: 0})) + + +@pytest.mark.parametrize( + ("constructor", "row_factory", "field"), + [ + (Category, category_row, "ZPARENTCATEGORY"), + (DepositTransaction, transaction_row, "ZPAYEE2"), + (RefundTransaction, refund_row, "ZPAYEE2"), + (WithdrawTransaction, withdraw_row, "ZPAYEE2"), + ], +) +def test_optional_identity_fields_accept_none(constructor, row_factory, field) -> None: + constructor(row_factory(**{field: None})) + + +TRANSACTION_ROWS = ( + (DepositTransaction, transaction_row), + (InvestmentExchangeTransaction, exchange_row), + (InvestmentBuyTransaction, buy_row), + (InvestmentSellTransaction, sell_row), + (ReconcileTransaction, reconcile_row), + (RefundTransaction, refund_row), + (TransferBudgetTransaction, transfer_budget_row), + (TransferDepositTransaction, transfer_deposit_row), + (TransferWithdrawTransaction, transfer_withdraw_row), + (WithdrawTransaction, withdraw_row), +) + + +@pytest.mark.parametrize(("constructor", "row_factory"), TRANSACTION_ROWS) +@pytest.mark.parametrize(("raw_value", "expected"), [(0, False), (1, True)]) +def test_reconciled_raw_integer_domain_is_preserved( + constructor, row_factory, raw_value, expected +) -> None: + assert constructor(row_factory(ZRECONCILED=raw_value)).reconciled is expected + + +@pytest.mark.parametrize(("constructor", "row_factory"), TRANSACTION_ROWS) +@pytest.mark.parametrize("raw_value", [None, True, 2, 1.0, "PRIVATE_PAYLOAD"]) +def test_reconciled_refuses_non_binary_or_coerced_values( + constructor, row_factory, raw_value +) -> None: + with pytest.raises(AssertionError) as error: + constructor(row_factory(ZRECONCILED=raw_value)) + assert "PRIVATE_PAYLOAD" not in str(error.value) + + +def test_manager_skips_invalid_reconciled_row_and_keeps_valid_row() -> None: + rows = [ + transaction_row(ZRECONCILED=2), + transaction_row(Z_PK=2, ZGID="transaction-2", ZRECONCILED=1), + ] + + report = TransactionManager().load(TransactionAccessor(rows)) + + assert report.source_ids == (1, 2) + assert report.parsed_ids == (2,) + assert report.skipped[0].error.value == "validation" + assert report.status == "partial" + + +@pytest.mark.parametrize(("raw_value", "expected"), [(0, False), (1, True)]) +def test_accessor_preserves_binary_reconciled_domain(raw_value, expected) -> None: + accessor = initialized_memory_accessor( + [transaction_row(ZRECONCILED=raw_value)], + [(37, "DepositTransaction", 0)], + ) + + transaction = accessor.get_record(1, DepositTransaction) + + assert transaction.reconciled is expected + + +@pytest.mark.parametrize("raw_value", [None, 2, 1.0, "PRIVATE_PAYLOAD"]) +def test_accessor_refuses_invalid_reconciled_domain(raw_value) -> None: + accessor = initialized_memory_accessor( + [transaction_row(ZRECONCILED=raw_value)], + [(37, "DepositTransaction", 0)], + ) + + with pytest.raises(AssertionError): + accessor.get_record(1, DepositTransaction) + + +def test_accessor_enforces_exact_string_gid_domain() -> None: + accessor = initialized_memory_accessor( + [record_row(ZGID=b"PRIVATE_PAYLOAD")], + [(37, "Record", 0)], + ) + + with pytest.raises(AssertionError, match="uncoerced string"): + accessor.get_record(1) + accessor.close() + + +class TransactionAccessor: + def __init__(self, rows): + self.rows = rows + + @contextmanager + def read_transaction(self): + yield + + def descendant_typenames(self, _roots): + return ["DepositTransaction"] + + def query_objects(self, _typenames): + return self.rows + + def typename_for(self, ent_id): + return {37: "DepositTransaction"}.get(ent_id) + + def read_category_assignments(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def read_refund_maps(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def read_tags_map(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + +class ManagerAccessor(TransactionAccessor): + def __init__(self, rows, typenames): + super().__init__(rows) + self.typenames = typenames + self.schema_profile = PROFILE + + def descendant_typenames(self, _roots): + return list(self.typenames.values()) + + def typename_for(self, ent_id): + return self.typenames.get(ent_id) + + +@pytest.mark.parametrize( + ("manager_type", "row_factory", "entity", "entity_id"), + [ + (AccountManager, account_row, "CashAccount", 10), + (PayeeManager, payee_row, "Payee", 28), + (CategoryManager, category_row, "Category", 19), + (InvestmentHoldingManager, holding_row, "InvestmentHolding", 24), + (TransactionManager, transaction_row, "DepositTransaction", 37), + (TagManager, tag_row, "Tag", 35), + ], +) +def test_all_managers_skip_wrong_type_gid_and_keep_valid_row( + manager_type, row_factory, entity, entity_id +) -> None: + invalid = row_factory(ZGID=b"PRIVATE_PAYLOAD") + valid = row_factory(Z_PK=2, ZGID="valid-2") + + report = manager_type().load(ManagerAccessor([invalid, valid], {entity_id: entity})) + + assert report.source_ids == (1, 2) + assert report.parsed_ids == (2,) + assert report.skipped[0].error.value == "validation" + assert report.status == "partial" + assert "PRIVATE_PAYLOAD" not in json.dumps(report.as_dict(), sort_keys=True) + + +@pytest.mark.parametrize( + ("manager_type", "row_factory", "field", "entity", "entity_id"), + [ + (AccountManager, account_row, "ZNAME", "CashAccount", 10), + (PayeeManager, payee_row, "ZNAME5", "Payee", 28), + (CategoryManager, category_row, "ZNAME2", "Category", 19), + (TagManager, tag_row, "ZNAME6", "Tag", 35), + ], +) +def test_named_entity_managers_report_blob_names_as_validation( + manager_type, row_factory, field, entity, entity_id +) -> None: + report = manager_type().load( + ManagerAccessor( + [ + row_factory(**{field: b"PRIVATE_PAYLOAD"}), + row_factory(Z_PK=2, ZGID="valid-2", **{field: ""}), + ], + {entity_id: entity}, + ) + ) + + assert report.source_count == 2 + assert report.parsed_ids == (2,) + assert report.skipped[0].error.value == "validation" + + +@pytest.mark.parametrize( + "invalid_row", + [ + transaction_row(Z_PK="PRIVATE_PAYLOAD"), + transaction_row(Z_ENT="PRIVATE_PAYLOAD"), + transaction_row(ZACCOUNT2="PRIVATE_PAYLOAD"), + transaction_row(ZRECONCILED="PRIVATE_PAYLOAD"), + ], +) +def test_manager_diagnostics_do_not_publish_invalid_identity_payloads( + invalid_row, caplog +) -> None: + valid_row = transaction_row(Z_PK=2, ZGID="transaction-2") + caplog.set_level("DEBUG") + + report = TransactionManager().load(TransactionAccessor([invalid_row, valid_row])) + serialized = json.dumps(report.as_dict(), sort_keys=True) + + assert report.source_count == 2 + assert report.parsed_ids == (2,) + assert len(report.skipped) == 1 + assert "PRIVATE_PAYLOAD" not in serialized + assert "PRIVATE_PAYLOAD" not in caplog.text diff --git a/tests/unit/test_manager_transaction_lifecycle.py b/tests/unit/test_manager_transaction_lifecycle.py new file mode 100644 index 0000000..73f9065 --- /dev/null +++ b/tests/unit/test_manager_transaction_lifecycle.py @@ -0,0 +1,212 @@ +from contextlib import contextmanager +from decimal import Decimal +import sqlite3 + +import pytest + +from moneywiz_api.database_accessor import DatabaseAccessor +from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.managers.category_manager import CategoryManager +from moneywiz_api.managers.investment_holding_manager import ( + InvestmentHoldingManager, +) +from moneywiz_api.managers.payee_manager import PayeeManager +from moneywiz_api.managers.tag_manager import TagManager +from moneywiz_api.managers.transaction_manager import TransactionManager +from moneywiz_api.read_result import RelationshipLoadReport, RelationshipStorage + + +BUILTIN_MANAGERS = ( + AccountManager, + PayeeManager, + CategoryManager, + TransactionManager, + InvestmentHoldingManager, + TagManager, +) + + +class EmptyAccessor: + def __init__(self): + self.entries = 0 + self.exits = 0 + self.depth = 0 + self.max_depth = 0 + self.fail = None + + @contextmanager + def read_transaction(self): + if self.fail == "enter": + raise RuntimeError("synthetic transaction enter failure") + self.entries += 1 + self.depth += 1 + self.max_depth = max(self.max_depth, self.depth) + try: + yield + finally: + self.depth -= 1 + self.exits += 1 + if self.fail == "exit": + raise RuntimeError("synthetic transaction exit failure") + + def descendant_typenames(self, _roots): + assert self.depth == 1 + return [] + + def query_objects(self, _typenames): + assert self.depth == 1 + if self.fail == "query": + raise RuntimeError("synthetic query failure") + return [] + + def read_category_assignments(self): + assert self.depth == 1 + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def read_refund_maps(self): + assert self.depth == 1 + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def read_tags_map(self): + assert self.depth == 1 + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + +@pytest.mark.parametrize("manager_type", BUILTIN_MANAGERS) +def test_each_public_manager_load_owns_one_outer_transaction(manager_type) -> None: + accessor = EmptyAccessor() + manager = manager_type() + + report = manager.load(accessor) + + assert report.complete + assert accessor.entries == accessor.exits == 1 + assert accessor.max_depth == 1 + + +@pytest.mark.parametrize("manager_type", BUILTIN_MANAGERS) +@pytest.mark.parametrize("failure", ["enter", "query", "exit"]) +def test_each_public_manager_failure_discards_observation( + manager_type, failure +) -> None: + accessor = EmptyAccessor() + manager = manager_type() + assert manager.load(accessor).complete + accessor.fail = failure + + with pytest.raises(RuntimeError, match="synthetic"): + manager.load(accessor) + + assert manager.records() == {} + assert manager.load_report.status == "unloaded" + if isinstance(manager, TransactionManager): + assert manager.category_assignment == {} + assert manager.refund_maps == {} + assert manager.tags_map == {} + accessor.fail = None + assert manager.load(accessor).complete + + +def _create_wal_transaction_store(path) -> None: + connection = sqlite3.connect(path) + connection.execute("PRAGMA journal_mode=WAL") + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, 0)", + [ + (2, "CategoryAssigment", 0), + (8, "SyncObject", 0), + (35, "Tag", 8), + (36, "Transaction", 8), + (37, "DepositTransaction", 36), + (50, "WithdrawRefundTransactionLink", 0), + ], + ) + connection.execute( + "CREATE TABLE ZSYNCOBJECT (" + "Z_PK INTEGER, Z_ENT INTEGER, ZOBJECTCREATIONDATE FLOAT, ZGID TEXT, " + "ZRECONCILED INTEGER, ZAMOUNT1 FLOAT, ZDESC2 TEXT, ZDATE1 FLOAT, " + "ZNOTES1 TEXT, ZACCOUNT2 INTEGER, ZPAYEE2 INTEGER, " + "ZORIGINALCURRENCY TEXT, ZORIGINALAMOUNT FLOAT, " + "ZORIGINALEXCHANGERATE FLOAT)" + ) + connection.execute( + "INSERT INTO ZSYNCOBJECT VALUES " + "(100, 37, 0, 'transaction-100', 0, 5, 'old', 0, NULL, 1, NULL, " + "'EUR', 5, 1)" + ) + connection.execute( + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)" + ) + connection.execute("INSERT INTO ZCATEGORYASSIGMENT VALUES (1, 7, 100, 5)") + connection.execute( + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, ZWITHDRAWTRANSACTION INTEGER)" + ) + connection.execute("INSERT INTO ZWITHDRAWREFUNDTRANSACTIONLINK VALUES (2, 100, 90)") + connection.execute( + "CREATE TABLE Z_36TAGS (Z_36TRANSACTIONS INTEGER, Z_35TAGS INTEGER)" + ) + connection.execute("INSERT INTO Z_36TAGS VALUES (100, 9)") + connection.commit() + connection.close() + + +def _mutate_wal_transaction_store(path) -> None: + connection = sqlite3.connect(path) + connection.execute("UPDATE ZSYNCOBJECT SET ZAMOUNT1 = 6, ZORIGINALAMOUNT = 6") + connection.execute("UPDATE ZCATEGORYASSIGMENT SET ZAMOUNT = 7") + connection.execute( + "UPDATE ZWITHDRAWREFUNDTRANSACTIONLINK SET ZWITHDRAWTRANSACTION = 91" + ) + connection.execute("UPDATE Z_36TAGS SET Z_35TAGS = 10") + connection.commit() + connection.close() + + +@pytest.mark.parametrize( + "interleave_after", + ["transaction_rows", "category_assignments", "refund_links"], +) +def test_direct_transaction_manager_load_uses_one_wal_generation( + tmp_path, monkeypatch, interleave_after +) -> None: + path = tmp_path / "manager-generation.sqlite" + _create_wal_transaction_store(path) + + with DatabaseAccessor(path) as accessor: + method_name = { + "transaction_rows": "query_objects", + "category_assignments": "read_category_assignments", + "refund_links": "read_refund_maps", + }[interleave_after] + original = getattr(accessor, method_name) + mutated = False + + def read_then_mutate(*args, **kwargs): + nonlocal mutated + result = original(*args, **kwargs) + if not mutated: + _mutate_wal_transaction_store(path) + mutated = True + return result + + monkeypatch.setattr(accessor, method_name, read_then_mutate) + manager = TransactionManager() + report = manager.load(accessor) + + assert report.complete + assert all(item.complete for item in report.relationships.values()) + assert manager.get(100).amount == Decimal("5.0") + assert manager.category_assignment == {100: [(7, Decimal("5.0"))]} + assert manager.refund_maps == {100: 90} + assert manager.tags_map == {100: [9]} + + +def test_transaction_manager_uses_shared_public_load_boundary() -> None: + assert "load" not in TransactionManager.__dict__ + assert "_load_in_transaction" in TransactionManager.__dict__ diff --git a/tests/unit/test_optimized_validation.py b/tests/unit/test_optimized_validation.py new file mode 100644 index 0000000..68dc259 --- /dev/null +++ b/tests/unit/test_optimized_validation.py @@ -0,0 +1,201 @@ +import ast +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +PROBE = Path(__file__).with_name("optimized_validation_probe.py") +RUNTIME_SOURCE = Path(__file__).parents[2] / "src" / "moneywiz_api" + + +def run_probe(*options, optimize_environment=None): + environment = os.environ.copy() + environment.pop("PYTHONOPTIMIZE", None) + source_path = str(RUNTIME_SOURCE.parent) + existing_pythonpath = environment.get("PYTHONPATH") + environment["PYTHONPATH"] = ( + f"{source_path}{os.pathsep}{existing_pythonpath}" + if existing_pythonpath + else source_path + ) + if optimize_environment is not None: + environment["PYTHONOPTIMIZE"] = optimize_environment + completed = subprocess.run( + [sys.executable, *options, str(PROBE)], + check=True, + capture_output=True, + text=True, + env=environment, + ) + return json.loads(completed.stdout) + + +@pytest.fixture(scope="module") +def optimization_results(): + return [ + run_probe(), + run_probe("-O"), + run_probe("-OO"), + run_probe(optimize_environment="1"), + ] + + +def test_validation_results_are_optimization_invariant(optimization_results) -> None: + assert [result["optimize"] for result in optimization_results] == [0, 1, 2, 1] + baseline = optimization_results[0]["cases"] + assert all(result["cases"] == baseline for result in optimization_results[1:]) + + +def test_probe_covers_success_error_and_completeness_paths( + optimization_results, +) -> None: + cases = optimization_results[0]["cases"] + + assert cases["decimal_valid"] == {"status": "ok", "value": "1.25"} + assert cases["decimal_string"] == { + "status": "error", + "type": "AssertionError", + "message": "decimal field must be numeric", + } + assert cases["date_string"]["type"] == "AssertionError" + assert cases["record_invalid_identity"]["type"] == "AssertionError" + assert cases["record_coerced_gid"]["type"] == "AssertionError" + assert cases["record_coerced_identity"]["type"] == "AssertionError" + assert cases["account_nullable_info"]["status"] == "ok" + assert cases["account_invalid_name_public"]["type"] == "AssertionError" + assert cases["account_blob_name"]["type"] == "AssertionError" + assert cases["account_blob_info"]["type"] == "AssertionError" + assert cases["account_coerced_display_order"]["type"] == "AssertionError" + direct_accounts = cases["direct_accounts"] + assert all( + outcomes["valid"] == {"status": "ok", "value": None} + for outcomes in direct_accounts.values() + ) + assert all( + outcomes["invalid_name"]["type"] == "AssertionError" + for outcomes in direct_accounts.values() + ) + assert all( + direct_accounts[name]["invalid_statement_day"]["type"] == "AssertionError" + for name in ("CreditCardAccount", "LoanAccount") + ) + + category_types = cases["category_types"] + assert category_types["supported"]["1"]["value"] == "Expenses" + assert category_types["supported"]["2"]["value"] == "Income" + assert all( + report["skipped"][0]["error"] == "invalid_value" + for report in category_types["unsupported"].values() + ) + assert category_types["missing"]["skipped"][0]["error"] == "missing_field" + assert category_types["invalid_owner"]["skipped"][0]["error"] == "validation" + assert category_types["unexpected"]["skipped"][0]["error"] == "construction" + + named_entities = cases["named_entities"] + assert all( + named_entities[name]["status"] == "ok" + for name in ("payee_valid", "tag_valid", "category_valid") + ) + assert all( + named_entities[name]["type"] == "AssertionError" + for name in ( + "payee_invalid", + "tag_invalid", + "category_invalid", + "payee_blob_name", + "tag_blob_name", + "category_blob_name", + ) + ) + assert cases["manager"]["source_ids"] == [10, 11, None] + assert cases["manager"]["parsed_count"] == 1 + assert all( + skipped["error"] == "validation" for skipped in cases["manager"]["skipped"] + ) + assert cases["holding_missing_quantity"]["type"] == "AssertionError" + holding_online_price_state = cases["holding_online_price_state"] + assert holding_online_price_state["zero"] == {"status": "ok", "value": False} + assert holding_online_price_state["one"] == {"status": "ok", "value": True} + assert all( + holding_online_price_state[value]["type"] == "AssertionError" + for value in ( + "none", + "false_boolean", + "true_boolean", + "two", + "float", + "text", + ) + ) + assert cases["deposit_invalid_sign"]["type"] == "AssertionError" + assert cases["deposit_coerced_account"]["type"] == "AssertionError" + reconciled = cases["reconciled"] + assert reconciled["zero"] == {"status": "ok", "value": False} + assert reconciled["one"] == {"status": "ok", "value": True} + assert all( + reconciled[value]["type"] == "AssertionError" + for value in ( + "none", + "false_boolean", + "true_boolean", + "two", + "float", + "text", + ) + ) + assert cases["deposit_invalid_fx"]["type"] == "AssertionError" + assert cases["deposit_zero_rate"] == {"status": "ok", "value": None} + assert cases["investment_buy_invalid_quantity"]["type"] == "AssertionError" + assert cases["investment_buy_invalid_total"]["type"] == "AssertionError" + assert cases["transfer_zero_rate"]["type"] == "ValueError" + + relationships = cases["relationships"] + assert relationships["category_report"]["parsed_count"] == 1 + assert all( + item["error"] == "validation" + for item in relationships["category_report"]["skipped"] + ) + assert relationships["category_report"]["source_ids"][-1] is None + assert relationships["refund_report"]["parsed_count"] == 1 + assert relationships["refund_report"]["source_ids"][-1] is None + assert relationships["tag_report"]["parsed_count"] == 1 + assert relationships["tag_report"]["source_ids"][-1] == "row:2" + assert not relationships["aggregate"]["complete"] + + serialized = json.dumps(cases["sentinel"], sort_keys=True) + assert "PRIVATE_PAYLOAD_SERIALIZED" not in serialized + assert all( + outcome["type"] == "AssertionError" for outcome in cases["sentinel"].values() + ) + assert "PRIVATE_PAYLOAD" not in json.dumps(cases, sort_keys=True) + + +def test_runtime_sources_contain_no_active_assert_statements() -> None: + active_asserts = [] + dynamic_validation_messages = [] + for source_path in sorted(RUNTIME_SOURCE.rglob("*.py")): + tree = ast.parse(source_path.read_text(), filename=str(source_path)) + active_asserts.extend( + (source_path.relative_to(RUNTIME_SOURCE), node.lineno) + for node in ast.walk(tree) + if isinstance(node, ast.Assert) + ) + dynamic_validation_messages.extend( + (source_path.relative_to(RUNTIME_SOURCE), node.lineno) + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "require_valid" + and ( + len(node.args) < 2 + or not isinstance(node.args[1], ast.Constant) + or not isinstance(node.args[1].value, str) + ) + ) + + assert active_asserts == [] + assert dynamic_validation_messages == [] diff --git a/tests/unit/test_profiled_investment_values.py b/tests/unit/test_profiled_investment_values.py index 9a7c37f..46fc711 100644 --- a/tests/unit/test_profiled_investment_values.py +++ b/tests/unit/test_profiled_investment_values.py @@ -1,31 +1,35 @@ +from contextlib import contextmanager from decimal import Decimal import pytest from moneywiz_api.managers.investment_holding_manager import InvestmentHoldingManager from moneywiz_api.managers.transaction_manager import TransactionManager -from moneywiz_api.database_accessor import DatabaseAccessor from moneywiz_api.model.investment_holding import InvestmentHolding from moneywiz_api.model.transaction import ( InvestmentBuyTransaction, InvestmentSellTransaction, ) from moneywiz_api.schema_profile import SchemaProfile +from moneywiz_api.read_result import RelationshipLoadReport, RelationshipStorage +from tests.unit.accessor_test_support import initialized_memory_accessor UNSUFFIXED_PROFILE = SchemaProfile( profile_id="unsuffixed-investment-columns", holding_number_of_shares_column="ZNUMBEROFSHARES", transaction_number_of_shares_column="ZNUMBEROFSHARES", - price_per_share_column="ZPRICEPERSHARE", + holding_price_per_share_column="ZPRICEPERSHARE", + transaction_price_per_share_column="ZPRICEPERSHARE", ) MIXED_PROFILE = SchemaProfile( profile_id="mixed-investment-columns", holding_number_of_shares_column="ZNUMBEROFSHARES", transaction_number_of_shares_column="ZNUMBEROFSHARES1", - price_per_share_column="ZPRICEPERSHARE1", + holding_price_per_share_column="ZPRICEPERSHARE", + transaction_price_per_share_column="ZPRICEPERSHARE1", ) -UNKNOWN_PROFILE = SchemaProfile("unknown", None, None, None) +UNKNOWN_PROFILE = SchemaProfile("unknown", None, None, None, None) def investment_transaction_row(ent: int, amount: float) -> dict: @@ -59,6 +63,8 @@ def investment_holding_row() -> dict: "ZOPENNINGNUMBEROFSHARES": None, "ZNUMBEROFSHARES": 2.0, "ZNUMBEROFSHARES1": 9.0, + "ZPRICEPERSHARE": 10.0, + "ZPRICEPERSHARE1": 1.0, "ZSYMBOL": "ACME", "ZHOLDINGTYPE": None, "ZDESC": "Acme Corp.", @@ -86,6 +92,63 @@ def test_holding_uses_selected_profile_alias() -> None: holding = InvestmentHolding(investment_holding_row(), UNSUFFIXED_PROFILE) assert holding.number_of_shares == Decimal("2.0") + assert holding.price_per_share == Decimal("10.0") + + +@pytest.mark.parametrize( + ("raw_value", "expected"), + [(0, False), (1, True)], +) +def test_holding_preserves_binary_online_price_domain(raw_value, expected) -> None: + row = investment_holding_row() + row["ZISPRICEPERSHAREAVAILABLEONLINE"] = raw_value + + assert ( + InvestmentHolding(row, UNSUFFIXED_PROFILE).price_per_share_available_online + is expected + ) + + +@pytest.mark.parametrize( + "raw_value", + [None, False, True, -1, 2, 1.0, "PRIVATE_PAYLOAD"], +) +def test_holding_rejects_non_binary_or_coerced_online_price_state( + raw_value, +) -> None: + row = investment_holding_row() + row["ZISPRICEPERSHAREAVAILABLEONLINE"] = raw_value + + with pytest.raises(AssertionError) as error: + InvestmentHolding(row, UNSUFFIXED_PROFILE) + + assert "PRIVATE_PAYLOAD" not in str(error.value) + + +def test_holding_manager_reports_invalid_online_price_state_as_incomplete() -> None: + invalid = investment_holding_row() + invalid["ZISPRICEPERSHAREAVAILABLEONLINE"] = 2 + valid = investment_holding_row() + valid.update( + { + "Z_PK": 25, + "ZGID": "holding-valid", + "ZISPRICEPERSHAREAVAILABLEONLINE": 1, + } + ) + + manager = InvestmentHoldingManager() + report = manager.load(ManagerAccessor("InvestmentHolding", invalid)) + + assert not report.complete + assert report.parsed_ids == () + assert report.skipped[0].error.value == "validation" + assert manager.records() == {} + + report = manager.load(ManagerAccessor("InvestmentHolding", valid)) + + assert report.complete + assert manager.get(25).price_per_share_available_online is True @pytest.mark.parametrize( @@ -106,6 +169,7 @@ def test_mixed_profile_uses_consumer_specific_share_aliases( assert transaction.number_of_shares == Decimal("9.0") assert transaction.price_per_share == Decimal("1.0") assert holding.number_of_shares == Decimal("2.0") + assert holding.price_per_share == Decimal("10.0") class ProfileAccessor: @@ -130,20 +194,27 @@ def __init__(self, typename: str, row: dict): self.typename = typename self.row = row + @contextmanager + def read_transaction(self): + yield + def query_objects(self, _typenames): return [self.row] + def descendant_typenames(self, _roots): + return [self.typename] + def typename_for(self, _ent_id): return self.typename - def get_category_assignment(self): - return {} + def read_category_assignments(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) - def get_refund_maps(self): - return {} + def read_refund_maps(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) - def get_tags_map(self): - return {} + def read_tags_map(self): + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) def test_managers_load_profiled_investment_records() -> None: @@ -160,32 +231,21 @@ def test_managers_load_profiled_investment_records() -> None: assert holding_manager.get(24).number_of_shares == Decimal("2.0") -class StaticCursor: - def __init__(self, row: dict): - self.row = row - - def execute(self, _query, _parameters): - return self - - def fetchone(self): - return self.row - - -class StaticConnection: - def __init__(self, row: dict): - self.row = row - - def cursor(self): - return StaticCursor(self.row) - - def test_accessor_public_constructors_receive_schema_profile() -> None: - transaction_accessor = DatabaseAccessor.__new__(DatabaseAccessor) - transaction_accessor._con = StaticConnection(investment_transaction_row(40, -20.0)) - transaction_accessor._schema_profile = UNSUFFIXED_PROFILE - holding_accessor = DatabaseAccessor.__new__(DatabaseAccessor) - holding_accessor._con = StaticConnection(investment_holding_row()) - holding_accessor._schema_profile = UNSUFFIXED_PROFILE + transaction_row = investment_transaction_row(40, -20.0) + transaction_row.pop("ZNUMBEROFSHARES1") + transaction_row.pop("ZPRICEPERSHARE1") + holding_row = investment_holding_row() + holding_row.pop("ZNUMBEROFSHARES1") + holding_row.pop("ZPRICEPERSHARE1") + transaction_accessor = initialized_memory_accessor( + [transaction_row], + [(40, "InvestmentBuyTransaction", 0)], + ) + holding_accessor = initialized_memory_accessor( + [holding_row], + [(24, "InvestmentHolding", 0)], + ) transaction = transaction_accessor.get_record(40, InvestmentBuyTransaction) holding = holding_accessor.get_record_by_gid("holding", InvestmentHolding) diff --git a/tests/unit/test_read_completeness.py b/tests/unit/test_read_completeness.py new file mode 100644 index 0000000..c402712 --- /dev/null +++ b/tests/unit/test_read_completeness.py @@ -0,0 +1,345 @@ +from contextlib import contextmanager +from decimal import Decimal +import json + +import pytest + +from moneywiz_api.managers.account_manager import AccountManager +from moneywiz_api.managers.category_manager import CategoryManager +from moneywiz_api.managers.investment_holding_manager import ( + InvestmentHoldingManager, +) +from moneywiz_api.managers.payee_manager import PayeeManager +from moneywiz_api.managers.record_manager import RecordManager +from moneywiz_api.managers.tag_manager import TagManager +from moneywiz_api.managers.transaction_manager import TransactionManager +from moneywiz_api.model.record import Record +from moneywiz_api.read_result import ( + LoadErrorKind, + ManagerLoadReport, + RelationshipLoadReport, + RelationshipStorage, +) +from moneywiz_api.schema_profile import SchemaProfile + + +class ExampleRecord(Record): + pass + + +class ExampleManager(RecordManager): + @property + def ents(self): + return {"ExampleRecord": ExampleRecord} + + @property + def entity_roots(self): + return ("ExampleRecord",) + + +@pytest.mark.parametrize( + "manager_type", + [ + AccountManager, + PayeeManager, + CategoryManager, + TransactionManager, + InvestmentHoldingManager, + TagManager, + ], +) +def test_fresh_public_manager_reports_unloaded(manager_type) -> None: + report = manager_type().load_report + + assert not report.observed + assert report.status == "unloaded" + assert not report.complete + assert report.as_dict()["status"] == "unloaded" + + +def test_default_report_remains_a_successful_empty_observation() -> None: + report = ManagerLoadReport() + + assert report.observed + assert report.complete + assert report.status == "complete" + + +def record_row(record_id=1, gid="record-1", ent=1): + return { + "Z_ENT": ent, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": gid, + "Z_PK": record_id, + } + + +class RecordAccessor: + def __init__(self, rows, typenames=None): + self.rows = rows + self.typenames = typenames or {1: "ExampleRecord"} + + @contextmanager + def read_transaction(self): + yield + + def descendant_typenames(self, _roots): + return list(self.typenames.values()) + + def query_objects(self, _typenames): + return self.rows + + def typename_for(self, ent_id): + return self.typenames.get(ent_id) + + +def test_report_counts_duplicate_and_unknown_rows_without_partial_mutation() -> None: + manager = ExampleManager() + rows = [ + record_row(), + record_row(gid="duplicate-id"), + record_row(record_id=2), + record_row(record_id=3, gid="record-3", ent=2), + ] + + report = manager.load(RecordAccessor(rows, {1: "ExampleRecord", 2: "FutureRecord"})) + + assert report.source_ids == (1, 1, 2, 3) + assert report.parsed_ids == (1,) + assert [item.error for item in report.skipped] == [ + LoadErrorKind.DUPLICATE_ID, + LoadErrorKind.DUPLICATE_GID, + LoadErrorKind.UNKNOWN_ENTITY, + ] + assert report.observed + assert report.status == "partial" + assert list(manager.records()) == [1] + + +def test_each_load_resets_records_and_diagnostics() -> None: + manager = ExampleManager() + manager.load(RecordAccessor([record_row(), record_row(gid="duplicate-id")])) + + report = manager.load(RecordAccessor([record_row(record_id=4, gid="record-4")])) + + assert report.complete + assert report.source_ids == (4,) + assert report.parsed_ids == (4,) + assert list(manager.records()) == [4] + assert manager.load_errors == [] + + +def test_failed_first_load_is_unloaded_and_can_retry() -> None: + manager = ExampleManager() + + class FailingAccessor(RecordAccessor): + def query_objects(self, _typenames): + raise RuntimeError("synthetic query failure") + + with pytest.raises(RuntimeError, match="synthetic query failure"): + manager.load(FailingAccessor([])) + + assert manager.load_report.status == "unloaded" + assert not manager.load_report.complete + assert manager.records() == {} + + report = manager.load(RecordAccessor([])) + + assert report.observed + assert report.status == "complete" + assert report.complete + + +def test_failed_direct_reload_replaces_prior_report_with_unloaded_state() -> None: + manager = ExampleManager() + manager.load(RecordAccessor([record_row()])) + + class FailingAccessor(RecordAccessor): + def query_objects(self, _typenames): + raise RuntimeError("synthetic reload failure") + + with pytest.raises(RuntimeError, match="synthetic reload failure"): + manager.load(FailingAccessor([])) + + assert manager.load_report.status == "unloaded" + assert not manager.load_report.complete + assert manager.records() == {} + + +@pytest.mark.parametrize("interruption_type", [KeyboardInterrupt, SystemExit]) +def test_interrupted_reload_discards_partial_state_and_reraises( + interruption_type, +) -> None: + manager = ExampleManager() + manager.load(RecordAccessor([record_row(record_id=9, gid="record-9")])) + interruption = interruption_type("synthetic interruption") + + class InterruptingAccessor(RecordAccessor): + def __init__(self): + super().__init__([record_row(), record_row(record_id=2, gid="record-2")]) + self.lookups = 0 + + def typename_for(self, ent_id): + self.lookups += 1 + if self.lookups == 2: + raise interruption + return super().typename_for(ent_id) + + with pytest.raises(interruption_type) as error: + manager.load(InterruptingAccessor()) + + assert error.value is interruption + assert manager.records() == {} + assert manager._gid_to_id == {} + assert manager.load_report.status == "unloaded" + + report = manager.load(RecordAccessor([record_row(record_id=3, gid="record-3")])) + + assert report.complete + assert list(manager.records()) == [3] + + +def test_completed_load_with_only_invalid_rows_reports_error() -> None: + manager = ExampleManager() + + report = manager.load( + RecordAccessor([record_row(ent=2)], {1: "ExampleRecord", 2: "FutureRecord"}) + ) + + assert report.observed + assert report.status == "error" + assert not report.complete + + +class FakeAccessor: + def __init__(self, _db_file): + self.schema_profile = SchemaProfile( + "unsuffixed-investment-columns", + "ZNUMBEROFSHARES", + "ZNUMBEROFSHARES", + "ZPRICEPERSHARE", + "ZPRICEPERSHARE", + ) + self.queries = [] + self.transaction_entries = 0 + self.transaction_depth = 0 + self.max_transaction_depth = 0 + self.closed = False + + @contextmanager + def read_transaction(self): + self.transaction_entries += 1 + self.transaction_depth += 1 + self.max_transaction_depth = max( + self.max_transaction_depth, self.transaction_depth + ) + try: + yield + finally: + self.transaction_depth -= 1 + + def descendant_typenames(self, _roots): + return [] + + def query_objects(self, typenames): + assert self.transaction_depth == 2 + self.queries.append(tuple(typenames)) + return [] + + def typename_for(self, _ent_id): + return None + + def read_category_assignments(self): + assert self.transaction_depth == 2 + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def read_refund_maps(self): + assert self.transaction_depth == 2 + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def read_tags_map(self): + assert self.transaction_depth == 2 + return {}, RelationshipLoadReport(RelationshipStorage.ABSENT) + + def close(self): + self.closed = True + + +def test_api_supports_scoped_loads_and_json_safe_snapshot(monkeypatch) -> None: + import moneywiz_api.moneywiz_api as api_module + + monkeypatch.setattr(api_module, "DatabaseAccessor", FakeAccessor) + + with api_module.MoneywizApi( + "unused.sqlite", managers=("accounts", "transactions") + ) as api: + report = api.completeness().as_dict() + snapshot = api.snapshot().as_dict() + + assert list(report["managers"]) == ["accounts", "transactions"] + assert report["complete"] + assert snapshot["records"] == {"accounts": [], "transactions": []} + assert snapshot["schema_profile"]["profile_id"] == ( + "unsuffixed-investment-columns" + ) + assert api.payee_manager.load_report.status == "unloaded" + assert api.accessor.transaction_entries == 3 + assert api.accessor.max_transaction_depth == 2 + json.dumps(snapshot) + + assert api.accessor.closed + + +def test_api_preserves_default_eager_loading(monkeypatch) -> None: + import moneywiz_api.moneywiz_api as api_module + + monkeypatch.setattr(api_module, "DatabaseAccessor", FakeAccessor) + + with api_module.MoneywizApi("unused.sqlite") as api: + assert list(api.completeness().managers) == list(api.MANAGER_NAMES) + assert len(api.accessor.queries) == len(api.MANAGER_NAMES) + assert api.accessor.transaction_entries == 7 + assert api.accessor.max_transaction_depth == 2 + + +def test_api_rejects_unloaded_or_unknown_manager(monkeypatch) -> None: + import moneywiz_api.moneywiz_api as api_module + + monkeypatch.setattr(api_module, "DatabaseAccessor", FakeAccessor) + api = api_module.MoneywizApi("unused.sqlite", managers=()) + + assert all( + api._managers[name].load_report.status == "unloaded" + for name in api.MANAGER_NAMES + ) + with pytest.raises(ValueError, match="manager not loaded"): + api.completeness(("accounts",)) + with pytest.raises(ValueError, match="manager not loaded"): + api.snapshot(("accounts",)) + with pytest.raises(ValueError, match="unknown manager"): + api.load(("missing",)) + + api.close() + + +def test_json_safe_snapshot_formats_decimal(monkeypatch) -> None: + import moneywiz_api.moneywiz_api as api_module + + monkeypatch.setattr(api_module, "DatabaseAccessor", FakeAccessor) + api = api_module.MoneywizApi("unused.sqlite", managers=()) + api.account_manager.add( + type( + "SnapshotRecord", + (), + { + "id": 1, + "gid": "snapshot-record", + "display_order": 1, + "as_dict": lambda self: {"amount": Decimal("1.20")}, + }, + )() + ) + api._loaded_managers.add("accounts") + + assert api.snapshot().as_dict()["records"]["accounts"][0]["amount"] == "1.20" + api.close() diff --git a/tests/unit/test_read_result_ownership.py b/tests/unit/test_read_result_ownership.py new file mode 100644 index 0000000..7df3d3c --- /dev/null +++ b/tests/unit/test_read_result_ownership.py @@ -0,0 +1,370 @@ +from copy import deepcopy +from dataclasses import asdict, dataclass, fields, replace +from datetime import date, datetime +from decimal import Decimal +from enum import Enum +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from moneywiz_api.managers.tag_manager import TagManager +from moneywiz_api.moneywiz_api import MoneywizApi +from moneywiz_api.model.record import Record +from moneywiz_api.read_result import ( + ApiCompleteness, + LoadErrorKind, + ManagerLoadReport, + ReadSnapshot, + RelationshipLoadReport, + RelationshipStorage, + SkippedRecord, + json_safe, +) +from moneywiz_api.schema_profile import SchemaProfile + + +def incomplete_relationship() -> RelationshipLoadReport: + return RelationshipLoadReport( + storage=RelationshipStorage.UNKNOWN, + source_ids=["source-1"], + skipped=[ + SkippedRecord( + record_id="source-1", + entity="Tag", + error=LoadErrorKind.MISSING_FIELD, + exception_type="KeyError", + ) + ], + ) + + +def test_report_constructors_copy_sequences_and_relationship_mapping() -> None: + source_ids = [1] + parsed_ids = [] + skipped = [ + SkippedRecord(1, "FutureRecord", LoadErrorKind.UNKNOWN_ENTITY, "ValueError") + ] + relationships = {"transaction_tags": incomplete_relationship()} + report = ManagerLoadReport( + source_ids=source_ids, + parsed_ids=parsed_ids, + skipped=skipped, + relationships=relationships, + ) + + source_ids.append(2) + parsed_ids.append(1) + skipped.clear() + relationships.clear() + + assert report.source_ids == (1,) + assert report.parsed_ids == () + assert len(report.skipped) == 1 + assert set(report.relationships) == {"transaction_tags"} + assert not report.complete + with pytest.raises(TypeError, match="immutable"): + report.relationships.clear() + + +def test_aggregate_owns_manager_mapping_and_cannot_become_complete() -> None: + managers = { + "incomplete": ManagerLoadReport( + relationships={"transaction_tags": incomplete_relationship()} + ), + "complete": ManagerLoadReport(), + } + completeness = ApiCompleteness(managers) + + del managers["incomplete"] + + assert set(completeness.managers) == {"incomplete", "complete"} + assert not completeness.complete + with pytest.raises(TypeError, match="immutable"): + completeness.managers.update({"replacement": ManagerLoadReport()}) + + +def test_snapshot_owns_nested_records_and_schema_profile() -> None: + nested_values = [1] + row = {"amount": Decimal("1.20"), "nested": {"values": nested_values}} + records = {"accounts": [row]} + schema_profile = {"aliases": {"columns": ["ZVALUE"]}} + snapshot = ReadSnapshot( + records=records, + completeness=ApiCompleteness({"accounts": ManagerLoadReport()}), + schema_profile=schema_profile, + ) + + nested_values.append(2) + row["nested"]["extra"] = True + records.clear() + schema_profile["aliases"]["columns"].append("ZVALUE1") + + retained_row = snapshot.records["accounts"][0] + assert retained_row["nested"] == {"values": (1,)} + assert snapshot.schema_profile == {"aliases": {"columns": ("ZVALUE",)}} + with pytest.raises(TypeError, match="immutable"): + retained_row["nested"]["extra"] = True + with pytest.raises(AttributeError, match="append"): + retained_row["nested"]["values"].append(3) + + +def test_result_outputs_are_mutable_detached_json_trees() -> None: + report = ManagerLoadReport( + relationships={"transaction_tags": incomplete_relationship()} + ) + snapshot = ReadSnapshot( + records={ + "accounts": ( + { + "amount": Decimal("1.20"), + "date": date(2026, 9, 13), + "nested": {"values": [1]}, + }, + ) + }, + completeness=ApiCompleteness({"accounts": report}), + schema_profile={"aliases": {"columns": ["ZVALUE"]}}, + ) + + report_output = report.as_dict() + snapshot_output = snapshot.as_dict() + safe_output = json_safe(snapshot) + report_output["relationships"].clear() + snapshot_output["records"]["accounts"][0]["nested"]["values"].append(2) + snapshot_output["schema_profile"]["aliases"]["columns"].append("ZVALUE1") + safe_output["records"]["accounts"][0]["nested"]["values"].append(3) + + assert not report.complete + assert snapshot.records["accounts"][0]["nested"]["values"] == (1,) + assert snapshot.schema_profile["aliases"]["columns"] == ("ZVALUE",) + assert snapshot_output["records"]["accounts"][0]["amount"] == "1.20" + assert snapshot_output["records"]["accounts"][0]["date"] == "2026-09-13" + + +def test_dataclass_asdict_replace_and_constructor_contracts_remain_supported() -> None: + unloaded = ManagerLoadReport.unloaded() + default = ManagerLoadReport() + report = replace( + default, + relationships={"transaction_tags": incomplete_relationship()}, + ) + snapshot = ReadSnapshot( + records={"accounts": ({"nested": [1]},)}, + completeness=ApiCompleteness({"accounts": report}), + schema_profile={"aliases": ["ZVALUE"]}, + ) + + report_dict = asdict(report) + snapshot_dict = asdict(snapshot) + copied_records = deepcopy(snapshot.records) + replaced_snapshot = replace(snapshot, schema_profile={"aliases": ["ZVALUE1"]}) + + assert [item.name for item in fields(ManagerLoadReport)] == [ + "source_ids", + "parsed_ids", + "skipped", + "relationships", + "observed", + ] + assert unloaded.status == "unloaded" + assert default.complete + assert report_dict["relationships"]["transaction_tags"]["storage"] == ( + RelationshipStorage.UNKNOWN + ) + assert snapshot_dict["records"]["accounts"][0]["nested"] == (1,) + assert copied_records == snapshot.records + assert copied_records is not snapshot.records + assert replaced_snapshot.schema_profile["aliases"] == ("ZVALUE1",) + + +def test_relationship_sequences_are_copied_and_as_dict_is_detached() -> None: + source_ids = ["source-1"] + parsed_ids = ["source-1"] + report = RelationshipLoadReport( + RelationshipStorage.PRESENT, + source_ids=source_ids, + parsed_ids=parsed_ids, + ) + + source_ids.append("source-2") + parsed_ids.clear() + output = report.as_dict() + output["source_ids"].append("source-3") + output["parsed_ids"].clear() + + assert report.source_ids == ("source-1",) + assert report.parsed_ids == ("source-1",) + assert report.complete + + +def test_live_manager_and_model_remain_mutable_after_snapshot_publication() -> None: + record = Record( + { + "Z_PK": 1, + "Z_ENT": 35, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": "record-1", + } + ) + manager = TagManager() + manager.records()[record.id] = record + snapshot = ReadSnapshot( + records={"tags": (record.as_dict(),)}, + completeness=ApiCompleteness({"tags": ManagerLoadReport()}), + schema_profile={}, + ) + + record.gid = "changed-record" + manager.records()[2] = record + + assert manager.get(2) is record + assert record.gid == "changed-record" + assert snapshot.records["tags"][0]["gid"] == "record-1" + + +class TextEnum(Enum): + VALUE = "value" + + +class BinaryEnum(Enum): + VALUE = b"PRIVATE_PAYLOAD" + + +@dataclass +class NestedValue: + value: object + + +class PrivatePayload: + def __repr__(self): + raise RuntimeError("PRIVATE_REPR") + + def __str__(self): + raise RuntimeError("PRIVATE_STR") + + +def test_json_safe_preserves_supported_scalar_and_converter_contract() -> None: + value = { + "none": None, + "bool": True, + "int": 3, + "float": -0.25, + "decimal": Decimal("NaN"), + "datetime": datetime(2026, 9, 13, 10, 30), + "date": date(2026, 9, 13), + "path": Path("relative/path"), + "enum": TextEnum.VALUE, + "dataclass": NestedValue(["nested"]), + } + + converted = json_safe(value) + + assert converted == { + "none": None, + "bool": True, + "int": 3, + "float": -0.25, + "decimal": "NaN", + "datetime": "2026-09-13T10:30:00", + "date": "2026-09-13", + "path": "relative/path", + "enum": "value", + "dataclass": {"value": ["nested"]}, + } + json.dumps(converted, allow_nan=False) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) +def test_json_safe_rejects_nonfinite_builtin_floats(value) -> None: + with pytest.raises(TypeError, match="float values must be finite"): + json_safe(value) + + +@pytest.mark.parametrize( + "value", + [b"PRIVATE_PAYLOAD", PrivatePayload(), BinaryEnum.VALUE, NestedValue(b"x")], +) +def test_json_safe_rejects_nested_unsupported_values_without_rendering(value) -> None: + with pytest.raises(TypeError) as error: + json_safe({"nested": [value]}) + + assert "PRIVATE" not in str(error.value) + + +def test_json_safe_key_domain_and_collision_are_deterministic() -> None: + assert json_safe({None: 0, True: 1, 2: 2, 1.5: 3, "name": 4}) == { + "None": 0, + "True": 1, + "2": 2, + "1.5": 3, + "name": 4, + } + with pytest.raises(TypeError, match="keys collide"): + json_safe({1: "integer", "1": "string"}) + + +@pytest.mark.parametrize( + "key", + [ + Decimal("1"), + TextEnum.VALUE, + Path("key"), + b"PRIVATE_PAYLOAD", + ("tuple",), + PrivatePayload(), + float("nan"), + ], +) +def test_json_safe_rejects_unsupported_keys_without_rendering(key) -> None: + with pytest.raises(TypeError) as error: + json_safe({key: "value"}) + + assert "PRIVATE" not in str(error.value) + + +def test_snapshot_refuses_unsupported_nested_values_before_publication() -> None: + with pytest.raises(TypeError, match="unsupported value"): + ReadSnapshot( + records={"accounts": ({"nested": [b"PRIVATE_PAYLOAD"]},)}, + completeness=ApiCompleteness({"accounts": ManagerLoadReport()}), + schema_profile={}, + ) + + +def test_api_snapshot_refuses_live_model_mutation_but_prior_snapshot_is_detached() -> ( + None +): + record = Record( + { + "Z_PK": 1, + "Z_ENT": 35, + "ZOBJECTCREATIONDATE": 0.0, + "ZGID": "record-1", + } + ) + manager = TagManager() + manager.records()[record.id] = record + manager._load_report = ManagerLoadReport(source_ids=(1,), parsed_ids=(1,)) + api = MoneywizApi.__new__(MoneywizApi) + api._managers = {"tags": manager} + api._loaded_managers = {"tags"} + api.accessor = SimpleNamespace( + schema_profile=SchemaProfile( + "unsuffixed-investment-columns", + "ZNUMBEROFSHARES", + "ZNUMBEROFSHARES", + "ZPRICEPERSHARE", + "ZPRICEPERSHARE", + ) + ) + + published = api.snapshot(("tags",)) + record.gid = b"PRIVATE_PAYLOAD" + + assert published.records["tags"][0]["gid"] == "record-1" + json.dumps(published.as_dict(), allow_nan=False) + assert record.as_dict()["gid"] == b"PRIVATE_PAYLOAD" + with pytest.raises(TypeError, match="unsupported value"): + api.snapshot(("tags",)) diff --git a/tests/unit/test_relationship_completeness.py b/tests/unit/test_relationship_completeness.py new file mode 100644 index 0000000..dd7b782 --- /dev/null +++ b/tests/unit/test_relationship_completeness.py @@ -0,0 +1,796 @@ +import json +import sqlite3 + +import pytest + +from moneywiz_api.database_accessor import DatabaseAccessor +from moneywiz_api.managers.transaction_manager import TransactionManager +from moneywiz_api.read_result import RelationshipStorage + + +SCHEDULED_TAG_TABLE = ( + "CREATE TABLE Z_31TAGS (Z_31SCHEDULEDTRANSACTIONS1 INTEGER, Z_35TAGS2 INTEGER)" +) +INFO_CARD_TAG_TABLE = ( + "CREATE TABLE Z_23TAGS (Z_23INFOCARDS5 INTEGER, Z_35TAGS1 INTEGER)" +) + + +def create_relationship_schema(path, *, include_tables=True) -> None: + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.execute("CREATE TABLE ZSYNCOBJECT (Z_PK INTEGER, Z_ENT INTEGER)") + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", + [ + (2, "CategoryAssigment", 0, 0), + (8, "SyncObject", 0, 0), + (36, "Tag", 8, 0), + (37, "Transaction", 8, 0), + (50, "WithdrawRefundTransactionLink", 0, 0), + ], + ) + if include_tables: + connection.execute( + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER PRIMARY KEY, ZCATEGORY INTEGER, ZAMOUNT FLOAT, " + "ZTRANSACTION INTEGER)" + ) + connection.executemany( + "INSERT INTO ZCATEGORYASSIGMENT VALUES (?, ?, ?, ?)", + [(1, 10, 5.0, 100), (2, 11, None, 101), (3, 12, 2.0, None)], + ) + connection.execute( + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER PRIMARY KEY, ZREFUNDTRANSACTION INTEGER, " + "ZWITHDRAWTRANSACTION INTEGER)" + ) + connection.executemany( + "INSERT INTO ZWITHDRAWREFUNDTRANSACTIONLINK VALUES (?, ?, ?)", + [(4, 102, 103), (5, 104, None)], + ) + connection.execute( + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_37TAGS VALUES (?, ?)", + [(100, 200), (101, None)], + ) + connection.commit() + connection.close() + + +def create_custom_relationship_schema(path, metadata=(), tables=()) -> None: + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.execute("CREATE TABLE ZSYNCOBJECT (Z_PK INTEGER, Z_ENT INTEGER)") + connection.executemany("INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", metadata) + for statement in tables: + connection.execute(statement) + connection.commit() + connection.close() + + +def read_tags_and_aggregate(path): + with DatabaseAccessor(path) as accessor: + tags, report = accessor.read_tags_map() + manager_report = TransactionManager().load(accessor) + return tags, report, manager_report + + +@pytest.mark.parametrize("populated", [False, True], ids=["empty", "populated"]) +@pytest.mark.parametrize( + "metadata", + [ + (), + ((37, "Transaction", 0, 0),), + ((36, "Tag", 0, 0),), + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + ], + ids=["absent", "transaction-only", "tag-only", "full"], +) +@pytest.mark.parametrize( + ("table_name", "include_expected"), + [("Z_37TAGS", False), ("Z_99TAGS", False), ("Z_99TAGS", True)], + ids=["expected", "alternate", "alternate-beside-direct"], +) +@pytest.mark.parametrize( + "columns", + [ + ("Z_37TRANSACTIONS",), + ("Z_36TAGS",), + ("Z_37TRANSACTIONS", "Z_36TAGS", "Z_37SCHEDULEDTRANSACTIONS1"), + ("Z_37TRANSACTIONS", "Z_36TAGS", "Z_37INFOCARDS5"), + ("Z_37TRANSACTIONS", "Z_36TAGS", "Z_99TRANSACTIONS"), + ("Z_37TRANSACTIONS", "Z_36TAGS", "Z_99TAGS"), + ("Z_37TRANSACTIONS", "Z_36TAGS", "Z_PK"), + ], + ids=[ + "missing-tag", + "missing-transaction", + "extra-scheduled", + "extra-info-card", + "extra-transaction", + "extra-tag", + "extra-auxiliary", + ], +) +def test_ambiguous_direct_tag_shape_is_unknown( + tmp_path, populated, metadata, table_name, include_expected, columns +) -> None: + path = tmp_path / "ambiguous-direct-tags.sqlite" + definition = ", ".join(f"{column} INTEGER" for column in columns) + tables = [f"CREATE TABLE {table_name} ({definition})"] + if include_expected: + tables.append( + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" + ) + create_custom_relationship_schema(path, metadata=metadata, tables=tables) + if populated: + with sqlite3.connect(path) as connection: + connection.execute( + f"INSERT INTO {table_name} VALUES ({', '.join('?' for _ in columns)})", + tuple(100 * (index + 1) for index in range(len(columns))), + ) + + tags, report, manager_report = read_tags_and_aggregate(path) + + assert report.storage == RelationshipStorage.UNKNOWN + assert tags == {} + assert not report.complete + assert not manager_report.complete + assert ( + manager_report.relationships["transaction_tags"].storage + == RelationshipStorage.UNKNOWN + ) + + +def test_shifted_relationship_layouts_report_valid_and_skipped_rows(tmp_path) -> None: + path = tmp_path / "relationships.sqlite" + create_relationship_schema(path) + + with DatabaseAccessor(path) as accessor: + categories, category_report = accessor.read_category_assignments() + refunds, refund_report = accessor.read_refund_maps() + tags, tag_report = accessor.read_tags_map() + + assert categories == {100: [(10, 5)]} + assert category_report.source_ids == (1, 2) + assert category_report.parsed_ids == (1,) + assert category_report.status == "partial" + assert refunds == {102: 103} + assert refund_report.source_ids == (4, 5) + assert refund_report.parsed_ids == (4,) + assert refund_report.status == "partial" + assert tags == {100: [200]} + assert tag_report.storage_name == "Z_37TAGS" + assert tag_report.source_ids == ("100:200", "row:1") + assert tag_report.parsed_ids == ("100:200",) + assert tag_report.status == "partial" + + manager_report = TransactionManager().load(accessor) + + assert manager_report.status == "partial" + assert not manager_report.complete + assert set(manager_report.relationships) == { + "category_assignments", + "refund_links", + "transaction_tags", + } + assert ( + manager_report.as_dict()["relationships"]["transaction_tags"]["storage_name"] + == "Z_37TAGS" + ) + + +@pytest.mark.parametrize( + ( + "reader_name", + "metadata", + "table_statement", + "valid_row", + "invalid_row", + "expected", + "expected_sources", + "expected_parsed", + ), + [ + ( + "read_category_assignments", + ((2, "CategoryAssigment", 0, 0),), + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)", + (1, 10, 100, 5.0), + (2, "PRIVATE_PAYLOAD", 101, 6.0), + {100: [(10, 5)]}, + (1, 2), + (1,), + ), + ( + "read_category_assignments", + ((2, "CategoryAssigment", 0, 0),), + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)", + (1, 10, 100, 5.0), + (2, 11, "PRIVATE_PAYLOAD", 6.0), + {100: [(10, 5)]}, + (1, 2), + (1,), + ), + ( + "read_category_assignments", + ((2, "CategoryAssigment", 0, 0),), + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)", + (1, 10, 100, 5.0), + ("PRIVATE_PAYLOAD", 11, 101, 6.0), + {100: [(10, 5)]}, + (1, None), + (1,), + ), + ( + "read_refund_maps", + ((50, "WithdrawRefundTransactionLink", 0, 0),), + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, " + "ZWITHDRAWTRANSACTION INTEGER)", + (1, 100, 90), + (2, "PRIVATE_PAYLOAD", 91), + {100: 90}, + (1, 2), + (1,), + ), + ( + "read_refund_maps", + ((50, "WithdrawRefundTransactionLink", 0, 0),), + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, " + "ZWITHDRAWTRANSACTION INTEGER)", + (1, 100, 90), + (2, 101, "PRIVATE_PAYLOAD"), + {100: 90}, + (1, 2), + (1,), + ), + ( + "read_refund_maps", + ((50, "WithdrawRefundTransactionLink", 0, 0),), + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, " + "ZWITHDRAWTRANSACTION INTEGER)", + (1, 100, 90), + ("PRIVATE_PAYLOAD", 101, 91), + {100: 90}, + (1, None), + (1,), + ), + ( + "read_tags_map", + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + (100, 200), + ("PRIVATE_PAYLOAD", 201), + {100: [200]}, + ("100:200", "row:1"), + ("100:200",), + ), + ( + "read_tags_map", + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + (100, 200), + (101, "PRIVATE_PAYLOAD"), + {100: [200]}, + ("100:200", "row:1"), + ("100:200",), + ), + ], +) +def test_integer_affinity_relationships_refuse_noninteger_raw_identities( + tmp_path, + reader_name, + metadata, + table_statement, + valid_row, + invalid_row, + expected, + expected_sources, + expected_parsed, +) -> None: + path = tmp_path / f"{reader_name}-identity-domain.sqlite" + create_custom_relationship_schema( + path, metadata=metadata, tables=(table_statement,) + ) + table_name = table_statement.split()[2] + with sqlite3.connect(path) as connection: + placeholders = ", ".join("?" for _ in valid_row) + connection.executemany( + f"INSERT INTO {table_name} VALUES ({placeholders})", + (valid_row, invalid_row), + ) + + with DatabaseAccessor(path) as accessor: + data, report = getattr(accessor, reader_name)() + + assert data == expected + assert report.source_ids == expected_sources + assert report.parsed_ids == expected_parsed + assert report.source_count == 2 + assert report.parsed_count == 1 + assert report.skipped[0].error.value == "validation" + assert "PRIVATE_PAYLOAD" not in json.dumps(report.as_dict(), sort_keys=True) + + +def test_category_counts_exclude_supported_nontransaction_owners(tmp_path) -> None: + path = tmp_path / "nontransaction-category-owners.sqlite" + create_custom_relationship_schema( + path, + metadata=((2, "CategoryAssigment", 0, 0),), + tables=( + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZAMOUNT FLOAT, " + "ZTRANSACTION INTEGER, ZBUDGET INTEGER, " + "ZSCHEDULEDTRANSACITION INTEGER, ZSTRINGHISTORYITEM INTEGER)", + ), + ) + with sqlite3.connect(path) as connection: + connection.executemany( + "INSERT INTO ZCATEGORYASSIGMENT VALUES (?, ?, ?, ?, ?, ?, ?)", + [ + (1, 10, 5.0, 100, None, None, None), + (2, 11, 2.0, None, 200, None, None), + (3, 12, 3.0, None, None, 300, None), + (4, 13, 4.0, None, None, None, 400), + ], + ) + + with DatabaseAccessor(path) as accessor: + categories, report = accessor.read_category_assignments() + + assert categories == {100: [(10, 5)]} + assert report.source_ids == (1,) + assert report.parsed_ids == (1,) + assert report.complete + + +@pytest.mark.parametrize( + "reader_name", + ["read_category_assignments", "read_refund_maps", "read_tags_map"], +) +def test_known_entity_with_missing_storage_is_unknown(tmp_path, reader_name) -> None: + path = tmp_path / "missing-storage.sqlite" + create_relationship_schema(path, include_tables=False) + + with DatabaseAccessor(path) as accessor: + data, report = getattr(accessor, reader_name)() + + assert data == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert report.status == "error" + assert not report.complete + + +def test_missing_optional_relationship_entities_are_absent(tmp_path) -> None: + path = tmp_path / "absent-relationships.sqlite" + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER PRIMARY KEY, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.execute("CREATE TABLE ZSYNCOBJECT (Z_PK INTEGER, Z_ENT INTEGER)") + connection.commit() + connection.close() + + with DatabaseAccessor(path) as accessor: + reports = [ + accessor.read_category_assignments()[1], + accessor.read_refund_maps()[1], + accessor.read_tags_map()[1], + ] + manager_report = TransactionManager().load(accessor) + + assert all(report.storage == RelationshipStorage.ABSENT for report in reports) + assert all(report.complete for report in reports) + assert manager_report.complete + + +def test_unknown_relationship_storage_propagates_to_manager(tmp_path) -> None: + path = tmp_path / "unknown-relationships.sqlite" + create_relationship_schema(path, include_tables=False) + + with DatabaseAccessor(path) as accessor: + manager_report = TransactionManager().load(accessor) + + assert not manager_report.complete + assert manager_report.status == "error" + assert all( + report.storage == RelationshipStorage.UNKNOWN + for report in manager_report.relationships.values() + ) + + +@pytest.mark.parametrize( + ("reader_name", "table_statement"), + [ + ( + "read_category_assignments", + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)", + ), + ( + "read_refund_maps", + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, " + "ZWITHDRAWTRANSACTION INTEGER)", + ), + ( + "read_tags_map", + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + ), + ], +) +def test_physical_storage_without_metadata_is_unknown( + tmp_path, reader_name, table_statement +) -> None: + path = tmp_path / f"{reader_name}-physical-only.sqlite" + create_custom_relationship_schema(path, tables=(table_statement,)) + + with DatabaseAccessor(path) as accessor: + data, report = getattr(accessor, reader_name)() + + assert data == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert report.status == "error" + assert not report.complete + + +@pytest.mark.parametrize( + ("reader_name", "metadata", "table_statement"), + [ + ( + "read_category_assignments", + ((2, "CategoryAssigment", 0, 0),), + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)", + ), + ( + "read_refund_maps", + ((50, "WithdrawRefundTransactionLink", 0, 0),), + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, " + "ZWITHDRAWTRANSACTION INTEGER)", + ), + ( + "read_tags_map", + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + ), + ], +) +def test_valid_empty_relationship_storage_is_present( + tmp_path, reader_name, metadata, table_statement +) -> None: + path = tmp_path / f"{reader_name}-empty.sqlite" + create_custom_relationship_schema( + path, metadata=metadata, tables=(table_statement,) + ) + + with DatabaseAccessor(path) as accessor: + data, report = getattr(accessor, reader_name)() + + assert data == {} + assert report.storage == RelationshipStorage.PRESENT + assert report.complete + assert report.status == "complete" + + +@pytest.mark.parametrize( + ("reader_name", "metadata", "table_statement"), + [ + ( + "read_category_assignments", + ((2, "CategoryAssigment", 0, 0),), + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER)", + ), + ( + "read_refund_maps", + ((50, "WithdrawRefundTransactionLink", 0, 0),), + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER)", + ), + ( + "read_tags_map", + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER)", + ), + ], +) +def test_relationship_storage_with_missing_columns_is_unknown( + tmp_path, reader_name, metadata, table_statement +) -> None: + path = tmp_path / f"{reader_name}-missing-column.sqlite" + create_custom_relationship_schema( + path, metadata=metadata, tables=(table_statement,) + ) + + with DatabaseAccessor(path) as accessor: + data, report = getattr(accessor, reader_name)() + + assert data == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert not report.complete + + +@pytest.mark.parametrize( + "metadata", + [ + ((37, "Transaction", 0, 0),), + ((36, "Tag", 0, 0),), + ], +) +def test_partial_transaction_tag_metadata_is_unknown(tmp_path, metadata) -> None: + path = tmp_path / "partial-tag-metadata.sqlite" + create_custom_relationship_schema(path, metadata=metadata) + + with DatabaseAccessor(path) as accessor: + tags, report = accessor.read_tags_map() + + assert tags == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert not report.complete + + +def test_scheduled_tag_storage_is_not_a_transaction_tag_candidate(tmp_path) -> None: + path = tmp_path / "scheduled-tags.sqlite" + create_custom_relationship_schema(path, tables=(SCHEDULED_TAG_TABLE,)) + + with DatabaseAccessor(path) as accessor: + tags, absent_report = accessor.read_tags_map() + + assert tags == {} + assert absent_report.storage == RelationshipStorage.ABSENT + assert absent_report.complete + + path_with_metadata = tmp_path / "scheduled-and-direct-tags.sqlite" + direct_table = "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" + create_custom_relationship_schema( + path_with_metadata, + metadata=((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + tables=(SCHEDULED_TAG_TABLE, direct_table), + ) + + with DatabaseAccessor(path_with_metadata) as accessor: + _, present_report = accessor.read_tags_map() + + assert present_report.storage == RelationshipStorage.PRESENT + assert present_report.complete + + +@pytest.mark.parametrize( + ("alternate_table", "include_expected"), + [ + ( + "CREATE TABLE Z_36TAGS (Z_36TRANSACTIONS INTEGER, Z_35TAGS INTEGER)", + False, + ), + ( + "CREATE TABLE Z_99TAGS (Z_99TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + True, + ), + ], +) +def test_wrong_or_conflicting_transaction_tag_layout_is_unknown( + tmp_path, alternate_table, include_expected +) -> None: + path = tmp_path / "conflicting-tags.sqlite" + expected_table = ( + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" + ) + tables = ( + (expected_table, alternate_table) if include_expected else (alternate_table,) + ) + create_custom_relationship_schema( + path, + metadata=((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + tables=tables, + ) + + with DatabaseAccessor(path) as accessor: + tags, report = accessor.read_tags_map() + manager_report = TransactionManager().load(accessor) + + assert tags == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert not report.complete + assert not manager_report.complete + assert ( + manager_report.relationships["transaction_tags"].storage + == RelationshipStorage.UNKNOWN + ) + + +@pytest.mark.parametrize("populated", [False, True]) +@pytest.mark.parametrize( + "metadata", + [ + (), + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + ((37, "Transaction", 0, 0),), + ((36, "Tag", 0, 0),), + ], +) +def test_malformed_numbered_tag_table_is_always_unknown( + tmp_path, populated, metadata +) -> None: + path = tmp_path / "malformed-numbered-tags.sqlite" + create_custom_relationship_schema( + path, + metadata=metadata, + tables=("CREATE TABLE Z_37TAGS (Z_PK INTEGER)",), + ) + if populated: + connection = sqlite3.connect(path) + connection.execute("INSERT INTO Z_37TAGS VALUES (1)") + connection.commit() + connection.close() + + tags, report, manager_report = read_tags_and_aggregate(path) + + assert tags == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert not report.complete + assert not manager_report.complete + assert ( + manager_report.relationships["transaction_tags"].storage + == RelationshipStorage.UNKNOWN + ) + + +def test_valid_direct_tags_with_malformed_alternate_are_unknown(tmp_path) -> None: + path = tmp_path / "valid-and-malformed-tags.sqlite" + create_custom_relationship_schema( + path, + metadata=((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + tables=( + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + "CREATE TABLE Z_99TAGS (Z_PK INTEGER)", + ), + ) + + tags, report, manager_report = read_tags_and_aggregate(path) + + assert tags == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert not report.complete + assert not manager_report.complete + + +@pytest.mark.parametrize("populated", [False, True]) +@pytest.mark.parametrize( + "metadata", + [ + (), + ((37, "Transaction", 0, 0),), + ((36, "Tag", 0, 0),), + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + ], +) +@pytest.mark.parametrize("unrelated_table", [SCHEDULED_TAG_TABLE, INFO_CARD_TAG_TABLE]) +def test_complete_unrelated_tag_shape_establishes_absence( + tmp_path, unrelated_table, metadata, populated +) -> None: + path = tmp_path / "unrelated-tags.sqlite" + create_custom_relationship_schema( + path, metadata=metadata, tables=(unrelated_table,) + ) + if populated: + table_name = unrelated_table.split()[2] + with sqlite3.connect(path) as connection: + connection.execute(f"INSERT INTO {table_name} VALUES (100, 200)") + + tags, report, manager_report = read_tags_and_aggregate(path) + + assert tags == {} + expected_storage = ( + RelationshipStorage.UNKNOWN if metadata else RelationshipStorage.ABSENT + ) + assert report.storage == expected_storage + assert report.complete == (not metadata) + assert manager_report.complete == (not metadata) + assert manager_report.relationships["transaction_tags"].storage == expected_storage + + +@pytest.mark.parametrize("populated", [False, True]) +@pytest.mark.parametrize("include_unrelated", [False, True]) +@pytest.mark.parametrize(("transaction_ent", "tag_ent"), [(36, 35), (37, 36), (61, 58)]) +def test_unrelated_controls_do_not_conflict_with_valid_direct_tags( + tmp_path, populated, include_unrelated, transaction_ent, tag_ent +) -> None: + path = tmp_path / "unrelated-controls-and-direct-tags.sqlite" + table_name = f"Z_{transaction_ent}TAGS" + tables = [ + f"CREATE TABLE {table_name} (Z_{transaction_ent}TRANSACTIONS INTEGER, Z_{tag_ent}TAGS INTEGER)" + ] + if include_unrelated: + tables.extend((INFO_CARD_TAG_TABLE, SCHEDULED_TAG_TABLE)) + create_custom_relationship_schema( + path, + metadata=((tag_ent, "Tag", 0, 0), (transaction_ent, "Transaction", 0, 0)), + tables=tables, + ) + if populated: + with sqlite3.connect(path) as connection: + connection.execute(f"INSERT INTO {table_name} VALUES (100, 200)") + if include_unrelated: + connection.execute("INSERT INTO Z_23TAGS VALUES (101, 201)") + connection.execute("INSERT INTO Z_31TAGS VALUES (102, 202)") + + tags, report, manager_report = read_tags_and_aggregate(path) + + assert tags == ({100: [200]} if populated else {}) + assert report.storage == RelationshipStorage.PRESENT + assert report.storage_name == table_name + assert report.complete + assert manager_report.complete + assert ( + manager_report.relationships["transaction_tags"].storage + == RelationshipStorage.PRESENT + ) + + +@pytest.mark.parametrize("populated", [False, True]) +@pytest.mark.parametrize( + "metadata", + [ + (), + ((36, "Tag", 0, 0), (37, "Transaction", 0, 0)), + ((37, "Transaction", 0, 0),), + ((36, "Tag", 0, 0),), + ], +) +@pytest.mark.parametrize( + "ambiguous_table", + [ + "CREATE TABLE Z_31TAGS (Z_31SCHEDULEDTRANSACTIONS1 INTEGER)", + "CREATE TABLE Z_31TAGS " + "(Z_31SCHEDULEDTRANSACTIONS1 INTEGER, Z_35TAGS2 INTEGER, " + "Z_31TRANSACTIONS INTEGER)", + "CREATE TABLE Z_23TAGS (Z_23INFOCARDS5 INTEGER)", + "CREATE TABLE Z_23TAGS " + "(Z_23INFOCARDS5 INTEGER, Z_35TAGS1 INTEGER, " + "Z_23TRANSACTIONS INTEGER)", + ], +) +def test_incomplete_or_hybrid_unrelated_tag_shape_is_unknown( + tmp_path, metadata, ambiguous_table, populated +) -> None: + path = tmp_path / "ambiguous-unrelated-tags.sqlite" + create_custom_relationship_schema( + path, metadata=metadata, tables=(ambiguous_table,) + ) + if populated: + table_name = ambiguous_table.split()[2] + columns_count = ambiguous_table.count("INTEGER") + with sqlite3.connect(path) as connection: + connection.execute( + f"INSERT INTO {table_name} VALUES ({', '.join('?' for _ in range(columns_count))})", + (1,) * columns_count, + ) + + tags, report, manager_report = read_tags_and_aggregate(path) + + assert tags == {} + assert report.storage == RelationshipStorage.UNKNOWN + assert not report.complete + assert not manager_report.complete + assert ( + manager_report.relationships["transaction_tags"].storage + == RelationshipStorage.UNKNOWN + ) diff --git a/tests/unit/test_schema_cache_binding.py b/tests/unit/test_schema_cache_binding.py new file mode 100644 index 0000000..2b0b99a --- /dev/null +++ b/tests/unit/test_schema_cache_binding.py @@ -0,0 +1,594 @@ +from contextlib import contextmanager +from pathlib import Path +import sqlite3 + +import pytest + +from moneywiz_api import MoneywizApi +from moneywiz_api.database_accessor import DatabaseAccessor, DatabaseSchemaError +from moneywiz_api.model.investment_holding import InvestmentHolding +from moneywiz_api.schema_profile import detect_schema_profile + + +SCHEMA_CHANGED = "database schema changed; close and reopen the accessor" + + +def open_writer(path: Path): + connection = sqlite3.connect(path) + connection.execute("PRAGMA journal_mode=WAL") + return connection + + +def create_account_store(path: Path, *, extra_metadata=()) -> None: + connection = open_writer(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", + [ + (8, "SyncObject", 0, 0), + (9, "Account", 8, 0), + (10, "CashAccount", 9, 1), + *extra_metadata, + ], + ) + connection.execute( + "CREATE TABLE ZSYNCOBJECT (" + "Z_PK INTEGER PRIMARY KEY, Z_ENT INTEGER, ZOBJECTCREATIONDATE FLOAT, " + "ZGID TEXT, ZDISPLAYORDER INTEGER, ZGROUPID INTEGER, ZNAME TEXT, " + "ZCURRENCYNAME TEXT, ZOPENINGBALANCE FLOAT, ZINFO TEXT, ZUSER INTEGER)" + ) + connection.execute( + "INSERT INTO ZSYNCOBJECT VALUES " + "(1, 10, 0, 'account-1', 1, 1, 'Before', 'EUR', 0, NULL, 1)" + ) + connection.commit() + connection.close() + + +def remap_cash_account(path: Path) -> None: + connection = open_writer(path) + connection.execute( + "UPDATE Z_PRIMARYKEY SET Z_NAME = 'RetiredCashAccount' WHERE Z_ENT = 10" + ) + connection.execute("INSERT INTO Z_PRIMARYKEY VALUES (11, 'CashAccount', 9, 1)") + connection.execute( + "UPDATE ZSYNCOBJECT SET Z_ENT = 11, ZNAME = 'After' WHERE Z_PK = 1" + ) + connection.commit() + connection.close() + + +def create_holding_store(path: Path) -> None: + connection = open_writer(path) + connection.execute( + "CREATE TABLE Z_PRIMARYKEY " + "(Z_ENT INTEGER, Z_NAME TEXT, Z_SUPER INTEGER, Z_MAX INTEGER)" + ) + connection.executemany( + "INSERT INTO Z_PRIMARYKEY VALUES (?, ?, ?, ?)", + [(8, "SyncObject", 0, 0), (24, "InvestmentHolding", 8, 1)], + ) + connection.execute( + "CREATE TABLE ZSYNCOBJECT (" + "Z_PK INTEGER PRIMARY KEY, Z_ENT INTEGER, ZOBJECTCREATIONDATE FLOAT, " + "ZGID TEXT, ZINVESTMENTACCOUNT INTEGER, ZOPENNINGNUMBEROFSHARES FLOAT, " + "ZNUMBEROFSHARES FLOAT, ZPRICEPERSHARE FLOAT, ZSYMBOL TEXT, " + "ZHOLDINGTYPE TEXT, ZDESC TEXT, ZISPRICEPERSHAREAVAILABLEONLINE INTEGER, " + "ZINVESTMENTOBJECTTYPE INTEGER, ZCOSTBASISOFMISSINGOBSHARES FLOAT)" + ) + connection.execute( + "INSERT INTO ZSYNCOBJECT VALUES " + "(1, 24, 0, 'holding-1', 5, NULL, 2, 10, 'ACME', NULL, 'Acme', 0, 0, 0)" + ) + connection.commit() + connection.close() + + +def migrate_holding_aliases(path: Path) -> None: + connection = open_writer(path) + connection.execute( + "ALTER TABLE ZSYNCOBJECT RENAME COLUMN ZNUMBEROFSHARES TO ZNUMBEROFSHARES1" + ) + connection.execute( + "ALTER TABLE ZSYNCOBJECT RENAME COLUMN ZPRICEPERSHARE TO ZPRICEPERSHARE1" + ) + connection.commit() + connection.close() + + +def create_relationship_store(path: Path) -> None: + create_account_store( + path, + extra_metadata=( + (2, "CategoryAssigment", 8, 1), + (36, "Tag", 8, 1), + (37, "Transaction", 8, 1), + (50, "WithdrawRefundTransactionLink", 8, 1), + ), + ) + connection = open_writer(path) + connection.execute( + "CREATE TABLE ZCATEGORYASSIGMENT " + "(Z_PK INTEGER, ZCATEGORY INTEGER, ZTRANSACTION INTEGER, ZAMOUNT FLOAT)" + ) + connection.execute("INSERT INTO ZCATEGORYASSIGMENT VALUES (1, 20, 30, 5)") + connection.execute( + "CREATE TABLE ZWITHDRAWREFUNDTRANSACTIONLINK " + "(Z_PK INTEGER, ZREFUNDTRANSACTION INTEGER, ZWITHDRAWTRANSACTION INTEGER)" + ) + connection.execute("INSERT INTO ZWITHDRAWREFUNDTRANSACTIONLINK VALUES (2, 31, 32)") + connection.execute( + "CREATE TABLE Z_37TAGS (Z_37TRANSACTIONS INTEGER, Z_36TAGS INTEGER)" + ) + connection.execute("INSERT INTO Z_37TAGS VALUES (30, 40)") + connection.commit() + connection.close() + + +def assert_old_account_generation( + api, manager, record, report, schema_identity +) -> None: + assert api.account_manager is manager + assert manager.get(1) is record + assert manager.get_by_gid("account-1") is record + assert manager.load_report is report + assert api.accessor._schema_identity is schema_identity + assert api._loaded_managers == {"accounts"} + snapshot = api.snapshot(("accounts",)).as_dict() + assert snapshot["records"]["accounts"][0]["name"] == "Before" + assert snapshot["completeness"]["managers"]["accounts"] == report.as_dict() + + +def test_metadata_remap_refuses_and_preserves_published_generation(tmp_path) -> None: + path = tmp_path / "metadata-remap.sqlite" + create_account_store(path) + with MoneywizApi(path, managers=("accounts",)) as api: + manager = api.account_manager + record = manager.get(1) + records = manager._records + gid_index = manager._gid_to_id + report = manager.load_report + schema_identity = api.accessor._schema_identity + cached_maps = ( + api.accessor._ent_to_typename, + api.accessor._ent_to_super, + api.accessor._typename_to_ent, + ) + remap_cash_account(path) + + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + api.load(("accounts",)) + + assert manager._records is records + assert manager._gid_to_id is gid_index + assert ( + api.accessor._ent_to_typename, + api.accessor._ent_to_super, + api.accessor._typename_to_ent, + ) == cached_maps + assert_old_account_generation(api, manager, record, report, schema_identity) + assert not api.accessor._con.in_transaction + + with MoneywizApi(path, managers=("accounts",)) as reopened: + assert reopened.account_manager.get(1).name == "After" + assert reopened.completeness().managers["accounts"].complete + + +@pytest.mark.parametrize( + "mutation", + [ + "UPDATE Z_PRIMARYKEY SET Z_NAME = 'CashAccount2' WHERE Z_ENT = 10", + "UPDATE Z_PRIMARYKEY SET Z_SUPER = 8 WHERE Z_ENT = 10", + "INSERT INTO Z_PRIMARYKEY VALUES (11, 'FutureAccount', 9, 0)", + "DELETE FROM Z_PRIMARYKEY WHERE Z_ENT = 10", + ], +) +def test_each_entity_metadata_change_refuses_reload(tmp_path, mutation) -> None: + path = tmp_path / "metadata-change.sqlite" + create_account_store(path) + with MoneywizApi(path, managers=("accounts",)) as api: + connection = open_writer(path) + connection.execute(mutation) + connection.commit() + connection.close() + + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + api.load(("accounts",)) + + assert api.account_manager.get(1).name == "Before" + + +def test_complete_metadata_inventory_has_no_thousand_row_cutoff(tmp_path) -> None: + path = tmp_path / "large-metadata.sqlite" + filler = tuple((ent_id, f"Entity{ent_id}", 8, 0) for ent_id in range(11, 1011)) + create_account_store(path, extra_metadata=filler) + connection = open_writer(path) + connection.execute( + "UPDATE Z_PRIMARYKEY SET Z_ENT = 1011 WHERE Z_NAME = 'CashAccount'" + ) + connection.execute("UPDATE ZSYNCOBJECT SET Z_ENT = 1011") + connection.commit() + connection.close() + + with MoneywizApi(path, managers=("accounts",)) as api: + assert len(api.accessor._schema_identity[1]) == 1003 + assert api.accessor.ent_for("CashAccount") == 1011 + assert "CashAccount" in api.accessor.descendant_typenames(("Account",)) + assert api.account_manager.get(1).name == "Before" + + +@pytest.mark.parametrize("duplicate", ["id", "name"]) +def test_duplicate_entity_map_keys_are_rejected(tmp_path, duplicate) -> None: + path = tmp_path / f"duplicate-{duplicate}.sqlite" + create_account_store(path) + connection = open_writer(path) + if duplicate == "id": + connection.execute( + "INSERT INTO Z_PRIMARYKEY VALUES (10, 'OtherCashAccount', 9, 0)" + ) + else: + connection.execute("INSERT INTO Z_PRIMARYKEY VALUES (11, 'CashAccount', 9, 0)") + connection.commit() + connection.close() + + with pytest.raises(DatabaseSchemaError, match="duplicate entity"): + DatabaseAccessor(path) + + +def test_physical_alias_drift_refuses_profiled_getters(tmp_path) -> None: + path = tmp_path / "holding-aliases.sqlite" + create_holding_store(path) + with DatabaseAccessor(path) as accessor: + baseline_profile = accessor.schema_profile + assert accessor.get_record(1, InvestmentHolding).number_of_shares == 2 + migrate_holding_aliases(path) + + for operation in ( + lambda: accessor.get_record(1, InvestmentHolding), + lambda: accessor.get_record_by_gid("holding-1", InvestmentHolding), + ): + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + operation() + assert accessor.schema_profile is baseline_profile + assert not accessor._con.in_transaction + + with DatabaseAccessor(path) as reopened: + assert reopened.schema_profile.profile_id == "suffixed-investment-columns" + assert reopened.get_record(1, InvestmentHolding).number_of_shares == 2 + + +@pytest.mark.parametrize( + "reader", + [ + "query_objects", + "read_category_assignments", + "get_category_assignment", + "read_refund_maps", + "get_refund_maps", + "read_tags_map", + "get_tags_map", + ], +) +def test_every_cache_dependent_reader_refuses_physical_drift(tmp_path, reader) -> None: + path = tmp_path / f"reader-{reader}.sqlite" + create_relationship_store(path) + with DatabaseAccessor(path) as accessor: + initial_profile = accessor.schema_profile + connection = open_writer(path) + connection.execute("ALTER TABLE ZCATEGORYASSIGMENT ADD COLUMN ZEXTRA") + connection.commit() + connection.close() + assert detect_schema_profile(accessor._con) == initial_profile + + operation = getattr(accessor, reader) + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + operation(["CashAccount"]) if reader == "query_objects" else operation() + + +@pytest.mark.parametrize( + "ddl", + [ + "CREATE TABLE Z_99TAGS (Z_99TRANSACTIONS INTEGER, Z_36TAGS INTEGER)", + "DROP TABLE ZCATEGORYASSIGMENT", + "ALTER TABLE ZCATEGORYASSIGMENT ADD COLUMN ZEXTRA", + ], +) +def test_relationship_schema_create_drop_and_alter_refuse_with_equal_profile( + tmp_path, ddl +) -> None: + path = tmp_path / "relationship-ddl.sqlite" + create_relationship_store(path) + with DatabaseAccessor(path) as accessor: + profile = accessor.schema_profile + connection = open_writer(path) + connection.execute(ddl) + connection.commit() + connection.close() + assert detect_schema_profile(accessor._con) == profile + + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + accessor.read_tags_map() + + +def test_schema_refusal_preserves_transaction_relationship_generation(tmp_path) -> None: + path = tmp_path / "relationship-state.sqlite" + create_relationship_store(path) + with MoneywizApi(path, managers=("accounts", "transactions")) as api: + manager = api.transaction_manager + report = manager.load_report + categories = manager.category_assignment + refunds = manager.refund_maps + tags = manager.tags_map + records = manager._records + gid_index = manager._gid_to_id + schema_identity = api.accessor._schema_identity + loaded_managers = set(api._loaded_managers) + connection = open_writer(path) + connection.execute("ALTER TABLE ZCATEGORYASSIGMENT ADD COLUMN ZEXTRA") + connection.commit() + connection.close() + + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + api.load(("transactions",)) + + assert api.transaction_manager is manager + assert manager.load_report is report + assert manager.category_assignment is categories + assert manager.refund_maps is refunds + assert manager.tags_map is tags + assert manager._records is records + assert manager._gid_to_id is gid_index + assert api.accessor._schema_identity is schema_identity + assert api._loaded_managers == loaded_managers + assert api.completeness().managers["transactions"] is report + + +def test_valid_direct_and_nested_reads_share_snapshot_ownership(tmp_path) -> None: + path = tmp_path / "nested.sqlite" + create_relationship_store(path) + with DatabaseAccessor(path) as accessor: + assert not accessor._con.in_transaction + with accessor.read_transaction(): + assert accessor._con.in_transaction + assert accessor.query_objects(["CashAccount"])[0]["Z_PK"] == 1 + assert accessor.get_record(1).gid == "account-1" + assert accessor.get_record_by_gid("account-1").id == 1 + assert accessor.read_category_assignments()[0] == {30: [(20, 5)]} + assert accessor.get_category_assignment() == {30: [(20, 5)]} + assert accessor.read_refund_maps()[0] == {31: 32} + assert accessor.get_refund_maps() == {31: 32} + assert accessor.read_tags_map()[0] == {30: [40]} + assert accessor.get_tags_map() == {30: [40]} + with accessor.read_transaction(): + assert accessor._con.in_transaction + assert accessor._con.in_transaction + assert not accessor._con.in_transaction + + +def test_normal_data_and_zmax_growth_reload_successfully(tmp_path) -> None: + path = tmp_path / "growth.sqlite" + create_account_store(path) + with MoneywizApi(path, managers=("accounts",)) as api: + manager = api.account_manager + connection = open_writer(path) + connection.execute("UPDATE Z_PRIMARYKEY SET Z_MAX = 2 WHERE Z_ENT = 10") + connection.execute("UPDATE ZSYNCOBJECT SET ZNAME = 'Edited' WHERE Z_PK = 1") + connection.execute( + "INSERT INTO ZSYNCOBJECT VALUES " + "(2, 10, 0, 'account-2', 2, 1, 'Added', 'EUR', 0, NULL, 1)" + ) + connection.commit() + connection.close() + + report = api.load(("accounts",)).managers["accounts"] + + assert api.account_manager is manager + assert report.complete + assert report.source_ids == (1, 2) + assert manager.get(1).name == "Edited" + assert manager.get(2).name == "Added" + + +def test_initialization_publishes_binding_only_after_transaction_exit( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "initialization-publication.sqlite" + create_account_store(path) + original_transaction = DatabaseAccessor._raw_read_transaction + exit_states = [] + + @contextmanager + def observe_transaction(accessor): + with original_transaction(accessor): + yield + exit_states.append( + (hasattr(accessor, "_schema_identity"), accessor._con.in_transaction) + ) + + monkeypatch.setattr(DatabaseAccessor, "_raw_read_transaction", observe_transaction) + + with DatabaseAccessor(path) as accessor: + assert exit_states == [(False, False)] + assert accessor._schema_identity + + +def test_missing_schema_binding_cannot_bypass_public_guard() -> None: + accessor = DatabaseAccessor.__new__(DatabaseAccessor) + accessor._con = sqlite3.connect(":memory:") + + with pytest.raises(DatabaseSchemaError, match="cache is not initialized"): + accessor.query_objects([]) + + assert not accessor._con.in_transaction + accessor.close() + + +def test_initialization_race_binds_one_old_snapshot_then_refuses(tmp_path, monkeypatch): + import moneywiz_api.database_accessor as accessor_module + + path = tmp_path / "initialization-race.sqlite" + create_holding_store(path) + original_detect = accessor_module.detect_schema_profile + migration_committed = False + + def migrate_then_detect(connection): + nonlocal migration_committed + writer = open_writer(path) + writer.execute( + "ALTER TABLE ZSYNCOBJECT RENAME COLUMN ZNUMBEROFSHARES TO ZNUMBEROFSHARES1" + ) + writer.execute( + "ALTER TABLE ZSYNCOBJECT RENAME COLUMN ZPRICEPERSHARE TO ZPRICEPERSHARE1" + ) + writer.execute( + "UPDATE Z_PRIMARYKEY SET Z_NAME = 'RetiredHolding' WHERE Z_ENT = 24" + ) + writer.execute( + "INSERT INTO Z_PRIMARYKEY VALUES (25, 'InvestmentHolding', 8, 1)" + ) + writer.execute("UPDATE ZSYNCOBJECT SET Z_ENT = 25") + writer.commit() + writer.close() + migration_committed = True + return original_detect(connection) + + monkeypatch.setattr(accessor_module, "detect_schema_profile", migrate_then_detect) + accessor = DatabaseAccessor(path) + + assert migration_committed + assert accessor.schema_profile.profile_id == "unsuffixed-investment-columns" + assert accessor.ent_for("InvestmentHolding") == 24 + assert (24, "InvestmentHolding", 8) in accessor._schema_identity[1] + assert (25, "InvestmentHolding", 8) not in accessor._schema_identity[1] + with accessor._raw_read_transaction(): + assert accessor._read_schema_identity() != accessor._schema_identity + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + accessor.query_objects(["InvestmentHolding"]) + accessor.close() + + monkeypatch.setattr(accessor_module, "detect_schema_profile", original_detect) + with DatabaseAccessor(path) as reopened: + assert reopened.schema_profile.profile_id == "suffixed-investment-columns" + assert reopened.ent_for("InvestmentHolding") == 25 + + +def test_first_load_schema_race_closes_constructor_accessor(tmp_path, monkeypatch): + import moneywiz_api.moneywiz_api as api_module + + path = tmp_path / "first-load.sqlite" + create_account_store(path) + accessor = DatabaseAccessor(path) + remap_cash_account(path) + monkeypatch.setattr(api_module, "DatabaseAccessor", lambda _path: accessor) + + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + api_module.MoneywizApi(path, managers=("accounts",)) + + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + accessor._con.execute("SELECT 1") + + +def test_reload_race_reads_old_snapshot_then_next_load_refuses( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "reload-race.sqlite" + create_account_store(path) + with MoneywizApi(path, managers=("accounts",)) as api: + original_verify = api.accessor._verify_schema_identity + migration_committed = False + + def verify_then_migrate(): + nonlocal migration_committed + original_verify() + if not migration_committed: + remap_cash_account(path) + migration_committed = True + + monkeypatch.setattr( + api.accessor, "_verify_schema_identity", verify_then_migrate + ) + + report = api.load(("accounts",)).managers["accounts"] + + assert migration_committed + assert report.complete + assert report.source_ids == (1,) + assert api.account_manager.get(1).name == "Before" + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + api.load(("accounts",)) + assert api.account_manager.get(1).name == "Before" + + +def test_transient_signature_query_error_preserves_state_and_retry( + tmp_path, monkeypatch +) -> None: + path = tmp_path / "signature-error.sqlite" + create_account_store(path) + with MoneywizApi(path, managers=("accounts",)) as api: + manager = api.account_manager + record = manager.get(1) + report = manager.load_report + schema_identity = api.accessor._schema_identity + original_reader = api.accessor._read_schema_identity + + def fail_signature_query(): + raise sqlite3.OperationalError("PRIVATE_SCHEMA_QUERY_ERROR") + + monkeypatch.setattr(api.accessor, "_read_schema_identity", fail_signature_query) + with pytest.raises(DatabaseSchemaError, match="could not be verified") as error: + api.load(("accounts",)) + assert "PRIVATE_SCHEMA_QUERY_ERROR" not in str(error.value) + assert_old_account_generation(api, manager, record, report, schema_identity) + assert not api.accessor._con.in_transaction + + monkeypatch.setattr(api.accessor, "_read_schema_identity", original_reader) + connection = open_writer(path) + connection.execute("UPDATE ZSYNCOBJECT SET ZNAME = 'Retried' WHERE Z_PK = 1") + connection.commit() + connection.close() + assert api.load(("accounts",)).complete + assert manager.get(1).name == "Retried" + + +def test_invalid_metadata_refusal_can_retry_after_metadata_restore(tmp_path) -> None: + path = tmp_path / "invalid-metadata.sqlite" + create_account_store(path) + with MoneywizApi(path, managers=("accounts",)) as api: + connection = open_writer(path) + connection.execute("UPDATE Z_PRIMARYKEY SET Z_NAME = NULL WHERE Z_ENT = 10") + connection.commit() + connection.close() + + with pytest.raises(DatabaseSchemaError, match="invalid metadata"): + api.load(("accounts",)) + assert api.account_manager.get(1).name == "Before" + assert not api.accessor._con.in_transaction + + connection = open_writer(path) + connection.execute( + "UPDATE Z_PRIMARYKEY SET Z_NAME = 'CashAccount' WHERE Z_ENT = 10" + ) + connection.execute("UPDATE ZSYNCOBJECT SET ZNAME = 'Restored' WHERE Z_PK = 1") + connection.commit() + connection.close() + assert api.load(("accounts",)).complete + assert api.account_manager.get(1).name == "Restored" + + +def test_existing_unverified_transaction_is_checked_and_not_rolled_back( + tmp_path, +) -> None: + path = tmp_path / "caller-transaction.sqlite" + create_account_store(path) + with DatabaseAccessor(path) as accessor: + remap_cash_account(path) + accessor._con.execute("BEGIN") + + with pytest.raises(DatabaseSchemaError, match=f"^{SCHEMA_CHANGED}$"): + accessor.query_objects(["CashAccount"]) + + assert accessor._con.in_transaction + accessor._con.rollback() diff --git a/tests/unit/test_schema_profiles.py b/tests/unit/test_schema_profiles.py index 3579411..27f4892 100644 --- a/tests/unit/test_schema_profiles.py +++ b/tests/unit/test_schema_profiles.py @@ -19,18 +19,32 @@ def test_detects_suffixed_fixture_profile() -> None: assert profile.profile_id == "suffixed-investment-columns" assert profile.holding_number_of_shares_column == "ZNUMBEROFSHARES1" assert profile.transaction_number_of_shares_column == "ZNUMBEROFSHARES1" - assert profile.price_per_share_column == "ZPRICEPERSHARE1" + assert profile.holding_price_per_share_column == "ZPRICEPERSHARE1" + assert profile.transaction_price_per_share_column == "ZPRICEPERSHARE1" def test_detects_unsuffixed_live_profile() -> None: - connection = make_connection("ZNUMBEROFSHARES", "ZPRICEPERSHARE", "ZPRICEPERSHARE1") + connection = make_connection("ZNUMBEROFSHARES", "ZPRICEPERSHARE") profile = detect_schema_profile(connection) assert profile.profile_id == "unsuffixed-investment-columns" assert profile.holding_number_of_shares_column == "ZNUMBEROFSHARES" assert profile.transaction_number_of_shares_column == "ZNUMBEROFSHARES" - assert profile.price_per_share_column == "ZPRICEPERSHARE" + assert profile.holding_price_per_share_column == "ZPRICEPERSHARE" + assert profile.transaction_price_per_share_column == "ZPRICEPERSHARE" + + +def test_detects_consumer_specific_mixed_price_columns() -> None: + connection = make_connection("ZNUMBEROFSHARES", "ZPRICEPERSHARE", "ZPRICEPERSHARE1") + + profile = detect_schema_profile(connection) + + assert profile.profile_id == "mixed-investment-columns" + assert profile.holding_number_of_shares_column == "ZNUMBEROFSHARES" + assert profile.transaction_number_of_shares_column == "ZNUMBEROFSHARES" + assert profile.holding_price_per_share_column == "ZPRICEPERSHARE" + assert profile.transaction_price_per_share_column == "ZPRICEPERSHARE1" def test_detects_mixed_investment_columns() -> None: @@ -41,7 +55,8 @@ def test_detects_mixed_investment_columns() -> None: assert profile.profile_id == "mixed-investment-columns" assert profile.holding_number_of_shares_column == "ZNUMBEROFSHARES" assert profile.transaction_number_of_shares_column == "ZNUMBEROFSHARES" - assert profile.price_per_share_column == "ZPRICEPERSHARE1" + assert profile.holding_price_per_share_column == "ZPRICEPERSHARE1" + assert profile.transaction_price_per_share_column == "ZPRICEPERSHARE1" def test_detects_consumer_specific_mixed_share_columns() -> None: @@ -54,10 +69,11 @@ def test_detects_consumer_specific_mixed_share_columns() -> None: assert profile.profile_id == "mixed-investment-columns" assert profile.holding_number_of_shares_column == "ZNUMBEROFSHARES" assert profile.transaction_number_of_shares_column == "ZNUMBEROFSHARES1" - assert profile.price_per_share_column == "ZPRICEPERSHARE1" + assert profile.holding_price_per_share_column == "ZPRICEPERSHARE1" + assert profile.transaction_price_per_share_column == "ZPRICEPERSHARE1" -def test_rejects_ambiguous_investment_column_profile() -> None: +def test_detects_consumer_specific_mixed_share_and_price_columns() -> None: connection = make_connection( "ZNUMBEROFSHARES", "ZNUMBEROFSHARES1", @@ -67,8 +83,11 @@ def test_rejects_ambiguous_investment_column_profile() -> None: profile = detect_schema_profile(connection) - assert profile.profile_id == "unknown" - assert not profile.is_known + assert profile.profile_id == "mixed-investment-columns" + assert profile.holding_number_of_shares_column == "ZNUMBEROFSHARES" + assert profile.transaction_number_of_shares_column == "ZNUMBEROFSHARES1" + assert profile.holding_price_per_share_column == "ZPRICEPERSHARE" + assert profile.transaction_price_per_share_column == "ZPRICEPERSHARE1" def test_decimal_alias_reads_both_profiles() -> None: diff --git a/tests/unit/test_transaction_compatibility.py b/tests/unit/test_transaction_compatibility.py index 907706b..aa7a0e6 100644 --- a/tests/unit/test_transaction_compatibility.py +++ b/tests/unit/test_transaction_compatibility.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager from decimal import Decimal import pytest @@ -67,6 +68,10 @@ class TransferWithdrawAccessor: def __init__(self, rows): self.rows = rows + @contextmanager + def read_transaction(self): + yield + def query_objects(self, _typenames): return self.rows diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 0000000..5b1b56d --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,26 @@ +import os +import time +from datetime import datetime, timezone + +import pytest + +from moneywiz_api.utils import get_date, get_datetime + + +@pytest.mark.skipif(not hasattr(time, "tzset"), reason="tzset is Unix-only") +def test_core_data_epoch_uses_utc_and_returns_naive_local_time(monkeypatch) -> None: + original_tz = os.environ.get("TZ") + monkeypatch.setenv("TZ", "America/New_York") + time.tzset() + try: + assert get_datetime(0) == datetime(2000, 12, 31, 19, 0) + summer = datetime(2024, 7, 1, 12, 0, tzinfo=timezone.utc) + core_data_value = get_date(summer) + assert get_datetime(core_data_value) == datetime(2024, 7, 1, 8, 0) + assert get_date(get_datetime(core_data_value)) == core_data_value + finally: + if original_tz is None: + monkeypatch.delenv("TZ", raising=False) + else: + monkeypatch.setenv("TZ", original_tz) + time.tzset()