From c0dd5acf5a55fe17baa7d9d0a3e852cce4b42a0a Mon Sep 17 00:00:00 2001 From: wackloner Date: Sun, 19 Jul 2026 14:16:12 +0700 Subject: [PATCH 01/10] fix replay-safe asset payouts --- BOARD.md | 5 +- Makefile | 35 +- api/nft_lottery.py | 295 ++++++--- api/wallet.py | 52 +- docs/architecture/outbound-asset-transfers.md | 52 ++ flex/application/asset_transfers.py | 335 +++++++++++ flex/application/transfer_runtime.py | 24 + flex/blockchain/asset_transfers.py | 199 +++++++ flex/db/asset_transfer_intents.py | 135 +++++ flex/db/cometa_database.py | 4 + flex/db/indexes.py | 55 +- flex/db/model/airdrop.py | 34 ++ flex/db/model/priced.py | 18 +- flex/db/model/transfers.py | 51 ++ flex/domain/allocation.py | 80 +++ flex/tools/airdrop.py | 560 +++++++++++++++--- pyproject.toml | 2 + tests/unit/test_airdrop.py | 455 ++++++++++++++ .../test_algorand_asset_transfer_gateway.py | 151 +++++ tests/unit/test_allocation.py | 73 +++ tests/unit/test_asset_transfer_repository.py | 78 +++ tests/unit/test_asset_transfers.py | 300 ++++++++++ tests/unit/test_database_indexes.py | 36 ++ tests/unit/test_nft_lottery_payouts.py | 117 ++++ tests/unit/test_wallet_transfers.py | 42 ++ 25 files changed, 2973 insertions(+), 215 deletions(-) create mode 100644 docs/architecture/outbound-asset-transfers.md create mode 100644 flex/application/asset_transfers.py create mode 100644 flex/application/transfer_runtime.py create mode 100644 flex/blockchain/asset_transfers.py create mode 100644 flex/db/asset_transfer_intents.py create mode 100644 flex/db/model/airdrop.py create mode 100644 flex/db/model/transfers.py create mode 100644 flex/domain/allocation.py create mode 100644 tests/unit/test_airdrop.py create mode 100644 tests/unit/test_algorand_asset_transfer_gateway.py create mode 100644 tests/unit/test_allocation.py create mode 100644 tests/unit/test_asset_transfer_repository.py create mode 100644 tests/unit/test_asset_transfers.py create mode 100644 tests/unit/test_nft_lottery_payouts.py create mode 100644 tests/unit/test_wallet_transfers.py diff --git a/BOARD.md b/BOARD.md index 359197ba..e9a19aac 100644 --- a/BOARD.md +++ b/BOARD.md @@ -1,6 +1,6 @@ # Cometa Backend — Task Board -> Last updated: 2026-07-18 +> Last updated: 2026-07-19 ## Conventions @@ -8,7 +8,7 @@ - **Statuses**: `todo` | `in_progress` | `blocked` | `done` - **Priorities**: `critical` | `high` | `medium` | `low` - **Tags**: `security` | `backend` | `infra` | `dx` | `arch` | `perf` -- Next available ID: **CB-078** +- Next available ID: **CB-079** ## Active @@ -18,6 +18,7 @@ | CB-074 | Atomic event projection | todo | critical | backend, arch | Crash-safe inbox/projector with duplicate, replay, and recovery tests | | CB-075 | Isolate transaction signing | todo | high | security, arch | Read-only API boundary; authenticated policy-limited signing service | | CB-076 | Async persistence boundary | todo | high | backend, perf | Storage outages cannot block the event loop; timeouts and readiness covered | +| CB-078 | Replay-safe outbound asset payouts | done | critical | security, backend, arch | Exact allocations, immutable airdrop manifests, persisted signed intents, on-chain reconciliation, and regression tests | ## Completed milestones diff --git a/Makefile b/Makefile index 209a122d..672c6452 100644 --- a/Makefile +++ b/Makefile @@ -1,28 +1,34 @@ PYTHON_LINT_PATHS := \ - api/background.py app.py blockchain/indexer.py bot/log.py env.py telegram_bot.py \ + api/background.py api/nft_lottery.py api/wallet.py app.py blockchain/indexer.py bot/log.py env.py telegram_bot.py \ core/circuit_breaker.py core/cometa.py core/decorators.py core/util.py \ - flex/__init__.py flex/api.py flex/application flex/blockchain/contract_state.py flex/data/asset_prices.py \ + flex/__init__.py flex/api.py flex/application flex/blockchain/asset_transfers.py \ + flex/blockchain/contract_state.py flex/data/asset_prices.py \ flex/data/lp_prices.py flex/data/lp_states.py flex/data/pool_state.py \ flex/data/tinyman_lps.py flex/data/transactions.py \ - flex/db/classes/collection_manager.py flex/db/indexes.py \ - flex/db/model/liquidity_pools.py \ - flex/db/model/priced.py flex/domain flex/providers/pact.py flex/providers/price_router.py \ + flex/db/asset_transfer_intents.py flex/db/classes/collection_manager.py flex/db/indexes.py \ + flex/db/model/airdrop.py flex/db/model/liquidity_pools.py \ + flex/db/model/priced.py flex/db/model/transfers.py flex/domain flex/providers/pact.py flex/providers/price_router.py \ flex/providers/vestige.py flex/sync_pools.py \ - scripts/verify_algorand_credentials.py tests + flex/tools/airdrop.py scripts/verify_algorand_credentials.py tests PYTHON_MODERN_PATHS := \ - api/background.py core/circuit_breaker.py flex/application flex/blockchain/contract_state.py \ - flex/data/asset_prices.py flex/data/lp_prices.py flex/db/model/priced.py \ + api/background.py api/nft_lottery.py api/wallet.py core/circuit_breaker.py flex/application \ + flex/blockchain/asset_transfers.py \ + flex/blockchain/contract_state.py flex/data/asset_prices.py flex/data/lp_prices.py \ + flex/db/asset_transfer_intents.py flex/db/model/airdrop.py flex/db/model/priced.py flex/db/model/transfers.py \ flex/domain flex/providers/pact.py flex/providers/price_router.py tests/unit PYTHON_FORMAT_PATHS := \ - api/background.py app.py blockchain/indexer.py bot/log.py core/circuit_breaker.py core/cometa.py core/util.py \ - flex/api.py flex/application flex/blockchain/contract_state.py flex/data/asset_prices.py flex/data/lp_prices.py \ + api/background.py api/nft_lottery.py api/wallet.py app.py blockchain/indexer.py bot/log.py \ + core/circuit_breaker.py core/cometa.py core/util.py \ + flex/api.py flex/application flex/blockchain/asset_transfers.py flex/blockchain/contract_state.py \ + flex/data/asset_prices.py flex/data/lp_prices.py \ flex/data/lp_states.py flex/data/pool_state.py flex/data/tinyman_lps.py \ - flex/data/transactions.py flex/db/indexes.py flex/db/model/liquidity_pools.py \ - flex/db/model/priced.py flex/domain flex/providers/pact.py flex/providers/price_router.py \ + flex/data/transactions.py flex/db/asset_transfer_intents.py flex/db/indexes.py \ + flex/db/model/airdrop.py flex/db/model/liquidity_pools.py flex/db/model/priced.py flex/db/model/transfers.py \ + flex/domain flex/providers/pact.py flex/providers/price_router.py \ flex/providers/vestige.py flex/sync_pools.py telegram_bot.py \ - tests/conftest.py tests/unit + flex/tools/airdrop.py tests/conftest.py tests/unit .PHONY: sync run lint format format-check typecheck test quality @@ -50,8 +56,11 @@ test: pipenv run pytest tests \ --cov=core.circuit_breaker \ --cov=core.decorators \ + --cov=flex.application.asset_transfers \ + --cov=flex.db.asset_transfer_intents \ --cov=flex.db.classes.collection_manager \ --cov=flex.blockchain.contract_state \ + --cov=flex.domain.allocation \ --cov=flex.domain.pricing \ --cov=flex.domain.transactions \ --cov=flex.providers.pact \ diff --git a/api/nft_lottery.py b/api/nft_lottery.py index 0062f56b..c2aab2f0 100644 --- a/api/nft_lottery.py +++ b/api/nft_lottery.py @@ -2,27 +2,31 @@ import random import time from dataclasses import dataclass -from datetime import datetime -from enum import Enum -from typing import Optional +from datetime import UTC, datetime +from enum import StrEnum +from hashlib import sha256 +from typing import Any +from uuid import uuid4 from algosdk.v2client import indexer from dataclasses_json import dataclass_json +from pymongo import ReturnDocument import flex from api.swaps import SwapInfo -from api.wallet import send_nft, cometa_public_key +from api.wallet import send_nft from blockchain.nfts import get_nft_info -from blockchain.node import init_algod_client, get_current_round +from blockchain.node import init_algod_client from core.db.db_manager import DbManager from env import settings - +from flex.blockchain.base import cometa_public_key MIN_DRAW_INTERVAL = 60 * 60 * 24 # 24 hours -class LotteryType(str, Enum): - SWAP = 'swap' - STAKING = 'staking' + +class LotteryType(StrEnum): + SWAP = "swap" + STAKING = "staking" def __str__(self): return self.value @@ -36,31 +40,38 @@ class NftLottery: min_amount: int probability: float available_nfts: list[int] - win_title: str = 'You have won a prize NFT!' - max_amount: Optional[int] = None - type: Optional[str] = None - only_for_buy: Optional[bool] = None - pool_id: Optional[int] = None + win_title: str = "You have won a prize NFT!" + max_amount: int | None = None + type: str | None = None + only_for_buy: bool | None = None + pool_id: int | None = None def is_eligible(self, entity_id: int, amount: float) -> bool: - return (entity_id == self.asset_id or entity_id == self.pool_id) and amount >= self.min_amount and ( - self.max_amount is None or amount <= self.max_amount) + return ( + (entity_id == self.asset_id or entity_id == self.pool_id) + and amount >= self.min_amount + and (self.max_amount is None or amount <= self.max_amount) + ) @dataclass_json @dataclass class LotteryDraw: wallet: str - prize: Optional[int] - timestamp: Optional[float] = None - lottery_name: Optional[str] = None + prize: int | None + timestamp: float | None = None + lottery_name: str | None = None claimed: bool = False - created_date: Optional[datetime] = None - send_error: Optional[str] = None + created_date: datetime | None = None + send_error: str | None = None + id: str | None = None + payout_operation_id: str | None = None + payout_txid: str | None = None + confirmed_round: int | None = None def __post_init__(self): if self.timestamp: - self.created_date = datetime.fromtimestamp(self.timestamp) + self.created_date = datetime.fromtimestamp(self.timestamp, tz=UTC) @dataclass_json @@ -71,17 +82,80 @@ class LotteryParticipant: last_draw_block: int -nft_lotteries = DbManager[NftLottery](settings.db_name, 'nft_lotteries', 'name', NftLottery) -lottery_draws = DbManager[LotteryDraw](settings.db_name, 'lottery_draws', 'swap_txid', LotteryDraw) -lottery_participants = DbManager[LotteryParticipant](settings.db_name, 'lottery_participants', 'address', LotteryParticipant) +nft_lotteries = DbManager[NftLottery](settings.db_name, "nft_lotteries", "name", NftLottery) +lottery_draws = DbManager[LotteryDraw](settings.db_name, "lottery_draws", "id", LotteryDraw) +lottery_participants = DbManager[LotteryParticipant]( + settings.db_name, "lottery_participants", "address", LotteryParticipant +) algod_client = init_algod_client() -indexer_client = indexer.IndexerClient(indexer_token=settings.algod_token, indexer_address=settings.algo_indexer_address) +indexer_client = indexer.IndexerClient( + indexer_token=settings.algod_token, indexer_address=settings.algo_indexer_address +) logger = logging.getLogger(__name__) -def draw_id(lottery: NftLottery) -> Optional[int]: +def ensure_lottery_indexes() -> None: + lottery_draws.collection.create_index( + "id", + unique=True, + name="id_unique", + partialFilterExpression={"id": {"$type": "string"}}, + ) + + +def _create_draw( + *, + lottery_name: str, + prize: int | None, + wallet: str, + timestamp: float, +) -> LotteryDraw: + ensure_lottery_indexes() + return lottery_draws.create( + LotteryDraw( + id=uuid4().hex, + lottery_name=lottery_name, + prize=prize, + wallet=wallet, + timestamp=timestamp, + ) + ) + + +def _backfill_draw_id(document: dict[str, Any]) -> tuple[LotteryDraw, Any]: + document_id = document.get("_id") + if document_id is None: + raise RuntimeError("lottery draw is missing its MongoDB identity") + + draw_id_value = document.get("id") + if not isinstance(draw_id_value, str) or not draw_id_value: + draw_id_value = f"legacy-{sha256(f'lottery-draw:{document_id}'.encode()).hexdigest()}" + updated = lottery_draws.collection.find_one_and_update( + { + "_id": document_id, + "$or": [ + {"id": {"$exists": False}}, + {"id": None}, + ], + }, + {"$set": {"id": draw_id_value}}, + return_document=ReturnDocument.AFTER, + ) + if updated is None: + updated = lottery_draws.collection.find_one({"_id": document_id}) + if updated is None or updated.get("id") != draw_id_value: + raise RuntimeError("lottery draw identity changed during migration") + document = updated + + draw = LotteryDraw.from_dict(document) + if draw.id is None: + raise RuntimeError("lottery draw migration did not persist an ID") + return draw, document_id + + +def draw_id(lottery: NftLottery) -> int | None: if random.random() > lottery.probability: return None while True: @@ -90,11 +164,11 @@ def draw_id(lottery: NftLottery) -> Optional[int]: # TODO: refactor to get balance once res = random.choice(lottery.available_nfts) data = indexer_client.lookup_account_assets(address=cometa_public_key, asset_id=res) - assets = data.get('assets', []) - if len(assets) > 0 and assets[0].get('amount', 0) > 0: + assets = data.get("assets", []) + if len(assets) > 0 and assets[0].get("amount", 0) > 0: # drawn nft persists in the wallet break - logger.info(f'NFT {res} is not in the wallet, drawing again') + logger.info(f"NFT {res} is not in the wallet, drawing again") lottery.available_nfts.remove(res) nft_lotteries.update(lottery) return res @@ -108,33 +182,32 @@ class NftPrize: title: str -def get_nft_prize(lottery: NftLottery, asa_id: Optional[int]) -> Optional[NftPrize]: +def get_nft_prize(lottery: NftLottery, asa_id: int | None) -> NftPrize | None: if asa_id is None: return None prize_info = get_nft_info(asa_id) - PINATA_URL = 'https://gateway.pinata.cloud/ipfs/' - image_url = prize_info.image_url.replace('ipfs://', PINATA_URL) + PINATA_URL = "https://gateway.pinata.cloud/ipfs/" + image_url = prize_info.image_url.replace("ipfs://", PINATA_URL) - return NftPrize(asa_id=asa_id, - name=prize_info.name, - image_url=image_url, - title=lottery.win_title) + return NftPrize(asa_id=asa_id, name=prize_info.name, image_url=image_url, title=lottery.win_title) -def draw_prize(lottery: NftLottery, address: str) -> Optional[NftPrize]: - logger.debug(f'Drawing lottery {lottery.name} for {address}') +def draw_prize(lottery: NftLottery, address: str) -> NftPrize | None: + logger.debug(f"Drawing lottery {lottery.name} for {address}") prize_id = draw_id(lottery) - lottery_draws.create(LotteryDraw(lottery_name=lottery.name, - prize=prize_id, - wallet=address, - timestamp=time.time())) + _create_draw( + lottery_name=lottery.name, + prize=prize_id, + wallet=address, + timestamp=time.time(), + ) return get_nft_prize(lottery, prize_id) -def lottery_for_swap(swap: SwapInfo) -> Optional[NftPrize]: - lotteries = nft_lotteries.get_many({'type': LotteryType.SWAP}) - logger.debug(f'Swap lotteries cnt = {len(lotteries)}') +def lottery_for_swap(swap: SwapInfo) -> NftPrize | None: + lotteries = nft_lotteries.get_many({"type": LotteryType.SWAP}) + logger.debug(f"Swap lotteries cnt = {len(lotteries)}") prize = None for lottery in lotteries: @@ -146,104 +219,138 @@ def lottery_for_swap(swap: SwapInfo) -> Optional[NftPrize]: return prize -async def lottery_for_staking(pool_id: int, address: str) -> Optional[NftPrize]: - lotteries = nft_lotteries.get_many({'type': LotteryType.STAKING, 'pool_id': pool_id}) +async def lottery_for_staking(pool_id: int, address: str) -> NftPrize | None: + lotteries = nft_lotteries.get_many({"type": LotteryType.STAKING, "pool_id": pool_id}) if not lotteries: - logger.info(f'No lotteries found for pools_id {pool_id}') + logger.info(f"No lotteries found for pools_id {pool_id}") return None - logger.debug(f'Lotteries found for pools_id {pool_id}: {lotteries}') + logger.debug(f"Lotteries found for pools_id {pool_id}: {lotteries}") pool_state = flex.db.pool_states.get_one(pool_id=pool_id) if pool_state is None: - logger.error(f'Pool state not found for pool {pool_id}') + logger.error(f"Pool state not found for pool {pool_id}") return None user_state = flex.db.user_states.get_one(address=address) if user_state is None: - logger.error(f'User state not found for address {address}') + logger.error(f"User state not found for address {address}") return None address_stake_micros = user_state.pool_by_address.get(pool_state.address) if address_stake_micros is None: - logger.info(f'No staking found for address {address} in pool {pool_id}') + logger.info(f"No staking found for address {address} in pool {pool_id}") return None lottery = None - for l in lotteries: - if len(l.available_nfts) == 0: - logger.warning(f'NFTS are OVER for lottery {l.name}') - nft_lotteries.remove(l) - logger.info(f'Lottery {l.name} removed') + for candidate in lotteries: + if len(candidate.available_nfts) == 0: + logger.warning(f"NFTS are OVER for lottery {candidate.name}") + nft_lotteries.remove(candidate) + logger.info(f"Lottery {candidate.name} removed") continue - if address_stake_micros >= l.min_amount and (lottery is None or l.probability > lottery.probability): - lottery = l + if address_stake_micros >= candidate.min_amount and ( + lottery is None or candidate.probability > lottery.probability + ): + lottery = candidate if lottery is None: - logger.info(f'No lottery found for address {address} in pool {pool_id}') + logger.info(f"No lottery found for address {address} in pool {pool_id}") return None - logger.info(f'Lottery {lottery.name} for pool {pool_id} and address {address} started') + logger.info(f"Lottery {lottery.name} for pool {pool_id} and address {address} started") - address_draws = lottery_draws.get_many({'wallet': address, 'lottery_name': lottery.name}) + address_draws = lottery_draws.get_many({"wallet": address, "lottery_name": lottery.name}) now_timestamp = time.time() # TODO: optimize the check, get only last timestamp if len(address_draws) > 0: last_draw_timestamp = max([d.timestamp for d in address_draws]) if now_timestamp - last_draw_timestamp < MIN_DRAW_INTERVAL: - logger.info(f'Lottery {lottery.name} for pool {pool_id} and address {address} already drawn recently') + logger.info(f"Lottery {lottery.name} for pool {pool_id} and address {address} already drawn recently") return None prize_id = draw_id(lottery) - lottery_draws.create( - LotteryDraw( - lottery_name=lottery.name, - prize=prize_id, - wallet=address, - timestamp=now_timestamp - ) + _create_draw( + lottery_name=lottery.name, + prize=prize_id, + wallet=address, + timestamp=now_timestamp, ) - logger.info(f'The prize is {prize_id}') + logger.info(f"The prize is {prize_id}") if prize_id is not None: prize_info = get_nft_prize(lottery, prize_id) - logger.info(f'Prize info: {prize_info}') + logger.info(f"Prize info: {prize_info}") return prize_info return None def send_all_prizes(): - logger.info('Sending all failed NFT prizes...') + logger.info("Sending all failed NFT prizes...") + ensure_lottery_indexes() res = [] sent_count = 0 error_count = 0 - for draw in lottery_draws.get_many({'claimed': False, 'prize': {'$ne': None}}): - info = { - 'wallet': draw.wallet, - 'prize': draw.prize, - 'lottery': draw.lottery_name - } + documents = list(lottery_draws.collection.find({"claimed": False, "prize": {"$ne": None}})) + for document in documents: + draw, document_id = _backfill_draw_id(document) + if draw.claimed or draw.prize is None: + continue + info = {"wallet": draw.wallet, "prize": draw.prize, "lottery": draw.lottery_name} try: - send_nft(draw.wallet, draw.prize) - info['sent'] = datetime.utcnow() - draw.claimed = True - sent_count += 1 + idempotency_key = f"lottery:{draw.id}" + payout_operation_id = f"nft:{idempotency_key}" + if draw.payout_operation_id not in (None, payout_operation_id): + raise RuntimeError("lottery draw belongs to a different payout operation") + lottery_draws.collection.update_one( + {"_id": document_id, "claimed": False}, + { + "$set": { + "payout_operation_id": payout_operation_id, + "send_error": None, + } + }, + ) + receipt = send_nft( + draw.wallet, + draw.prize, + idempotency_key=idempotency_key, + ) + info["txid"] = receipt.txid + info["sent"] = datetime.now(UTC) + claimed = lottery_draws.collection.find_one_and_update( + {"_id": document_id, "claimed": False}, + { + "$set": { + "claimed": True, + "payout_operation_id": receipt.operation_id, + "payout_txid": receipt.txid, + "confirmed_round": receipt.confirmed_round, + "send_error": None, + } + }, + return_document=ReturnDocument.AFTER, + ) + if claimed is None: + claimed = lottery_draws.collection.find_one({"_id": document_id}) + if claimed is None or not claimed.get("claimed") or claimed.get("payout_txid") != receipt.txid: + raise RuntimeError("lottery draw changed while recording its confirmed payout") + info["already_claimed"] = True + else: + sent_count += 1 except Exception as e: - info['error'] = str(e) - draw.send_error = str(e) + info["error"] = str(e) + lottery_draws.collection.update_one( + {"_id": document_id, "claimed": False}, + {"$set": {"send_error": str(e)[:500]}}, + ) error_count += 1 - logger.info(f'Sent NFT: {info}') + logger.info(f"Sent NFT: {info}") - lottery_draws.update(draw) res.append(info) - - return { - 'sent_count': sent_count, - 'error_count': error_count, - 'results': res - } + return {"sent_count": sent_count, "error_count": error_count, "results": res} diff --git a/api/wallet.py b/api/wallet.py index 96899c21..1fdb8679 100644 --- a/api/wallet.py +++ b/api/wallet.py @@ -1,29 +1,37 @@ -import logging - -from algosdk import mnemonic, account -from algosdk.transaction import AssetTransferTxn, wait_for_confirmation +"""Wallet-side asset transfer operations.""" -from blockchain.node import init_algod_client -from env import settings +import logging -cometa_private_key = mnemonic.to_private_key(settings.algo_mnemonic) -cometa_public_key = account.address_from_private_key(cometa_private_key) +from flex.application.asset_transfers import ( + AssetTransferReceipt, + AssetTransferRequest, +) +from flex.application.transfer_runtime import get_asset_transfer_service -algod = init_algod_client() logger = logging.getLogger(__name__) -def send_nft(address: str, nft_id: int, amount: int = 1) -> None: - logger.info(f'Sending {amount} NFT {nft_id} to {address}') - params = algod.suggested_params() - txn = AssetTransferTxn( - sender=cometa_public_key, - sp=params, - receiver=address, - amt=amount, - index=nft_id) - stxn = txn.sign(cometa_private_key) +def send_nft( + address: str, + nft_id: int, + amount: int = 1, + *, + idempotency_key: str, +) -> AssetTransferReceipt: + """Send an NFT once for a stable business operation.""" - txid = algod.send_transaction(stxn) - wait_for_confirmation(algod, txid) - logger.info(f'Sent {amount} NFT {nft_id} to {address} with tx {txid}') + receipt = get_asset_transfer_service().execute( + AssetTransferRequest( + operation_id=f"nft:{idempotency_key}", + receiver=address, + asset_id=nft_id, + amount_micros=amount, + ) + ) + logger.info( + "NFT transfer %s confirmed in round %s as %s", + receipt.operation_id, + receipt.confirmed_round, + receipt.txid, + ) + return receipt diff --git a/docs/architecture/outbound-asset-transfers.md b/docs/architecture/outbound-asset-transfers.md new file mode 100644 index 00000000..18751117 --- /dev/null +++ b/docs/architecture/outbound-asset-transfers.md @@ -0,0 +1,52 @@ +# Replay-Safe Outbound Asset Transfers + +Cometa treats every payout as a durable business operation, not as a retryable +SDK call. Callers provide a stable operation ID such as +`airdrop::
` or `nft:lottery:`. + +Before the first broadcast, the service stores: + +- immutable receiver, ASA, amount, and note; +- the signed Algorand transaction and its transaction ID; +- its first/last valid rounds and execution status. + +A retry loads and rebroadcasts the same signed payload. Before network I/O, the +adapter verifies its signature, transaction ID, sender, receiver, ASA, amount, +note, lease, validity window, and absence of close, clawback, group, or rekey +fields. This closes the crash window where Algorand accepted a transaction but +MongoDB did not record the result and prevents a corrupted intent from sending +another valid treasury transaction. + +Algorand uint64 values are stored as decimal strings because BSON integers are +signed int64. Conflicting immutable intent or manifest IDs abort index setup; +the application never deletes financial evidence automatically. + +A deterministic Algorand lease adds defense in depth, but persistence of the +exact signed transaction is the primary idempotency mechanism. If the +transaction expires without confirmed status, the service stops and requires +on-chain reconciliation before any replacement can be authorized. + +## Airdrop invariants + +`send_airdrop` uses exact rational arithmetic and the largest-remainder method, +so recipient base units sum to the declared budget exactly. The first run also +reserves an immutable SHA-256 manifest covering the asset, total, complete +recipient set, allocations, and selected notes. Reusing an `airdrop_id` with a +different manifest fails before any broadcast. + +Legacy campaigns have no trustworthy complete-recipient manifest. They +therefore fail closed until explicitly reviewed and migrated. Legacy reward +transactions must be confirmed on-chain and match the stored sender, receiver, +ASA, and amount before they can be marked complete. + +Operational retries must always reuse the original operation or airdrop ID. +`AirdropIncompleteError` reports unresolved recipients and transaction IDs; +never invent a replacement ID to bypass reconciliation. + +## Lottery payouts + +Every lottery draw receives an immutable ID. Legacy draws are assigned a stable +ID derived from their MongoDB identity before payment. The payout intent is +stored against that ID, and the exact draw is marked claimed with a conditional +update only after confirmation. Multiple workers may race safely: they resolve +to the same persisted transaction and cannot update another draw. diff --git a/flex/application/asset_transfers.py b/flex/application/asset_transfers.py new file mode 100644 index 00000000..e17859df --- /dev/null +++ b/flex/application/asset_transfers.py @@ -0,0 +1,335 @@ +"""Idempotent application service for outbound Algorand asset transfers.""" + +from dataclasses import dataclass +from typing import Protocol + +from flex.db.model.transfers import AssetTransferIntent + +MAX_ALGORAND_UINT = 2**64 - 1 +MAX_NOTE_BYTES = 1_000 +MAX_OPERATION_ID_BYTES = 200 + + +class AssetTransferError(RuntimeError): + """Base class for outbound transfer failures.""" + + +class InvalidAssetTransferError(ValueError): + """Raised before persistence when a transfer request is invalid.""" + + +class AssetTransferConflictError(AssetTransferError): + """Raised when an idempotency key is reused for a different transfer.""" + + +class AssetTransferPendingError(AssetTransferError): + """Raised when broadcast outcome is ambiguous and must be reconciled.""" + + def __init__(self, operation_id: str, txid: str) -> None: + super().__init__(f"transfer {operation_id!r} is unresolved; reconcile transaction {txid}") + self.operation_id = operation_id + self.txid = txid + + +class AssetTransferExpiredError(AssetTransferError): + """Raised when an unconfirmed persisted transaction is no longer valid.""" + + def __init__(self, operation_id: str, txid: str) -> None: + super().__init__( + f"transfer {operation_id!r} expired unconfirmed; verify transaction {txid} on-chain before replacement" + ) + self.operation_id = operation_id + self.txid = txid + + +class TransferStatus: + PREPARED = "prepared" + SUBMITTED = "submitted" + CONFIRMED = "confirmed" + + +@dataclass(frozen=True, slots=True) +class AssetTransferRequest: + operation_id: str + receiver: str + asset_id: int + amount_micros: int + note: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.operation_id, str) or not self.operation_id.strip(): + raise InvalidAssetTransferError("operation_id must be a non-empty string") + if len(self.operation_id.encode()) > MAX_OPERATION_ID_BYTES: + raise InvalidAssetTransferError("operation_id is too long") + if not isinstance(self.receiver, str) or not self.receiver.strip(): + raise InvalidAssetTransferError("receiver must be a non-empty string") + if isinstance(self.asset_id, bool) or not isinstance(self.asset_id, int): + raise InvalidAssetTransferError("asset_id must be an integer") + if not 0 < self.asset_id <= MAX_ALGORAND_UINT: + raise InvalidAssetTransferError("asset_id is outside the Algorand uint64 range") + if isinstance(self.amount_micros, bool) or not isinstance(self.amount_micros, int): + raise InvalidAssetTransferError("amount_micros must be an integer") + if not 0 < self.amount_micros <= MAX_ALGORAND_UINT: + raise InvalidAssetTransferError("amount_micros is outside the Algorand uint64 range") + if self.note is not None: + if not isinstance(self.note, str): + raise InvalidAssetTransferError("note must be a string") + if len(self.note.encode()) > MAX_NOTE_BYTES: + raise InvalidAssetTransferError("note exceeds Algorand's 1000-byte limit") + + +@dataclass(frozen=True, slots=True) +class PreparedAssetTransfer: + signed_transaction: str + txid: str + first_valid_round: int + last_valid_round: int + + +@dataclass(frozen=True, slots=True) +class AssetTransferReceipt: + operation_id: str + txid: str + confirmed_round: int + already_confirmed: bool + + +@dataclass(frozen=True, slots=True) +class ConfirmedAssetTransfer: + txid: str + sender: str + receiver: str + asset_id: int + amount_micros: int + confirmed_round: int + + +@dataclass(frozen=True, slots=True) +class TransferReconciliation: + operation_id: str + txid: str + status: str + confirmed_round: int | None + + +class AssetTransferIntentRepository(Protocol): + def get(self, operation_id: str) -> AssetTransferIntent | None: ... + + def reserve(self, intent: AssetTransferIntent) -> AssetTransferIntent: ... + + def record_attempt(self, operation_id: str, txid: str) -> AssetTransferIntent: ... + + def mark_submitted(self, operation_id: str, txid: str) -> AssetTransferIntent: ... + + def mark_confirmed( + self, + operation_id: str, + txid: str, + confirmed_round: int, + ) -> AssetTransferIntent: ... + + def record_error( + self, + operation_id: str, + txid: str, + error: str, + ) -> AssetTransferIntent: ... + + +class AssetTransferGateway(Protocol): + def prepare(self, request: AssetTransferRequest) -> PreparedAssetTransfer: ... + + def broadcast( + self, + prepared: PreparedAssetTransfer, + request: AssetTransferRequest, + ) -> str: ... + + def wait_for_confirmation(self, txid: str) -> int: ... + + def lookup_confirmed_round(self, txid: str) -> int | None: ... + + def lookup_confirmed_transfer(self, txid: str) -> ConfirmedAssetTransfer | None: ... + + def current_round(self) -> int: ... + + +@dataclass(slots=True) +class AssetTransferService: + repository: AssetTransferIntentRepository + gateway: AssetTransferGateway + + def validate(self, request: AssetTransferRequest) -> None: + """Reject an idempotency-key conflict without touching the network.""" + + intent = self.repository.get(request.operation_id) + if intent is not None: + self._assert_same_transfer(intent, request) + + def lookup_confirmed_round(self, txid: str) -> int | None: + """Look up a legacy transaction without creating a new intent.""" + + return self.gateway.lookup_confirmed_round(txid) + + def lookup_confirmed_transfer(self, txid: str) -> ConfirmedAssetTransfer | None: + """Return authoritative on-chain fields for legacy reconciliation.""" + + return self.gateway.lookup_confirmed_transfer(txid) + + def execute(self, request: AssetTransferRequest) -> AssetTransferReceipt: + intent = self.repository.get(request.operation_id) + if intent is None: + prepared = self.gateway.prepare(request) + intent = self.repository.reserve( + AssetTransferIntent( + id=request.operation_id, + receiver=request.receiver, + asset_id=str(request.asset_id), + amount_micros=str(request.amount_micros), + note=request.note, + signed_transaction=prepared.signed_transaction, + txid=prepared.txid, + first_valid_round=prepared.first_valid_round, + last_valid_round=prepared.last_valid_round, + status=TransferStatus.PREPARED, + ) + ) + + self._assert_same_transfer(intent, request) + if intent.status == TransferStatus.CONFIRMED: + return self._confirmed_receipt(intent, already_confirmed=True) + + if self.gateway.current_round() > intent.last_valid_round: + try: + confirmed_round = self.gateway.lookup_confirmed_round(intent.txid) + except Exception as exc: + self.repository.record_error( + intent.id, + intent.txid, + f"expired transaction lookup unavailable after {type(exc).__name__}", + ) + raise AssetTransferExpiredError(intent.id, intent.txid) from exc + if confirmed_round is not None: + intent = self.repository.mark_confirmed( + intent.id, + intent.txid, + confirmed_round, + ) + return self._confirmed_receipt(intent, already_confirmed=True) + + self.repository.record_error( + intent.id, + intent.txid, + "signed transaction expired before confirmation; manual reconciliation required", + ) + raise AssetTransferExpiredError(intent.id, intent.txid) + + self.repository.record_attempt(intent.id, intent.txid) + broadcast_error: Exception | None = None + try: + returned_txid = self.gateway.broadcast( + PreparedAssetTransfer( + signed_transaction=intent.signed_transaction, + txid=intent.txid, + first_valid_round=intent.first_valid_round, + last_valid_round=intent.last_valid_round, + ), + request, + ) + if returned_txid != intent.txid: + raise AssetTransferError("Algorand node returned a different transaction ID") + self.repository.mark_submitted(intent.id, intent.txid) + except Exception as exc: + # A transport failure can happen after the node accepted the + # transaction. Confirmation of the persisted txid is authoritative. + broadcast_error = exc + + try: + confirmed_round = self.gateway.wait_for_confirmation(intent.txid) + except Exception as exc: + failure_kind = type(broadcast_error or exc).__name__ + self.repository.record_error( + intent.id, + intent.txid, + f"confirmation unresolved after {failure_kind}", + ) + raise AssetTransferPendingError(intent.id, intent.txid) from exc + + intent = self.repository.mark_confirmed( + intent.id, + intent.txid, + confirmed_round, + ) + return self._confirmed_receipt(intent, already_confirmed=False) + + def reconcile(self, operation_id: str) -> TransferReconciliation: + intent = self.repository.get(operation_id) + if intent is None: + raise AssetTransferError(f"transfer {operation_id!r} does not exist") + if intent.status == TransferStatus.CONFIRMED: + receipt = self._confirmed_receipt(intent, already_confirmed=True) + return TransferReconciliation( + operation_id=receipt.operation_id, + txid=receipt.txid, + status=TransferStatus.CONFIRMED, + confirmed_round=receipt.confirmed_round, + ) + + confirmed_round = self.gateway.lookup_confirmed_round(intent.txid) + if confirmed_round is not None: + confirmed = self.repository.mark_confirmed( + intent.id, + intent.txid, + confirmed_round, + ) + return TransferReconciliation( + operation_id=confirmed.id, + txid=confirmed.txid, + status=TransferStatus.CONFIRMED, + confirmed_round=confirmed.confirmed_round, + ) + + status = "expired_unconfirmed" if self.gateway.current_round() > intent.last_valid_round else intent.status + return TransferReconciliation( + operation_id=intent.id, + txid=intent.txid, + status=status, + confirmed_round=None, + ) + + @staticmethod + def _assert_same_transfer( + intent: AssetTransferIntent, + request: AssetTransferRequest, + ) -> None: + persisted = ( + intent.receiver, + intent.asset_id_int, + intent.amount_micros_int, + intent.note, + ) + requested = ( + request.receiver, + request.asset_id, + request.amount_micros, + request.note, + ) + if persisted != requested: + raise AssetTransferConflictError( + f"idempotency key {request.operation_id!r} already belongs to a different transfer" + ) + + @staticmethod + def _confirmed_receipt( + intent: AssetTransferIntent, + *, + already_confirmed: bool, + ) -> AssetTransferReceipt: + if intent.confirmed_round is None: + raise AssetTransferError("confirmed transfer is missing confirmed_round") + return AssetTransferReceipt( + operation_id=intent.id, + txid=intent.txid, + confirmed_round=intent.confirmed_round, + already_confirmed=already_confirmed, + ) diff --git a/flex/application/transfer_runtime.py b/flex/application/transfer_runtime.py new file mode 100644 index 00000000..d5e46b0d --- /dev/null +++ b/flex/application/transfer_runtime.py @@ -0,0 +1,24 @@ +"""Composition root for outbound asset transfer infrastructure.""" + +from functools import lru_cache + +from flex import db +from flex.application.asset_transfers import AssetTransferService +from flex.blockchain.asset_transfers import AlgorandAssetTransferGateway +from flex.blockchain.base import algod_client, cometa_private_key, cometa_public_key, indexer_client +from flex.db.asset_transfer_intents import MongoAssetTransferIntentRepository + + +@lru_cache(maxsize=1) +def get_asset_transfer_service() -> AssetTransferService: + repository = MongoAssetTransferIntentRepository( + db.asset_transfer_intents.mongodb_collection, + ) + repository.ensure_indexes() + gateway = AlgorandAssetTransferGateway( + algod=algod_client, + indexer=indexer_client, + sender=cometa_public_key, + private_key=cometa_private_key, + ) + return AssetTransferService(repository=repository, gateway=gateway) diff --git a/flex/blockchain/asset_transfers.py b/flex/blockchain/asset_transfers.py new file mode 100644 index 00000000..3f7d328f --- /dev/null +++ b/flex/blockchain/asset_transfers.py @@ -0,0 +1,199 @@ +"""Algorand adapter for preparing and submitting persisted asset transfers.""" + +import base64 +import hmac +from dataclasses import dataclass +from hashlib import sha256 +from typing import Any + +from algosdk import account, encoding, transaction +from algosdk.error import AlgodHTTPError +from algosdk.v2client.algod import AlgodClient +from algosdk.v2client.indexer import IndexerClient + +from flex.application.asset_transfers import ( + AssetTransferRequest, + ConfirmedAssetTransfer, + InvalidAssetTransferError, + PreparedAssetTransfer, +) + +TX_WAIT_ROUNDS = 4 +LEASE_DOMAIN = b"cometa-asset-transfer:v1:" + + +@dataclass(slots=True) +class AlgorandAssetTransferGateway: + algod: AlgodClient + indexer: IndexerClient + sender: str + private_key: str + wait_rounds: int = TX_WAIT_ROUNDS + + def prepare(self, request: AssetTransferRequest) -> PreparedAssetTransfer: + if not encoding.is_valid_address(request.receiver): + raise InvalidAssetTransferError("receiver is not a valid Algorand address") + + params = self.algod.suggested_params() + unsigned_transaction = transaction.AssetTransferTxn( + sender=self.sender, + sp=params, + receiver=request.receiver, + amt=request.amount_micros, + index=request.asset_id, + note=request.note.encode() if request.note is not None else None, + lease=sha256(LEASE_DOMAIN + request.operation_id.encode()).digest(), + ) + signed_transaction = unsigned_transaction.sign(self.private_key) + return PreparedAssetTransfer( + signed_transaction=encoding.msgpack_encode(signed_transaction), + txid=unsigned_transaction.get_txid(), + first_valid_round=unsigned_transaction.first_valid_round, + last_valid_round=unsigned_transaction.last_valid_round, + ) + + def broadcast( + self, + prepared: PreparedAssetTransfer, + request: AssetTransferRequest, + ) -> str: + decoded = encoding.msgpack_decode(prepared.signed_transaction) + if not isinstance(decoded, transaction.SignedTransaction): + raise InvalidAssetTransferError("persisted payload is not a signed transaction") + self._validate_persisted_transaction(decoded, prepared, request) + return self.algod.send_transaction(decoded) + + def _validate_persisted_transaction( + self, + signed: transaction.SignedTransaction, + prepared: PreparedAssetTransfer, + request: AssetTransferRequest, + ) -> None: + txn = signed.transaction + expected_note = request.note.encode() if request.note is not None else None + expected_lease = sha256(LEASE_DOMAIN + request.operation_id.encode()).digest() + expected_authorizer = account.address_from_private_key(self.private_key) + expected_signer = None if expected_authorizer == self.sender else expected_authorizer + + expected_fields = ( + self.sender, + request.receiver, + request.asset_id, + request.amount_micros, + expected_note, + expected_lease, + prepared.first_valid_round, + prepared.last_valid_round, + None, + None, + None, + None, + ) + observed_fields = ( + txn.sender, + getattr(txn, "receiver", None), + getattr(txn, "index", None), + getattr(txn, "amount", None), + txn.note, + txn.lease, + txn.first_valid_round, + txn.last_valid_round, + getattr(txn, "close_assets_to", None), + getattr(txn, "revocation_target", None), + txn.group, + txn.rekey_to, + ) + if not isinstance(txn, transaction.AssetTransferTxn) or observed_fields != expected_fields: + raise InvalidAssetTransferError("persisted transaction fields do not match the transfer intent") + if txn.get_txid() != prepared.txid: + raise InvalidAssetTransferError("persisted transaction ID does not match the transfer intent") + if signed.authorizing_address != expected_signer: + raise InvalidAssetTransferError("persisted transaction has an unexpected authorizing address") + + expected_signature = base64.b64encode(txn.raw_sign(self.private_key)).decode() + if not isinstance(signed.signature, str) or not hmac.compare_digest( + signed.signature, + expected_signature, + ): + raise InvalidAssetTransferError("persisted transaction signature is invalid") + + def wait_for_confirmation(self, txid: str) -> int: + response = transaction.wait_for_confirmation( + self.algod, + txid, + self.wait_rounds, + ) + return self._confirmed_round(response) + + def lookup_confirmed_round(self, txid: str) -> int | None: + try: + pending = self.algod.pending_transaction_info(txid) + except AlgodHTTPError as exc: + if exc.code != 404: + raise + else: + confirmed_round = self._optional_confirmed_round(pending) + if confirmed_round is not None: + return confirmed_round + + # The indexer is the durable lookup path after algod's pending window. + observed = self.lookup_confirmed_transfer(txid) + return observed.confirmed_round if observed is not None else None + + def lookup_confirmed_transfer(self, txid: str) -> ConfirmedAssetTransfer | None: + response = self.indexer.transaction(txid) + transaction_info = response.get("transaction") + if not isinstance(transaction_info, dict): + raise RuntimeError("Algorand indexer response has no transaction") + confirmed_round = self._optional_confirmed_round(transaction_info) + if confirmed_round is None: + return None + + sender = transaction_info.get("sender") + asset_transfer = transaction_info.get("asset-transfer-transaction") + if not isinstance(sender, str) or not encoding.is_valid_address(sender): + raise RuntimeError("Algorand indexer response has an invalid sender") + if not isinstance(asset_transfer, dict): + raise RuntimeError("Algorand transaction is not an asset transfer") + + receiver = asset_transfer.get("receiver") + asset_id = asset_transfer.get("asset-id") + amount_micros = asset_transfer.get("amount") + if not isinstance(receiver, str) or not encoding.is_valid_address(receiver): + raise RuntimeError("Algorand indexer response has an invalid receiver") + if isinstance(asset_id, bool) or not isinstance(asset_id, int) or asset_id <= 0: + raise RuntimeError("Algorand indexer response has an invalid asset ID") + if isinstance(amount_micros, bool) or not isinstance(amount_micros, int) or amount_micros <= 0: + raise RuntimeError("Algorand indexer response has an invalid transfer amount") + + return ConfirmedAssetTransfer( + txid=txid, + sender=sender, + receiver=receiver, + asset_id=asset_id, + amount_micros=amount_micros, + confirmed_round=confirmed_round, + ) + + def current_round(self) -> int: + status = self.algod.status() + current_round = status.get("last-round") + if isinstance(current_round, bool) or not isinstance(current_round, int) or current_round < 0: + raise RuntimeError("Algorand node returned an invalid current round") + return current_round + + @staticmethod + def _confirmed_round(response: dict[str, Any]) -> int: + confirmed_round = AlgorandAssetTransferGateway._optional_confirmed_round(response) + if confirmed_round is None: + raise RuntimeError("Algorand confirmation response has no confirmed round") + return confirmed_round + + @staticmethod + def _optional_confirmed_round(response: dict[str, Any]) -> int | None: + confirmed_round = response.get("confirmed-round") + if confirmed_round is None or confirmed_round == 0: + return None + if isinstance(confirmed_round, bool) or not isinstance(confirmed_round, int) or confirmed_round < 0: + raise RuntimeError("Algorand response contains an invalid confirmed round") + return confirmed_round diff --git a/flex/db/asset_transfer_intents.py b/flex/db/asset_transfer_intents.py new file mode 100644 index 00000000..de63bdf2 --- /dev/null +++ b/flex/db/asset_transfer_intents.py @@ -0,0 +1,135 @@ +"""MongoDB repository for persisted outbound transfer intents.""" + +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any + +from pymongo import ReturnDocument +from pymongo.collection import Collection +from pymongo.errors import DuplicateKeyError + +from flex.db.model.transfers import AssetTransferIntent + + +class TransferIntentPersistenceError(RuntimeError): + """Raised when an expected transfer intent cannot be persisted.""" + + +@dataclass(slots=True) +class MongoAssetTransferIntentRepository: + collection: Collection[dict[str, Any]] + + def ensure_indexes(self) -> None: + self.collection.create_index("id", unique=True, name="id_unique") + self.collection.create_index( + [("status", 1), ("updated", 1)], + name="status_updated_idx", + ) + + def get(self, operation_id: str) -> AssetTransferIntent | None: + document = self.collection.find_one({"id": operation_id}) + return self._from_document(document) + + def reserve(self, intent: AssetTransferIntent) -> AssetTransferIntent: + try: + document = self.collection.find_one_and_update( + {"id": intent.id}, + {"$setOnInsert": intent.to_dict()}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + except DuplicateKeyError: + document = self.collection.find_one({"id": intent.id}) + reserved = self._from_document(document) + if reserved is None: + raise TransferIntentPersistenceError(f"failed to reserve transfer {intent.id!r}") + return reserved + + def record_attempt(self, operation_id: str, txid: str) -> AssetTransferIntent: + return self._update( + operation_id, + txid, + { + "$inc": {"attempt_count": 1}, + "$set": {"updated": datetime.now(UTC)}, + }, + ) + + def mark_submitted(self, operation_id: str, txid: str) -> AssetTransferIntent: + now = datetime.now(UTC) + return self._update( + operation_id, + txid, + { + "$set": { + "status": "submitted", + "submitted_at": now, + "last_error": None, + "updated": now, + } + }, + ) + + def mark_confirmed( + self, + operation_id: str, + txid: str, + confirmed_round: int, + ) -> AssetTransferIntent: + now = datetime.now(UTC) + return self._update( + operation_id, + txid, + { + "$set": { + "status": "confirmed", + "confirmed_round": confirmed_round, + "confirmed_at": now, + "last_error": None, + "updated": now, + } + }, + ) + + def record_error( + self, + operation_id: str, + txid: str, + error: str, + ) -> AssetTransferIntent: + return self._update( + operation_id, + txid, + { + "$set": { + "last_error": error[:500], + "updated": datetime.now(UTC), + } + }, + ) + + def _update( + self, + operation_id: str, + txid: str, + update: dict[str, Any], + ) -> AssetTransferIntent: + document = self.collection.find_one_and_update( + {"id": operation_id, "txid": txid}, + update, + return_document=ReturnDocument.AFTER, + ) + intent = self._from_document(document) + if intent is None: + raise TransferIntentPersistenceError(f"transfer {operation_id!r} changed while updating transaction {txid}") + return intent + + @staticmethod + def _from_document( + document: dict[str, Any] | None, + ) -> AssetTransferIntent | None: + if document is None: + return None + payload = dict(document) + payload.pop("_id", None) + return AssetTransferIntent.from_dict(payload) diff --git a/flex/db/cometa_database.py b/flex/db/cometa_database.py index a0b238ea..a8299a81 100644 --- a/flex/db/cometa_database.py +++ b/flex/db/cometa_database.py @@ -1,11 +1,13 @@ from pymongo.database import Database as MongoDatabase from flex.db.classes.database import EntitiesDatabase +from flex.db.model.airdrop import AirdropManifest from flex.db.model.blockchain import LpToken, Asset, PoolTransaction, SyncState, SyncBlock from flex.db.model.liquidity_pools import LpState, LpTransaction from flex.db.model.pool_states import UserState, PoolState from flex.db.model.pools import StakingPool, FarmingPool from flex.db.model.priced import AirdropReward, AssetPrice +from flex.db.model.transfers import AssetTransferIntent class CometaDatabase(EntitiesDatabase): @@ -31,3 +33,5 @@ def __init__(self, mongodb_database: MongoDatabase): self.sync_blocks = self.create_collection_manager_for_type(SyncBlock) self.airdrop_rewards = self.create_collection_manager_for_type(AirdropReward) + self.airdrop_manifests = self.create_collection_manager_for_type(AirdropManifest) + self.asset_transfer_intents = self.create_collection_manager_for_type(AssetTransferIntent) diff --git a/flex/db/indexes.py b/flex/db/indexes.py index 40a5b36b..629d2d45 100644 --- a/flex/db/indexes.py +++ b/flex/db/indexes.py @@ -8,10 +8,12 @@ logger = logging.getLogger(__name__) -_UNIQUE_ID_COLLECTIONS = ( - "asset_prices", - "pool_transactions", - "lp_transactions", +_UNIQUE_ID_POLICIES = ( + ("airdrop_manifests", False), + ("asset_prices", True), + ("asset_transfer_intents", False), + ("pool_transactions", False), + ("lp_transactions", False), ) _HOT_INDEXES = ( @@ -69,20 +71,55 @@ def deduplicate_and_create_unique_id_index( return removed +def create_unique_id_index_fail_closed( + collection: Collection[dict[str, Any]], + *, + collection_name: str, +) -> None: + """Preserve conflicting immutable records for explicit reconciliation.""" + + duplicate_groups: Sequence[dict[str, Any]] = list( + collection.aggregate( + _duplicate_id_pipeline(), + allowDiskUse=True, + ) + ) + if duplicate_groups: + raise RuntimeError( + f"{collection_name} contains {len(duplicate_groups)} duplicate immutable ID group(s); " + "reconcile or quarantine them before startup" + ) + collection.create_index("id", unique=True, name="id_unique") + + def ensure_database_indexes(database: CometaDatabase) -> dict[str, int]: """Install correctness-critical unique indexes and hot query indexes.""" removed_by_collection: dict[str, int] = {} - for collection_name in _UNIQUE_ID_COLLECTIONS: + for collection_name, can_deduplicate in _UNIQUE_ID_POLICIES: manager = getattr(database, collection_name) - removed_by_collection[collection_name] = deduplicate_and_create_unique_id_index( - manager.mongodb_collection, - collection_name=collection_name, - ) + if can_deduplicate: + removed_by_collection[collection_name] = deduplicate_and_create_unique_id_index( + manager.mongodb_collection, + collection_name=collection_name, + ) + else: + create_unique_id_index_fail_closed( + manager.mongodb_collection, + collection_name=collection_name, + ) + removed_by_collection[collection_name] = 0 for manager_name, field_name, index_name in _HOT_INDEXES: manager = getattr(database, manager_name) manager.mongodb_collection.create_index(field_name, name=index_name) + database.airdrop_rewards.mongodb_collection.create_index( + "operation_id", + unique=True, + name="operation_id_unique", + partialFilterExpression={"operation_id": {"$type": "string"}}, + ) + logger.info("Ensured hot query indexes for Flex collections") return removed_by_collection diff --git a/flex/db/model/airdrop.py b/flex/db/model/airdrop.py new file mode 100644 index 00000000..332025ad --- /dev/null +++ b/flex/db/model/airdrop.py @@ -0,0 +1,34 @@ +"""Persistence model for immutable airdrop batch manifests.""" + +from dataclasses import dataclass, field +from datetime import UTC, datetime + +from dataclasses_json import dataclass_json + +from flex.db.classes.base_entity import BaseEntity + + +def _utc_now() -> datetime: + return datetime.now(UTC) + + +@dataclass_json +@dataclass +class AirdropManifest(BaseEntity["AirdropManifest"]): + id: str + asset_id: str + total_amount_micros: str + recipient_count: int + manifest_hash: str + status: str = "prepared" + + created: datetime = field(default_factory=_utc_now) + updated: datetime = field(default_factory=_utc_now) + + def __post_init__(self) -> None: + self.asset_id = str(self.asset_id) + self.total_amount_micros = str(self.total_amount_micros) + + @property + def asset_id_int(self) -> int: + return int(self.asset_id) diff --git a/flex/db/model/priced.py b/flex/db/model/priced.py index 34d1f14a..318f9d0a 100644 --- a/flex/db/model/priced.py +++ b/flex/db/model/priced.py @@ -47,14 +47,28 @@ class UserCost: class AirdropReward(BaseEntity["AirdropReward"]): airdrop_id: str address: str - asa_id: int - amount_micros: int + asa_id: str + amount_micros: str txid: str + operation_id: str | None = None + confirmed_round: int | None = None id: str = field(default_factory=get_uuid) created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) + def __post_init__(self) -> None: + self.asa_id = str(self.asa_id) + self.amount_micros = str(self.amount_micros) + + @property + def asa_id_int(self) -> int: + return int(self.asa_id) + + @property + def amount_micros_int(self) -> int: + return int(self.amount_micros) + @dataclass_json @dataclass diff --git a/flex/db/model/transfers.py b/flex/db/model/transfers.py new file mode 100644 index 00000000..7f0e9989 --- /dev/null +++ b/flex/db/model/transfers.py @@ -0,0 +1,51 @@ +"""Persistence model for crash-safe outbound Algorand transfers.""" + +from dataclasses import dataclass, field +from datetime import UTC, datetime + +from dataclasses_json import dataclass_json + +from flex.db.classes.base_entity import BaseEntity + + +def _utc_now() -> datetime: + return datetime.now(UTC) + + +@dataclass_json +@dataclass +class AssetTransferIntent(BaseEntity["AssetTransferIntent"]): + """A signed transaction persisted before its first broadcast attempt.""" + + id: str + receiver: str + asset_id: str + amount_micros: str + note: str | None + signed_transaction: str + txid: str + first_valid_round: int + last_valid_round: int + status: str + + attempt_count: int = 0 + confirmed_round: int | None = None + last_error: str | None = None + submitted_at: datetime | None = None + confirmed_at: datetime | None = None + created: datetime = field(default_factory=_utc_now) + updated: datetime = field(default_factory=_utc_now) + + def __post_init__(self) -> None: + # Legacy BSON documents may contain int64 values; new writes stay + # string-backed so the full Algorand uint64 domain is lossless. + self.asset_id = str(self.asset_id) + self.amount_micros = str(self.amount_micros) + + @property + def amount_micros_int(self) -> int: + return int(self.amount_micros) + + @property + def asset_id_int(self) -> int: + return int(self.asset_id) diff --git a/flex/domain/allocation.py b/flex/domain/allocation.py new file mode 100644 index 00000000..d935ed8b --- /dev/null +++ b/flex/domain/allocation.py @@ -0,0 +1,80 @@ +"""Exact, deterministic allocation of integer asset base units.""" + +from collections.abc import Mapping +from decimal import Decimal, InvalidOperation +from fractions import Fraction + +type ShareInput = Decimal | int | float | str + +MAX_SHARE_EXPONENT = 300 +MAX_SHARE_SIGNIFICANT_DIGITS = 50 + + +class AllocationError(ValueError): + """Raised when an allocation cannot preserve its financial invariants.""" + + +def _positive_decimal(value: ShareInput, *, field: str) -> Decimal: + if isinstance(value, bool): + raise AllocationError(f"{field} must be numeric, not bool") + try: + parsed = value if isinstance(value, Decimal) else Decimal(str(value)) + except (InvalidOperation, ValueError) as exc: + raise AllocationError(f"{field} is not a valid decimal") from exc + + if not parsed.is_finite(): + raise AllocationError(f"{field} must be finite") + if parsed <= 0: + raise AllocationError(f"{field} must be positive") + if abs(parsed.adjusted()) > MAX_SHARE_EXPONENT: + raise AllocationError(f"{field} exponent is outside the supported range") + if len(parsed.as_tuple().digits) > MAX_SHARE_SIGNIFICANT_DIGITS: + raise AllocationError(f"{field} has too many significant digits") + return parsed + + +def allocate_proportionally( + total_micros: int, + shares: Mapping[str, ShareInput], +) -> dict[str, int]: + """Allocate an integer budget exactly using the largest-remainder method. + + Fractional remainders are resolved by recipient ID, making the result + reproducible regardless of mapping insertion order. + """ + + if isinstance(total_micros, bool) or not isinstance(total_micros, int): + raise AllocationError("total_micros must be an integer") + if total_micros <= 0: + raise AllocationError("total_micros must be positive") + if not shares: + raise AllocationError("shares must not be empty") + + weights: dict[str, Fraction] = {} + for recipient, share in shares.items(): + if not isinstance(recipient, str) or not recipient.strip(): + raise AllocationError("recipient IDs must be non-empty strings") + parsed = _positive_decimal(share, field=f"shares[{recipient!r}]") + weights[recipient] = Fraction(parsed) + + total_weight = sum(weights.values(), start=Fraction()) + allocations: dict[str, int] = {} + remainders: list[tuple[Fraction, str]] = [] + + for recipient in sorted(weights): + exact_amount = Fraction(total_micros) * weights[recipient] / total_weight + floor_amount = exact_amount.numerator // exact_amount.denominator + allocations[recipient] = floor_amount + remainders.append((exact_amount - floor_amount, recipient)) + + undistributed = total_micros - sum(allocations.values()) + ranked_remainders = sorted( + remainders, + key=lambda item: (-item[0], item[1]), + ) + for _, recipient in ranked_remainders[:undistributed]: + allocations[recipient] += 1 + + if sum(allocations.values()) != total_micros: + raise AssertionError("allocation must preserve the integer budget") + return allocations diff --git a/flex/tools/airdrop.py b/flex/tools/airdrop.py index cd740b0c..1d6534ea 100644 --- a/flex/tools/airdrop.py +++ b/flex/tools/airdrop.py @@ -1,91 +1,505 @@ +"""Operational airdrop tool with exact allocation and replay-safe transfers.""" + import json import logging -import random +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from hashlib import sha256 +from pathlib import Path + +from algosdk import encoding +from pymongo import ReturnDocument from flex import db +from flex.application.asset_transfers import ( + MAX_ALGORAND_UINT, + AssetTransferReceipt, + AssetTransferRequest, + AssetTransferService, +) +from flex.application.transfer_runtime import get_asset_transfer_service from flex.blockchain.base import cometa_public_key -from flex.blockchain.info import get_current_round, is_opted_in +from flex.blockchain.info import get_current_round +from flex.db.model.airdrop import AirdropManifest from flex.db.model.blockchain import AssetInfo from flex.db.model.priced import AirdropReward -from flex.txns import TxInfo, send_asset_micros_with_wait, send_asset_micros +from flex.domain.allocation import ShareInput, allocate_proportionally +from flex.txns import TxInfo logger = logging.getLogger(__name__) +class AirdropError(RuntimeError): + """Base class for invalid or incomplete airdrop execution.""" + + +class AirdropConflictError(AirdropError): + """Raised when an airdrop ID is reused with different transfer details.""" + + +@dataclass(frozen=True, slots=True) +class AirdropFailure: + address: str + error_type: str + txid: str | None = None + + +class AirdropIncompleteError(AirdropError): + """Raised after all recipients were attempted and at least one is unresolved.""" + + def __init__( + self, + failures: Sequence[AirdropFailure], + confirmed_transactions: Sequence[TxInfo], + ) -> None: + super().__init__(f"airdrop incomplete: {len(failures)} recipient(s) unresolved; rerun with the same airdrop_id") + self.failures = tuple(failures) + self.confirmed_transactions = tuple(confirmed_transactions) + + +def _operation_id(airdrop_id: str, address: str) -> str: + return f"airdrop:{airdrop_id}:{address}" + + +def _select_note(operation_id: str, notes: Sequence[str]) -> str: + digest = sha256(operation_id.encode()).digest() + index = int.from_bytes(digest[:8], "big") % len(notes) + return notes[index] + + +def _validate_airdrop( + *, + airdrop_id: str, + addresses: Sequence[str], + notes: Sequence[str], + amounts: Mapping[str, int], +) -> None: + if not isinstance(airdrop_id, str) or not airdrop_id.strip(): + raise AirdropError("airdrop_id must be a non-empty string") + if isinstance(notes, (str, bytes)) or not notes: + raise AirdropError("notes must not be empty") + if any(not isinstance(note, str) or len(note.encode()) > 1_000 for note in notes): + raise AirdropError("each note must be a string within Algorand's 1000-byte limit") + invalid_addresses = [address for address in addresses if not encoding.is_valid_address(address)] + if invalid_addresses: + raise AirdropError(f"airdrop contains {len(invalid_addresses)} invalid Algorand address(es)") + zero_allocations = [address for address, amount in amounts.items() if amount == 0] + if zero_allocations: + raise AirdropError( + f"airdrop budget is too small: {len(zero_allocations)} recipient(s) would receive zero base units" + ) + + +def _assert_reward_matches( + reward: AirdropReward, + *, + asset_id: int, + amount_micros: int, +) -> None: + if (reward.asa_id_int, reward.amount_micros_int) != (asset_id, amount_micros): + raise AirdropConflictError( + f"existing reward {reward.airdrop_id!r}/{reward.address} has different transfer details" + ) + + +def _get_existing_reward( + *, + airdrop_id: str, + address: str, +) -> AirdropReward | None: + rewards = db.airdrop_rewards.get_many( + address=address, + airdrop_id=airdrop_id, + ) + if len(rewards) > 1: + raise AirdropError( + f"multiple legacy rewards exist for {airdrop_id!r}/{address}; manual reconciliation required" + ) + return rewards[0] if rewards else None + + +def _manifest_hash( + *, + airdrop_id: str, + asset_id: int, + total_amount_micros: int, + requests: Mapping[str, AssetTransferRequest], +) -> str: + canonical = { + "airdrop_id": airdrop_id, + "asset_id": asset_id, + "total_amount_micros": str(total_amount_micros), + "recipients": [ + [ + address, + str(requests[address].amount_micros), + requests[address].note, + ] + for address in sorted(requests) + ], + } + encoded = json.dumps( + canonical, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode() + return sha256(encoded).hexdigest() + + +def _reserve_manifest( + manifest: AirdropManifest, +) -> AirdropManifest: + collection = db.airdrop_manifests.mongodb_collection + collection.create_index("id", unique=True, name="id_unique") + document = collection.find_one_and_update( + {"id": manifest.id}, + {"$setOnInsert": manifest.to_dict()}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + if document is None: + raise AirdropError(f"failed to reserve airdrop manifest {manifest.id!r}") + payload = dict(document) + payload.pop("_id", None) + persisted = AirdropManifest.from_dict(payload) + immutable_fields = ( + persisted.asset_id_int, + persisted.total_amount_micros, + persisted.recipient_count, + persisted.manifest_hash, + ) + requested_fields = ( + manifest.asset_id_int, + manifest.total_amount_micros, + manifest.recipient_count, + manifest.manifest_hash, + ) + if immutable_fields != requested_fields: + raise AirdropConflictError(f"airdrop_id {manifest.id!r} is already reserved for a different immutable manifest") + return persisted + + +def _has_manifest(airdrop_id: str) -> bool: + return ( + db.airdrop_manifests.mongodb_collection.find_one( + {"id": airdrop_id}, + projection={"_id": 1}, + ) + is not None + ) + + +def _mark_manifest_status(airdrop_id: str, status: str) -> None: + db.airdrop_manifests.mongodb_collection.update_one( + {"id": airdrop_id}, + { + "$set": { + "status": status, + "updated": datetime.now(UTC), + } + }, + ) + + +def _reconcile_existing_reward( + reward: AirdropReward, + *, + operation_id: str, + transfer_service: AssetTransferService, +) -> AirdropReward: + if reward.operation_id not in (None, operation_id): + raise AirdropConflictError(f"reward {reward.airdrop_id!r}/{reward.address} belongs to another operation") + + confirmed_round = reward.confirmed_round + if reward.operation_id is None or confirmed_round is None: + observed = transfer_service.lookup_confirmed_transfer(reward.txid) + if observed is None: + raise AirdropError( + f"legacy reward {reward.airdrop_id!r}/{reward.address} is not confirmed; manual reconciliation required" + ) + expected = ( + cometa_public_key, + reward.address, + reward.asa_id_int, + reward.amount_micros_int, + ) + actual = ( + observed.sender, + observed.receiver, + observed.asset_id, + observed.amount_micros, + ) + if actual != expected: + raise AirdropConflictError( + f"legacy transaction {reward.txid} does not match its stored sender/receiver/asset/amount" + ) + confirmed_round = observed.confirmed_round + + collection = db.airdrop_rewards.mongodb_collection + document = collection.find_one_and_update( + { + "airdrop_id": reward.airdrop_id, + "address": reward.address, + "txid": reward.txid, + }, + { + "$set": { + "operation_id": operation_id, + "confirmed_round": confirmed_round, + "updated": datetime.now(UTC), + } + }, + return_document=ReturnDocument.AFTER, + ) + if document is None: + raise AirdropError(f"failed to backfill legacy reward {reward.airdrop_id!r}/{reward.address}") + payload = dict(document) + payload.pop("_id", None) + return AirdropReward.from_dict(payload) + + +def _persist_reward( + reward: AirdropReward, +) -> AirdropReward: + if reward.operation_id is None: + raise AirdropError("operation_id is required for a persisted reward") + collection = db.airdrop_rewards.mongodb_collection + document = collection.find_one_and_update( + {"operation_id": reward.operation_id}, + {"$setOnInsert": reward.to_dict()}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + if document is None: + raise AirdropError(f"failed to persist reward {reward.operation_id!r}") + payload = dict(document) + payload.pop("_id", None) + persisted = AirdropReward.from_dict(payload) + _assert_reward_matches( + persisted, + asset_id=reward.asa_id_int, + amount_micros=reward.amount_micros_int, + ) + if persisted.txid != reward.txid: + raise AirdropConflictError(f"reward {reward.operation_id!r} points to another transaction") + if persisted.confirmed_round != reward.confirmed_round: + raise AirdropConflictError(f"reward {reward.operation_id!r} has a different confirmation round") + return persisted + + +def _to_tx_info( + *, + receipt: AssetTransferReceipt, + asset_id: int, + amount_micros: int, + address: str, + note: str, +) -> TxInfo: + return TxInfo( + id=receipt.txid, + amount=amount_micros, + asa_id=asset_id, + receiver=address, + note=note, + sender=cometa_public_key, + confirmed_round=receipt.confirmed_round, + ) + + +def _write_manifest( + directory: Path, + *, + asset_id: int, + current_round: int, + amounts: Mapping[str, int], + transactions: Sequence[TxInfo], +) -> None: + directory.mkdir(parents=True, exist_ok=True) + amounts_path = directory / f"airdrop_amounts_{asset_id}_{current_round}.json" + transactions_path = directory / f"airdrop_txns_{asset_id}_{current_round}.json" + amounts_path.write_text( + json.dumps(amounts, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + transactions_path.write_text( + json.dumps([transaction_info.to_dict() for transaction_info in transactions], indent=2) + "\n", + encoding="utf-8", + ) + + async def send_airdrop( - asset_info: AssetInfo, - total_amount_micros: int, - address_shares: dict[str, float], - notes: list[str], - airdrop_id: str + asset_info: AssetInfo, + total_amount_micros: int, + address_shares: Mapping[str, ShareInput], + notes: Sequence[str], + airdrop_id: str, + *, + manifest_directory: Path | None = None, + transfer_service: AssetTransferService | None = None, ) -> list[TxInfo]: + """Execute or resume an airdrop without exceeding its integer budget.""" + current_round = await get_current_round() - logger.info(f'Sending airdrop for {asset_info.name}! Round: {current_round}. Cometa address: {cometa_public_key}') - - # logger.info('Checking opted-in addresses...') - # opted_in_shares = {} - # total_shares = 0 - # it_num = 0 - # for address, share in address_shares.items(): - # it_num += 1 - # if is_opted_in(address, asset_info.id): - # opted_in_shares[address] = share - # total_shares += share - # if it_num % 10 == 0: - # logger.info(f'{len(opted_in_shares)}/{it_num}') - - opted_in_shares = address_shares - total_shares = sum(opted_in_shares.values()) - - amount_per_share = asset_info.micros_to_amount(int(total_amount_micros / total_shares + 1)) - logger.info(f'Total shares: {total_shares}. Total amount: {asset_info.micros_to_amount(total_amount_micros)}. Amount per share: {amount_per_share}') - - opted_in_amounts = {} - for address, share in opted_in_shares.items(): - amount_micros = int(share * total_amount_micros / total_shares + 1) - opted_in_amounts[address] = amount_micros - - with open(f'airdrop_amounts_{asset_info.name.lower()}_{current_round}.json', 'w') as f: - json.dump(opted_in_amounts, f, indent=4) - - sent_txns: list[TxInfo] = [] - total_sent_micros = 0 - - it_num = 0 - for address, amount_micros in opted_in_amounts.items(): - try: - it_num += 1 - logger.info(f'{it_num}/{len(opted_in_amounts)}') + if total_amount_micros > MAX_ALGORAND_UINT: + raise AirdropError("total_amount_micros exceeds Algorand's uint64 range") + amounts = allocate_proportionally(total_amount_micros, address_shares) + _validate_airdrop( + airdrop_id=airdrop_id, + addresses=list(address_shares), + notes=notes, + amounts=amounts, + ) + service = transfer_service or get_asset_transfer_service() + requests = { + address: AssetTransferRequest( + operation_id=_operation_id(airdrop_id, address), + receiver=address, + asset_id=asset_info.id, + amount_micros=amount_micros, + note=_select_note(_operation_id(airdrop_id, address), notes), + ) + for address, amount_micros in amounts.items() + } - already_sent = db.airdrop_rewards.get_one(address=address, airdrop_id=airdrop_id) - if already_sent is not None: - logger.info(f'Skipping {address}: already sent {asset_info.micros_to_amount(already_sent.amount_micros)} {asset_info.name} tokens with txid {already_sent.txid}!') - continue + campaign_rewards = db.airdrop_rewards.get_many(airdrop_id=airdrop_id) + if campaign_rewards and not _has_manifest(airdrop_id): + raise AirdropError( + f"legacy airdrop {airdrop_id!r} has rewards but no immutable manifest; reconcile and migrate it explicitly" + ) - note = random.choice(notes) - txid = send_asset_micros(asset_info, address, amount_micros, note) - db.airdrop_rewards.create(AirdropReward( + # Configuration conflicts must abort the entire batch before any new + # transaction is broadcast. + for address, request in requests.items(): + service.validate(request) + existing_reward = _get_existing_reward( + airdrop_id=airdrop_id, + address=address, + ) + if existing_reward is not None: + _assert_reward_matches( + existing_reward, + asset_id=asset_info.id, + amount_micros=amounts[address], + ) + _reconcile_existing_reward( + existing_reward, + operation_id=request.operation_id, + transfer_service=service, + ) + + _reserve_manifest( + AirdropManifest( + id=airdrop_id, + asset_id=str(asset_info.id), + total_amount_micros=str(total_amount_micros), + recipient_count=len(requests), + manifest_hash=_manifest_hash( + airdrop_id=airdrop_id, + asset_id=asset_info.id, + total_amount_micros=total_amount_micros, + requests=requests, + ), + ) + ) + + logger.info( + "Executing airdrop %s for asset %s at round %s: %s base units across %s recipients", + airdrop_id, + asset_info.id, + current_round, + total_amount_micros, + len(amounts), + ) + + confirmed_transactions: list[TxInfo] = [] + failures: list[AirdropFailure] = [] + + for position, (address, amount_micros) in enumerate(amounts.items(), start=1): + request = requests[address] + try: + existing_reward = _get_existing_reward( airdrop_id=airdrop_id, address=address, - asa_id=asset_info.id, - amount_micros=amount_micros, - txid=txid - )) - sent_txns.append(TxInfo( - id=txid, - amount=amount_micros, - asa_id=asset_info.id, - receiver=address, - note=note, - sender=cometa_public_key - )) - total_sent_micros += amount_micros - except Exception as e: - logger.error(f'Error while sending airdrop to {address}: {e}', exc_info=True) - - logger.info(f'\nSent {asset_info.micros_to_amount(total_sent_micros)} {asset_info.name} tokens to {len(sent_txns)} addresses!') - - with open(f'airdrop_txns_{asset_info.name.lower()}_{current_round}.json', 'w') as f: - json.dump([tx.to_dict() for tx in sent_txns], f, indent=4) - - return sent_txns + ) + if existing_reward is not None: + _assert_reward_matches( + existing_reward, + asset_id=asset_info.id, + amount_micros=amount_micros, + ) + existing_reward = _reconcile_existing_reward( + existing_reward, + operation_id=request.operation_id, + transfer_service=service, + ) + logger.info( + "Airdrop %s recipient %s/%s already confirmed as %s", + airdrop_id, + position, + len(amounts), + existing_reward.txid, + ) + continue + + receipt = service.execute( + request, + ) + _persist_reward( + AirdropReward( + airdrop_id=airdrop_id, + address=address, + asa_id=str(asset_info.id), + amount_micros=str(amount_micros), + txid=receipt.txid, + operation_id=request.operation_id, + confirmed_round=receipt.confirmed_round, + ) + ) + if not receipt.already_confirmed: + confirmed_transactions.append( + _to_tx_info( + receipt=receipt, + asset_id=asset_info.id, + amount_micros=amount_micros, + address=address, + note=request.note or "", + ) + ) + except Exception as exc: + failures.append( + AirdropFailure( + address=address, + error_type=type(exc).__name__, + txid=getattr(exc, "txid", None), + ) + ) + logger.exception( + "Airdrop %s recipient %s/%s is unresolved", + airdrop_id, + position, + len(amounts), + ) + + if manifest_directory is not None: + _write_manifest( + manifest_directory, + asset_id=asset_info.id, + current_round=current_round, + amounts=amounts, + transactions=confirmed_transactions, + ) + + if failures: + _mark_manifest_status(airdrop_id, "partial") + raise AirdropIncompleteError(failures, confirmed_transactions) + + _mark_manifest_status(airdrop_id, "complete") + logger.info( + "Airdrop %s confirmed %s new transaction(s); allocated total remains exactly %s base units", + airdrop_id, + len(confirmed_transactions), + sum(amounts.values()), + ) + return confirmed_transactions diff --git a/pyproject.toml b/pyproject.toml index 8cab0df7..c61bb702 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,8 @@ quote-style = "double" files = [ "core/circuit_breaker.py", "flex/blockchain/contract_state.py", + "flex/application/asset_transfers.py", + "flex/domain/allocation.py", "flex/domain/pricing.py", "flex/domain/transactions.py", "flex/providers/pact.py", diff --git a/tests/unit/test_airdrop.py b/tests/unit/test_airdrop.py new file mode 100644 index 00000000..32cfa659 --- /dev/null +++ b/tests/unit/test_airdrop.py @@ -0,0 +1,455 @@ +import asyncio +from types import SimpleNamespace + +import pytest +from algosdk import account + +from flex.application.asset_transfers import ( + AssetTransferPendingError, + AssetTransferReceipt, + AssetTransferRequest, + ConfirmedAssetTransfer, +) +from flex.db.model.airdrop import AirdropManifest +from flex.db.model.blockchain import AssetInfo +from flex.db.model.priced import AirdropReward +from flex.tools import airdrop + + +class FakeRewardCollection: + def __init__(self, rewards: list[AirdropReward] | None = None) -> None: + self.documents = [reward.to_dict() for reward in rewards or []] + + def find_one_and_update(self, query, update, **kwargs): + document = next( + ( + document + for document in self.documents + if all(document.get(field) == value for field, value in query.items()) + ), + None, + ) + if document is None and "$setOnInsert" in update: + document = dict(update["$setOnInsert"]) + self.documents.append(document) + if document is not None and "$set" in update: + document.update(update["$set"]) + return dict(document) if document is not None else None + + +class FakeRewardManager: + def __init__(self, rewards: list[AirdropReward] | None = None) -> None: + self.mongodb_collection = FakeRewardCollection(rewards) + + def get_many(self, **query): + matches = [] + for document in self.mongodb_collection.documents: + reward = AirdropReward.from_dict(document) + if all(getattr(reward, field) == value for field, value in query.items()): + matches.append(reward) + return matches + + +class FakeManifestCollection: + def __init__(self) -> None: + self.documents: dict[str, dict] = {} + + def create_index(self, *args, **kwargs) -> None: + return None + + def find_one(self, query, projection=None): + document = self.documents.get(query["id"]) + return dict(document) if document is not None else None + + def find_one_and_update(self, query, update, **kwargs): + manifest_id = query["id"] + self.documents.setdefault(manifest_id, dict(update["$setOnInsert"])) + return dict(self.documents[manifest_id]) + + def update_one(self, query, update) -> None: + self.documents[query["id"]].update(update["$set"]) + + +class FakeTransferService: + def __init__( + self, + *, + fail_address: str | None = None, + legacy_transfer: ConfirmedAssetTransfer | None = None, + ) -> None: + self.validated = [] + self.executed = [] + self.fail_address = fail_address + self.legacy_transfer = legacy_transfer + + def validate(self, request) -> None: + self.validated.append(request) + + def execute(self, request) -> AssetTransferReceipt: + self.executed.append(request) + if request.receiver == self.fail_address: + raise AssetTransferPendingError(request.operation_id, f"tx-{request.receiver}") + return AssetTransferReceipt( + operation_id=request.operation_id, + txid=f"tx-{request.receiver}", + confirmed_round=123, + already_confirmed=False, + ) + + def lookup_confirmed_transfer(self, txid: str) -> ConfirmedAssetTransfer | None: + return self.legacy_transfer + + +def _addresses(count: int) -> list[str]: + return [account.generate_account()[1] for _ in range(count)] + + +def _asset() -> AssetInfo: + return AssetInfo( + name="Test Asset", + decimals=6, + unit_name="TEST", + id=42, + ) + + +def _install_fake_db( + monkeypatch, + rewards: list[AirdropReward] | None = None, +) -> tuple[FakeRewardManager, FakeManifestCollection]: + reward_manager = FakeRewardManager(rewards) + manifest_collection = FakeManifestCollection() + monkeypatch.setattr( + airdrop, + "db", + SimpleNamespace( + airdrop_rewards=reward_manager, + airdrop_manifests=SimpleNamespace( + mongodb_collection=manifest_collection, + ), + ), + ) + + async def current_round() -> int: + return 100 + + monkeypatch.setattr(airdrop, "get_current_round", current_round) + return reward_manager, manifest_collection + + +def _seed_manifest( + manifests: FakeManifestCollection, + *, + airdrop_id: str, + amounts: dict[str, int], + note: str = "hello", +) -> None: + requests = { + address: AssetTransferRequest( + operation_id=f"airdrop:{airdrop_id}:{address}", + receiver=address, + asset_id=42, + amount_micros=amount_micros, + note=note, + ) + for address, amount_micros in amounts.items() + } + total_amount_micros = sum(amounts.values()) + manifests.documents[airdrop_id] = AirdropManifest( + id=airdrop_id, + asset_id=42, + total_amount_micros=str(total_amount_micros), + recipient_count=len(requests), + manifest_hash=airdrop._manifest_hash( + airdrop_id=airdrop_id, + asset_id=42, + total_amount_micros=total_amount_micros, + requests=requests, + ), + ).to_dict() + + +def test_airdrop_allocates_the_exact_budget_and_persists_every_receipt(monkeypatch) -> None: + rewards, manifests = _install_fake_db(monkeypatch) + addresses = _addresses(3) + service = FakeTransferService() + + transactions = asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares=dict.fromkeys(reversed(addresses), 1), + notes=["one", "two"], + airdrop_id="summer-2026", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert len(service.validated) == 3 + assert len(service.executed) == 3 + assert sum(request.amount_micros for request in service.executed) == 10 + assert sorted(request.amount_micros for request in service.executed) == [3, 3, 4] + assert len(transactions) == 3 + assert len(rewards.mongodb_collection.documents) == 3 + assert manifests.documents["summer-2026"]["status"] == "complete" + + +def test_airdrop_configuration_conflict_aborts_before_any_broadcast(monkeypatch) -> None: + address, other_address = _addresses(2) + existing = AirdropReward( + airdrop_id="summer-2026", + address=other_address, + asa_id=42, + amount_micros=999, + txid="legacy-txid", + ) + _, manifests = _install_fake_db(monkeypatch, [existing]) + _seed_manifest( + manifests, + airdrop_id="summer-2026", + amounts={address: 5, other_address: 5}, + ) + service = FakeTransferService() + + with pytest.raises(airdrop.AirdropConflictError): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares={address: 1, other_address: 1}, + notes=["hello"], + airdrop_id="summer-2026", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert service.executed == [] + assert manifests.documents["summer-2026"]["status"] == "prepared" + + +def test_airdrop_reports_partial_failure_and_continues_safe_recipients(monkeypatch) -> None: + rewards, manifests = _install_fake_db(monkeypatch) + addresses = _addresses(3) + service = FakeTransferService(fail_address=addresses[1]) + + with pytest.raises(airdrop.AirdropIncompleteError) as error: + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=9, + address_shares=dict.fromkeys(addresses, 1), + notes=["hello"], + airdrop_id="summer-2026", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert len(service.executed) == 3 + assert error.value.failures == ( + airdrop.AirdropFailure( + address=addresses[1], + error_type="AssetTransferPendingError", + txid=f"tx-{addresses[1]}", + ), + ) + assert len(error.value.confirmed_transactions) == 2 + assert len(rewards.mongodb_collection.documents) == 2 + assert manifests.documents["summer-2026"]["status"] == "partial" + + +def test_airdrop_rejects_zero_allocations_before_execution(monkeypatch) -> None: + _install_fake_db(monkeypatch) + addresses = _addresses(2) + service = FakeTransferService() + + with pytest.raises(airdrop.AirdropError, match="would receive zero"): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=1, + address_shares={addresses[0]: 1, addresses[1]: 1}, + notes=["hello"], + airdrop_id="summer-2026", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert service.executed == [] + + +def test_airdrop_manifest_rejects_recipient_set_changes_after_partial_run(monkeypatch) -> None: + _, manifests = _install_fake_db(monkeypatch) + first, second, added = _addresses(3) + first_service = FakeTransferService(fail_address=second) + + with pytest.raises(airdrop.AirdropIncompleteError): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares={first: 1, second: 1}, + notes=["hello"], + airdrop_id="immutable-batch", + transfer_service=first_service, # type: ignore[arg-type] + ) + ) + + assert manifests.documents["immutable-batch"]["status"] == "partial" + changed_service = FakeTransferService() + with pytest.raises(airdrop.AirdropConflictError, match="immutable manifest"): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=15, + address_shares={first: 1, second: 1, added: 1}, + notes=["hello"], + airdrop_id="immutable-batch", + transfer_service=changed_service, # type: ignore[arg-type] + ) + ) + + assert changed_service.executed == [] + + +def test_legacy_unconfirmed_reward_is_not_treated_as_paid(monkeypatch) -> None: + address = _addresses(1)[0] + legacy_reward = AirdropReward( + airdrop_id="legacy", + address=address, + asa_id=42, + amount_micros=10, + txid="legacy-txid", + ) + _, manifests = _install_fake_db(monkeypatch, [legacy_reward]) + _seed_manifest( + manifests, + airdrop_id="legacy", + amounts={address: 10}, + ) + service = FakeTransferService() + + with pytest.raises(airdrop.AirdropError, match="not confirmed"): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares={address: 1}, + notes=["hello"], + airdrop_id="legacy", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert service.executed == [] + + +def test_legacy_confirmed_reward_is_backfilled_and_skipped(monkeypatch) -> None: + address = _addresses(1)[0] + legacy_reward = AirdropReward( + airdrop_id="legacy", + address=address, + asa_id=42, + amount_micros=10, + txid="legacy-txid", + ) + rewards, manifests = _install_fake_db(monkeypatch, [legacy_reward]) + _seed_manifest( + manifests, + airdrop_id="legacy", + amounts={address: 10}, + ) + service = FakeTransferService( + legacy_transfer=ConfirmedAssetTransfer( + txid="legacy-txid", + sender=airdrop.cometa_public_key, + receiver=address, + asset_id=42, + amount_micros=10, + confirmed_round=456, + ) + ) + + transactions = asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares={address: 1}, + notes=["hello"], + airdrop_id="legacy", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert transactions == [] + assert service.executed == [] + assert rewards.mongodb_collection.documents[0]["confirmed_round"] == 456 + assert rewards.mongodb_collection.documents[0]["operation_id"] == f"airdrop:legacy:{address}" + + +def test_legacy_confirmation_metadata_does_not_bypass_on_chain_field_validation(monkeypatch) -> None: + address, wrong_receiver = _addresses(2) + legacy_reward = AirdropReward( + airdrop_id="legacy", + address=address, + asa_id=42, + amount_micros=10, + txid="legacy-txid", + confirmed_round=456, + ) + _, manifests = _install_fake_db(monkeypatch, [legacy_reward]) + _seed_manifest( + manifests, + airdrop_id="legacy", + amounts={address: 10}, + ) + service = FakeTransferService( + legacy_transfer=ConfirmedAssetTransfer( + txid="legacy-txid", + sender=airdrop.cometa_public_key, + receiver=wrong_receiver, + asset_id=42, + amount_micros=10, + confirmed_round=456, + ) + ) + + with pytest.raises(airdrop.AirdropConflictError, match="does not match"): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares={address: 1}, + notes=["hello"], + airdrop_id="legacy", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert service.executed == [] + + +def test_legacy_campaign_cannot_expand_without_explicit_manifest_migration(monkeypatch) -> None: + paid, new_recipient = _addresses(2) + legacy_reward = AirdropReward( + airdrop_id="legacy", + address=paid, + asa_id=42, + amount_micros=10, + txid="legacy-txid", + ) + _install_fake_db(monkeypatch, [legacy_reward]) + service = FakeTransferService() + + with pytest.raises(airdrop.AirdropError, match="no immutable manifest"): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=20, + address_shares={paid: 1, new_recipient: 1}, + notes=["hello"], + airdrop_id="legacy", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert service.executed == [] diff --git a/tests/unit/test_algorand_asset_transfer_gateway.py b/tests/unit/test_algorand_asset_transfer_gateway.py new file mode 100644 index 00000000..a05451de --- /dev/null +++ b/tests/unit/test_algorand_asset_transfer_gateway.py @@ -0,0 +1,151 @@ +import pytest +from algosdk import account, encoding, transaction + +from flex.application.asset_transfers import ( + AssetTransferRequest, + InvalidAssetTransferError, +) +from flex.blockchain.asset_transfers import AlgorandAssetTransferGateway + + +class FakeAlgod: + def __init__(self) -> None: + self.sent: list[transaction.SignedTransaction] = [] + + def suggested_params(self) -> transaction.SuggestedParams: + return transaction.SuggestedParams( + fee=1_000, + first=100, + last=1_100, + gh=b"0" * 32, + flat_fee=True, + ) + + def send_transaction(self, signed: transaction.SignedTransaction) -> str: + self.sent.append(signed) + return signed.transaction.get_txid() + + def status(self) -> dict[str, int]: + return {"last-round": 100} + + +class FakeIndexer: + def __init__(self, *, sender: str, receiver: str) -> None: + self.sender = sender + self.receiver = receiver + + def transaction(self, txid: str) -> dict: + return { + "transaction": { + "confirmed-round": 123, + "sender": self.sender, + "asset-transfer-transaction": { + "receiver": self.receiver, + "asset-id": 42, + "amount": 1_000, + }, + } + } + + +def test_gateway_prepares_a_replay_safe_signed_transaction() -> None: + private_key, sender = account.generate_account() + _, receiver = account.generate_account() + algod = FakeAlgod() + gateway = AlgorandAssetTransferGateway( + algod=algod, # type: ignore[arg-type] + indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] + sender=sender, + private_key=private_key, + ) + request = AssetTransferRequest( + operation_id="airdrop:summer:recipient", + receiver=receiver, + asset_id=42, + amount_micros=1_000, + note="hello", + ) + + prepared = gateway.prepare(request) + decoded = encoding.msgpack_decode(prepared.signed_transaction) + + assert isinstance(decoded, transaction.SignedTransaction) + assert decoded.transaction.get_txid() == prepared.txid + assert decoded.transaction.receiver == receiver + assert decoded.transaction.index == 42 + assert decoded.transaction.amount == 1_000 + assert decoded.transaction.note == b"hello" + assert len(decoded.transaction.lease) == 32 + assert prepared.first_valid_round == 100 + assert prepared.last_valid_round == 1_100 + + returned_txid = gateway.broadcast(prepared, request) + + assert returned_txid == prepared.txid + assert len(algod.sent) == 1 + + observed = gateway.lookup_confirmed_transfer(prepared.txid) + + assert observed is not None + assert observed.sender == sender + assert observed.receiver == receiver + assert observed.asset_id == 42 + assert observed.amount_micros == 1_000 + assert observed.confirmed_round == 123 + + +def test_operation_id_deterministically_selects_the_lease() -> None: + private_key, sender = account.generate_account() + _, receiver = account.generate_account() + gateway = AlgorandAssetTransferGateway( + algod=FakeAlgod(), # type: ignore[arg-type] + indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] + sender=sender, + private_key=private_key, + ) + request = AssetTransferRequest( + operation_id="lottery:draw:42", + receiver=receiver, + asset_id=7, + amount_micros=1, + ) + + first = encoding.msgpack_decode(gateway.prepare(request).signed_transaction) + second = encoding.msgpack_decode(gateway.prepare(request).signed_transaction) + + assert first.transaction.lease == second.transaction.lease + assert first.transaction.get_txid() == second.transaction.get_txid() + + +def test_gateway_rejects_a_swapped_signed_payload_before_network_io() -> None: + private_key, sender = account.generate_account() + _, receiver = account.generate_account() + _, other_receiver = account.generate_account() + algod = FakeAlgod() + gateway = AlgorandAssetTransferGateway( + algod=algod, # type: ignore[arg-type] + indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] + sender=sender, + private_key=private_key, + ) + expected = AssetTransferRequest( + operation_id="airdrop:summer:recipient", + receiver=receiver, + asset_id=42, + amount_micros=1_000, + note="hello", + ) + swapped = gateway.prepare( + AssetTransferRequest( + operation_id=expected.operation_id, + receiver=other_receiver, + asset_id=42, + amount_micros=1_000, + note="hello", + ) + ) + + with pytest.raises(InvalidAssetTransferError, match="do not match"): + gateway.broadcast(swapped, expected) + + assert algod.sent == [] diff --git a/tests/unit/test_allocation.py b/tests/unit/test_allocation.py new file mode 100644 index 00000000..bbfcc2ac --- /dev/null +++ b/tests/unit/test_allocation.py @@ -0,0 +1,73 @@ +from decimal import Decimal + +import pytest + +from flex.domain.allocation import AllocationError, allocate_proportionally + + +def test_allocation_never_exceeds_the_integer_budget() -> None: + allocations = allocate_proportionally( + 2, + { + "ADDRESS-B": 1, + "ADDRESS-A": 1, + }, + ) + + assert allocations == {"ADDRESS-A": 1, "ADDRESS-B": 1} + assert sum(allocations.values()) == 2 + + +def test_allocation_uses_deterministic_largest_remainders() -> None: + allocations = allocate_proportionally( + 10, + { + "ADDRESS-C": Decimal("1"), + "ADDRESS-B": Decimal("1"), + "ADDRESS-A": Decimal("1"), + }, + ) + + assert allocations == { + "ADDRESS-A": 4, + "ADDRESS-B": 3, + "ADDRESS-C": 3, + } + + +def test_allocation_is_independent_of_input_order() -> None: + forwards = allocate_proportionally( + 100, + {"ADDRESS-A": 0.1, "ADDRESS-B": 0.2}, + ) + backwards = allocate_proportionally( + 100, + {"ADDRESS-B": 0.2, "ADDRESS-A": 0.1}, + ) + + assert forwards == backwards == {"ADDRESS-A": 33, "ADDRESS-B": 67} + + +@pytest.mark.parametrize("total_micros", [True, 0, -1, 1.5]) +def test_allocation_rejects_invalid_budgets(total_micros: object) -> None: + with pytest.raises(AllocationError): + allocate_proportionally(total_micros, {"ADDRESS": 1}) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + "shares", + [ + {}, + {"": 1}, + {"ADDRESS": True}, + {"ADDRESS": 0}, + {"ADDRESS": -1}, + {"ADDRESS": float("nan")}, + {"ADDRESS": float("inf")}, + {"ADDRESS": "not-a-number"}, + {"ADDRESS": Decimal("1e-301")}, + ], +) +def test_allocation_rejects_invalid_shares(shares: dict[str, object]) -> None: + with pytest.raises(AllocationError): + allocate_proportionally(100, shares) # type: ignore[arg-type] diff --git a/tests/unit/test_asset_transfer_repository.py b/tests/unit/test_asset_transfer_repository.py new file mode 100644 index 00000000..51fbdcf9 --- /dev/null +++ b/tests/unit/test_asset_transfer_repository.py @@ -0,0 +1,78 @@ +from datetime import UTC, datetime +from unittest.mock import Mock + +from pymongo import ReturnDocument + +from flex.db.asset_transfer_intents import MongoAssetTransferIntentRepository +from flex.db.model.transfers import AssetTransferIntent + + +def _intent() -> AssetTransferIntent: + timestamp = datetime(2026, 1, 1, tzinfo=UTC) + return AssetTransferIntent( + id="airdrop:summer:address", + receiver="address", + asset_id="42", + amount_micros="1000", + note="hello", + signed_transaction="signed", + txid="txid", + first_valid_round=100, + last_valid_round=1_100, + status="prepared", + created=timestamp, + updated=timestamp, + ) + + +def test_repository_installs_unique_and_reconciliation_indexes() -> None: + collection = Mock() + repository = MongoAssetTransferIntentRepository(collection) + + repository.ensure_indexes() + + collection.create_index.assert_any_call("id", unique=True, name="id_unique") + collection.create_index.assert_any_call( + [("status", 1), ("updated", 1)], + name="status_updated_idx", + ) + + +def test_reserve_uses_one_atomic_set_on_insert() -> None: + collection = Mock() + intent = _intent() + collection.find_one_and_update.return_value = intent.to_dict() + repository = MongoAssetTransferIntentRepository(collection) + + reserved = repository.reserve(intent) + + assert reserved == intent + collection.find_one_and_update.assert_called_once_with( + {"id": intent.id}, + {"$setOnInsert": intent.to_dict()}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + + +def test_confirmation_is_guarded_by_operation_and_transaction_id() -> None: + collection = Mock() + confirmed = _intent() + confirmed.status = "confirmed" + confirmed.confirmed_round = 777 + collection.find_one_and_update.return_value = confirmed.to_dict() + repository = MongoAssetTransferIntentRepository(collection) + + result = repository.mark_confirmed( + confirmed.id, + confirmed.txid, + confirmed.confirmed_round, + ) + + assert result.status == "confirmed" + assert result.confirmed_round == 777 + args, kwargs = collection.find_one_and_update.call_args + assert args[0] == {"id": confirmed.id, "txid": confirmed.txid} + assert args[1]["$set"]["status"] == "confirmed" + assert args[1]["$set"]["confirmed_round"] == 777 + assert kwargs == {"return_document": ReturnDocument.AFTER} diff --git a/tests/unit/test_asset_transfers.py b/tests/unit/test_asset_transfers.py new file mode 100644 index 00000000..748f8fbf --- /dev/null +++ b/tests/unit/test_asset_transfers.py @@ -0,0 +1,300 @@ +from dataclasses import replace +from datetime import UTC, datetime + +import pytest + +from flex.application.asset_transfers import ( + AssetTransferConflictError, + AssetTransferExpiredError, + AssetTransferPendingError, + AssetTransferRequest, + AssetTransferService, + ConfirmedAssetTransfer, + InvalidAssetTransferError, + PreparedAssetTransfer, + TransferStatus, +) +from flex.db.model.transfers import AssetTransferIntent + + +class InMemoryIntentRepository: + def __init__(self) -> None: + self.intents: dict[str, AssetTransferIntent] = {} + + def get(self, operation_id: str) -> AssetTransferIntent | None: + return self.intents.get(operation_id) + + def reserve(self, intent: AssetTransferIntent) -> AssetTransferIntent: + return self.intents.setdefault(intent.id, intent) + + def record_attempt(self, operation_id: str, txid: str) -> AssetTransferIntent: + intent = self._intent(operation_id, txid) + intent.attempt_count += 1 + return intent + + def mark_submitted(self, operation_id: str, txid: str) -> AssetTransferIntent: + intent = self._intent(operation_id, txid) + intent.status = TransferStatus.SUBMITTED + return intent + + def mark_confirmed( + self, + operation_id: str, + txid: str, + confirmed_round: int, + ) -> AssetTransferIntent: + intent = self._intent(operation_id, txid) + intent.status = TransferStatus.CONFIRMED + intent.confirmed_round = confirmed_round + return intent + + def record_error( + self, + operation_id: str, + txid: str, + error: str, + ) -> AssetTransferIntent: + intent = self._intent(operation_id, txid) + intent.last_error = error + return intent + + def _intent(self, operation_id: str, txid: str) -> AssetTransferIntent: + intent = self.intents[operation_id] + assert intent.txid == txid + return intent + + +class FakeTransferGateway: + def __init__(self) -> None: + self.prepare_count = 0 + self.broadcasted: list[str] = [] + self.current = 100 + self.confirmed_round = 101 + self.lookup_round: int | None = None + self.broadcast_error: Exception | None = None + self.wait_error: Exception | None = None + + def prepare(self, request: AssetTransferRequest) -> PreparedAssetTransfer: + self.prepare_count += 1 + return PreparedAssetTransfer( + signed_transaction=f"signed-{request.operation_id}", + txid=f"txid-{request.operation_id}", + first_valid_round=100, + last_valid_round=1_100, + ) + + def broadcast( + self, + prepared: PreparedAssetTransfer, + request: AssetTransferRequest, + ) -> str: + self.broadcasted.append(prepared.signed_transaction) + if self.broadcast_error is not None: + raise self.broadcast_error + return prepared.signed_transaction.replace("signed-", "txid-") + + def wait_for_confirmation(self, txid: str) -> int: + if self.wait_error is not None: + raise self.wait_error + return self.confirmed_round + + def lookup_confirmed_round(self, txid: str) -> int | None: + return self.lookup_round + + def lookup_confirmed_transfer(self, txid: str) -> ConfirmedAssetTransfer | None: + if self.lookup_round is None: + return None + return ConfirmedAssetTransfer( + txid=txid, + sender="SENDER", + receiver="ADDRESS", + asset_id=42, + amount_micros=1_000, + confirmed_round=self.lookup_round, + ) + + def current_round(self) -> int: + return self.current + + +def _request(**changes: object) -> AssetTransferRequest: + values = { + "operation_id": "airdrop:summer:ADDRESS", + "receiver": "ADDRESS", + "asset_id": 42, + "amount_micros": 1_000, + "note": "thank you", + } + values.update(changes) + return AssetTransferRequest(**values) # type: ignore[arg-type] + + +def _intent(**changes: object) -> AssetTransferIntent: + timestamp = datetime(2026, 1, 1, tzinfo=UTC) + values = { + "id": "airdrop:summer:ADDRESS", + "receiver": "ADDRESS", + "asset_id": "42", + "amount_micros": "1000", + "note": "thank you", + "signed_transaction": "persisted-signed-transaction", + "txid": "persisted-txid", + "first_valid_round": 1, + "last_valid_round": 1_000, + "status": TransferStatus.SUBMITTED, + "created": timestamp, + "updated": timestamp, + } + values.update(changes) + return AssetTransferIntent(**values) # type: ignore[arg-type] + + +def test_retry_rebroadcasts_the_same_persisted_signed_transaction() -> None: + repository = InMemoryIntentRepository() + gateway = FakeTransferGateway() + gateway.wait_error = TimeoutError("confirmation timeout") + service = AssetTransferService(repository, gateway) + + with pytest.raises(AssetTransferPendingError): + service.execute(_request()) + + persisted = repository.intents["airdrop:summer:ADDRESS"] + assert persisted.txid == "txid-airdrop:summer:ADDRESS" + assert persisted.signed_transaction == "signed-airdrop:summer:ADDRESS" + + gateway.wait_error = None + receipt = service.execute(_request()) + + assert receipt.txid == persisted.txid + assert gateway.prepare_count == 1 + assert gateway.broadcasted == [ + "signed-airdrop:summer:ADDRESS", + "signed-airdrop:summer:ADDRESS", + ] + assert persisted.attempt_count == 2 + + +def test_ambiguous_broadcast_is_resolved_by_confirmation() -> None: + repository = InMemoryIntentRepository() + gateway = FakeTransferGateway() + gateway.broadcast_error = TimeoutError("connection dropped after submit") + service = AssetTransferService(repository, gateway) + + receipt = service.execute(_request()) + + assert receipt.confirmed_round == 101 + assert repository.intents[receipt.operation_id].status == TransferStatus.CONFIRMED + + +def test_confirmed_retry_does_not_contact_the_gateway() -> None: + repository = InMemoryIntentRepository() + confirmed = _intent( + status=TransferStatus.CONFIRMED, + confirmed_round=500, + ) + repository.intents[confirmed.id] = confirmed + gateway = FakeTransferGateway() + service = AssetTransferService(repository, gateway) + + receipt = service.execute(_request()) + + assert receipt.already_confirmed is True + assert receipt.confirmed_round == 500 + assert gateway.prepare_count == 0 + assert gateway.broadcasted == [] + + +def test_idempotency_key_cannot_be_reused_for_another_amount() -> None: + repository = InMemoryIntentRepository() + existing = _intent() + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + service = AssetTransferService(repository, gateway) + + with pytest.raises(AssetTransferConflictError): + service.execute(_request(amount_micros=1_001)) + + assert gateway.broadcasted == [] + + +def test_expired_intent_reconciles_a_confirmed_transaction() -> None: + repository = InMemoryIntentRepository() + existing = _intent(last_valid_round=99) + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + gateway.current = 100 + gateway.lookup_round = 88 + service = AssetTransferService(repository, gateway) + + receipt = service.execute(_request()) + + assert receipt.already_confirmed is True + assert receipt.confirmed_round == 88 + assert gateway.broadcasted == [] + + +def test_expired_unconfirmed_intent_requires_manual_reconciliation() -> None: + repository = InMemoryIntentRepository() + existing = _intent(last_valid_round=99) + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + gateway.current = 100 + service = AssetTransferService(repository, gateway) + + with pytest.raises(AssetTransferExpiredError): + service.execute(_request()) + + assert gateway.prepare_count == 0 + assert gateway.broadcasted == [] + assert "manual reconciliation" in existing.last_error + + +def test_reconcile_marks_an_observed_transaction_confirmed() -> None: + repository = InMemoryIntentRepository() + existing = _intent() + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + gateway.lookup_round = 777 + service = AssetTransferService(repository, gateway) + + result = service.reconcile(existing.id) + + assert result.status == TransferStatus.CONFIRMED + assert result.confirmed_round == 777 + + +def test_full_algorand_uint64_values_are_persisted_without_bson_integers() -> None: + repository = InMemoryIntentRepository() + gateway = FakeTransferGateway() + service = AssetTransferService(repository, gateway) + + request = _request(asset_id=2**64 - 1, amount_micros=2**64 - 1) + service.execute(request) + + intent = repository.intents[request.operation_id] + assert intent.asset_id == str(2**64 - 1) + assert intent.amount_micros == str(2**64 - 1) + assert intent.asset_id_int == 2**64 - 1 + assert intent.amount_micros_int == 2**64 - 1 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("operation_id", ""), + ("operation_id", "x" * 201), + ("receiver", ""), + ("asset_id", True), + ("asset_id", 0), + ("asset_id", 2**64), + ("amount_micros", True), + ("amount_micros", 0), + ("amount_micros", 2**64), + ("note", "x" * 1_001), + ], +) +def test_transfer_request_rejects_invalid_values(field: str, value: object) -> None: + request = _request() + + with pytest.raises(InvalidAssetTransferError): + replace(request, **{field: value}) diff --git a/tests/unit/test_database_indexes.py b/tests/unit/test_database_indexes.py index f83482c8..20f15c74 100644 --- a/tests/unit/test_database_indexes.py +++ b/tests/unit/test_database_indexes.py @@ -4,6 +4,7 @@ import pytest from flex.db.indexes import ( + create_unique_id_index_fail_closed, deduplicate_and_create_unique_id_index, ensure_database_indexes, ) @@ -15,13 +16,16 @@ def _manager(collection: Mock) -> SimpleNamespace: def _database(**collections: Mock) -> SimpleNamespace: names = ( + "airdrop_manifests", "asset_prices", + "asset_transfer_intents", "pool_transactions", "lp_transactions", "lp_states", "pool_states", "user_states", "lp_tokens", + "airdrop_rewards", ) return SimpleNamespace(**{name: _manager(collections.get(name, Mock())) for name in names}) @@ -62,11 +66,34 @@ def test_deduplication_keeps_newest_record_before_creating_index() -> None: ) < collection.mock_calls.index(call.create_index("id", unique=True, name="id_unique")) +def test_immutable_ledger_duplicates_abort_without_deleting_evidence() -> None: + collection = Mock() + collection.aggregate.return_value = [ + { + "_id": "operation", + "count": 2, + "keep_id": "confirmed", + "all_ids": ["confirmed", "prepared"], + } + ] + + with pytest.raises(RuntimeError, match="duplicate immutable ID"): + create_unique_id_index_fail_closed( + collection, + collection_name="asset_transfer_intents", + ) + + collection.delete_many.assert_not_called() + collection.create_index.assert_not_called() + + def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: unique_collections = { name: Mock() for name in ( + "airdrop_manifests", "asset_prices", + "asset_transfer_intents", "pool_transactions", "lp_transactions", ) @@ -79,7 +106,9 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: removed = ensure_database_indexes(database) assert removed == { + "airdrop_manifests": 0, "asset_prices": 0, + "asset_transfer_intents": 0, "pool_transactions": 0, "lp_transactions": 0, } @@ -90,10 +119,17 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: database.pool_states.mongodb_collection.create_index.assert_called_once_with("pool_id", name="pool_id_idx") database.user_states.mongodb_collection.create_index.assert_called_once_with("address", name="address_idx") database.lp_tokens.mongodb_collection.create_index.assert_called_once_with("id", name="lp_token_id_idx") + database.airdrop_rewards.mongodb_collection.create_index.assert_called_once_with( + "operation_id", + unique=True, + name="operation_id_unique", + partialFilterExpression={"operation_id": {"$type": "string"}}, + ) def test_correctness_critical_index_failure_is_not_swallowed() -> None: database = _database() + database.airdrop_manifests.mongodb_collection.aggregate.return_value = [] database.asset_prices.mongodb_collection.aggregate.return_value = [] database.asset_prices.mongodb_collection.create_index.side_effect = RuntimeError("index build failed") diff --git a/tests/unit/test_nft_lottery_payouts.py b/tests/unit/test_nft_lottery_payouts.py new file mode 100644 index 00000000..4fac286d --- /dev/null +++ b/tests/unit/test_nft_lottery_payouts.py @@ -0,0 +1,117 @@ +from copy import deepcopy +from types import SimpleNamespace + +from flex.application.asset_transfers import AssetTransferReceipt + + +def _matches(document: dict, query: dict) -> bool: + for field, expected in query.items(): + if field == "$or": + if not any(_matches(document, branch) for branch in expected): + return False + continue + if isinstance(expected, dict): + if "$exists" in expected: + if (field in document) is not expected["$exists"]: + return False + continue + if "$ne" in expected: + if document.get(field) == expected["$ne"]: + return False + continue + if document.get(field) != expected: + return False + return True + + +class FakeLotteryCollection: + def __init__(self, documents: list[dict]) -> None: + self.documents = documents + self.indexes: list[tuple[tuple, dict]] = [] + + def create_index(self, *args, **kwargs) -> None: + self.indexes.append((args, kwargs)) + + def find(self, query): + return [deepcopy(document) for document in self.documents if _matches(document, query)] + + def find_one(self, query): + document = next((item for item in self.documents if _matches(item, query)), None) + return deepcopy(document) if document is not None else None + + def find_one_and_update(self, query, update, **kwargs): + document = next((item for item in self.documents if _matches(item, query)), None) + if document is None: + return None + document.update(update.get("$set", {})) + return deepcopy(document) + + def update_one(self, query, update): + document = next((item for item in self.documents if _matches(item, query)), None) + if document is not None: + document.update(update.get("$set", {})) + return SimpleNamespace(modified_count=int(document is not None)) + + +def test_lottery_payouts_backfill_unique_draw_ids_and_claim_exact_documents(monkeypatch) -> None: + # Import after test collection helpers so module-level infrastructure stays + # outside the behavioral assertion. + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "draw-a", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + }, + { + "_id": "draw-b", + "wallet": "WALLET-B", + "prize": 202, + "timestamp": 2.0, + "lottery_name": "summer", + "claimed": False, + }, + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + operation_id = f"nft:{idempotency_key}" + return AssetTransferReceipt( + operation_id=operation_id, + txid=f"tx-{asset_id}", + confirmed_round=777, + already_confirmed=False, + ) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 2 + assert result["error_count"] == 0 + assert len(calls) == 2 + assert len({call[2] for call in calls}) == 2 + assert [document["payout_txid"] for document in collection.documents] == [ + "tx-101", + "tx-202", + ] + assert all(document["claimed"] for document in collection.documents) + assert all(document["id"].startswith("legacy-") for document in collection.documents) + + first_calls = list(calls) + retry = nft_lottery.send_all_prizes() + + assert retry["sent_count"] == 0 + assert calls == first_calls diff --git a/tests/unit/test_wallet_transfers.py b/tests/unit/test_wallet_transfers.py new file mode 100644 index 00000000..00452be7 --- /dev/null +++ b/tests/unit/test_wallet_transfers.py @@ -0,0 +1,42 @@ +from api import wallet +from flex.application.asset_transfers import ( + AssetTransferReceipt, + AssetTransferRequest, +) + + +def test_send_nft_requires_and_namespaces_a_stable_idempotency_key(monkeypatch) -> None: + requests = [] + + class FakeService: + def execute(self, request): + requests.append(request) + return AssetTransferReceipt( + operation_id=request.operation_id, + txid="TXID", + confirmed_round=123, + already_confirmed=False, + ) + + monkeypatch.setattr( + wallet, + "get_asset_transfer_service", + lambda: FakeService(), + ) + + receipt = wallet.send_nft( + "RECEIVER", + 42, + idempotency_key="lottery:draw-1", + ) + + assert receipt.txid == "TXID" + assert requests == [ + AssetTransferRequest( + operation_id="nft:lottery:draw-1", + receiver="RECEIVER", + asset_id=42, + amount_micros=1, + note=None, + ) + ] From afd35bca508cf5f709bf3df76484b3762e171c54 Mon Sep 17 00:00:00 2001 From: wackloner Date: Sun, 19 Jul 2026 15:28:01 +0700 Subject: [PATCH 02/10] fix crash-safe lp projection --- .env.example | 5 + BOARD.md | 6 +- Makefile | 29 +- docs/architecture/lp-projection.md | 75 +++ env.py | 5 + flex/blockchain/info.py | 76 ++- flex/data/asset_prices.py | 30 +- flex/data/lp_states.py | 302 ++++++++--- flex/data/tinyman_lps.py | 33 +- flex/db/bson.py | 51 ++ flex/db/classes/base_entity.py | 20 +- flex/db/classes/bson_uint64.py | 33 ++ flex/db/classes/collection_manager.py | 64 +-- flex/db/indexes.py | 82 ++- flex/db/lp_projection.py | 351 +++++++++++++ flex/db/model/blockchain.py | 201 +++++-- flex/db/model/liquidity_pools.py | 125 ++++- flex/db/model/priced.py | 53 +- flex/db/sync_coordinator.py | 193 +++++++ flex/domain/lp_projection.py | 119 +++++ flex/domain/pricing.py | 80 ++- flex/migrations/fix_dex_providers.py | 18 +- flex/sync_pools.py | 210 ++++++-- flex/sync_state.py | 8 +- pyproject.toml | 3 + tests/unit/test_asset_supply.py | 94 ++++ tests/unit/test_bson_uint64_models.py | 78 +++ tests/unit/test_database_indexes.py | 64 ++- tests/unit/test_lp_projection_repository.py | 519 +++++++++++++++++++ tests/unit/test_lp_transaction_projection.py | 245 ++++++++- tests/unit/test_pricing_domain.py | 23 +- tests/unit/test_tinyman_price_projection.py | 20 +- 32 files changed, 2888 insertions(+), 327 deletions(-) create mode 100644 docs/architecture/lp-projection.md create mode 100644 flex/db/bson.py create mode 100644 flex/db/classes/bson_uint64.py create mode 100644 flex/db/lp_projection.py create mode 100644 flex/db/sync_coordinator.py create mode 100644 flex/domain/lp_projection.py create mode 100644 tests/unit/test_asset_supply.py create mode 100644 tests/unit/test_bson_uint64_models.py create mode 100644 tests/unit/test_lp_projection_repository.py diff --git a/.env.example b/.env.example index c192fa84..0a949e06 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,11 @@ MONGODB_USERNAME= MONGODB_PASSWORD= NEW_DB_NAME=cometa-updated +# Financial projectors are opt-in until their chain classifiers are verified. +SYNC_NEW_POOLS=true +SYNC_LIQUIDITY_POOLS=false +SYNC_STAKING_POOLS=false + # Notifications # Syntactically valid, non-secret local placeholder; replace in production. TELEGRAM_BOT_API_TOKEN=123456:test-token diff --git a/BOARD.md b/BOARD.md index e9a19aac..3723762a 100644 --- a/BOARD.md +++ b/BOARD.md @@ -8,17 +8,18 @@ - **Statuses**: `todo` | `in_progress` | `blocked` | `done` - **Priorities**: `critical` | `high` | `medium` | `low` - **Tags**: `security` | `backend` | `infra` | `dx` | `arch` | `perf` -- Next available ID: **CB-079** +- Next available ID: **CB-080** ## Active | ID | Task | Status | Priority | Tags | Definition of done | | --- | --- | --- | --- | --- | --- | | CB-073 | Public repository hardening | in_progress | critical | security, dx | Credentials rotated, history sanitized, secret protection enabled, clean-clone scan passes | -| CB-074 | Atomic event projection | todo | critical | backend, arch | Crash-safe inbox/projector with duplicate, replay, and recovery tests | +| CB-074 | Atomic event projection | in_progress | critical | backend, arch | LP projector is crash-safe; replace the disabled legacy staking projector with verified grouped events and recovery tests | | CB-075 | Isolate transaction signing | todo | high | security, arch | Read-only API boundary; authenticated policy-limited signing service | | CB-076 | Async persistence boundary | todo | high | backend, perf | Storage outages cannot block the event loop; timeouts and readiness covered | | CB-078 | Replay-safe outbound asset payouts | done | critical | security, backend, arch | Exact allocations, immutable airdrop manifests, persisted signed intents, on-chain reconciliation, and regression tests | +| CB-079 | Crash-safe LP projection | done | critical | backend, arch | Decimal128 balances, ordered per-state CAS cursor, fenced round checkpoint, snapshot guards, and crash/concurrency tests | ## Completed milestones @@ -28,6 +29,7 @@ | Precision-safe pricing | done | Decimal observations, provenance, freshness policy, guarded legacy boundary | | Provider resilience | done | Typed fallback errors, bounded stale data, retry classification, circuit breaker | | Replay identity | done | Deterministic nested event IDs and collection-level uniqueness constraints | +| LP financial ledger | done | Marker-gap recovery, uint64-safe BSON operations, full-block preflight, snapshot coverage guards, and fenced round CAS | | Container baseline | done | Digest-pinned Alpine base, multi-stage non-root runtime, healthcheck, image exclusions, Trivy CI gate | | API hardening | done | Fail-closed header authentication, trusted hosts, explicit CORS policy, bounded LP/asset/wallet requests | | Native Reach decoding | done | Versioned global/local codecs, exact-width integers, deterministic layout tests, no private npm runtime | diff --git a/Makefile b/Makefile index 672c6452..c243cd04 100644 --- a/Makefile +++ b/Makefile @@ -2,32 +2,40 @@ PYTHON_LINT_PATHS := \ api/background.py api/nft_lottery.py api/wallet.py app.py blockchain/indexer.py bot/log.py env.py telegram_bot.py \ core/circuit_breaker.py core/cometa.py core/decorators.py core/util.py \ flex/__init__.py flex/api.py flex/application flex/blockchain/asset_transfers.py \ - flex/blockchain/contract_state.py flex/data/asset_prices.py \ + flex/blockchain/contract_state.py flex/blockchain/info.py flex/data/asset_prices.py \ flex/data/lp_prices.py flex/data/lp_states.py flex/data/pool_state.py \ flex/data/tinyman_lps.py flex/data/transactions.py \ - flex/db/asset_transfer_intents.py flex/db/classes/collection_manager.py flex/db/indexes.py \ - flex/db/model/airdrop.py flex/db/model/liquidity_pools.py \ + flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/base_entity.py \ + flex/db/classes/bson_uint64.py flex/db/classes/collection_manager.py \ + flex/db/indexes.py flex/db/lp_projection.py \ + flex/db/sync_coordinator.py flex/db/model/airdrop.py flex/db/model/blockchain.py flex/db/model/liquidity_pools.py \ flex/db/model/priced.py flex/db/model/transfers.py flex/domain flex/providers/pact.py flex/providers/price_router.py \ - flex/providers/vestige.py flex/sync_pools.py \ + flex/providers/vestige.py flex/sync_pools.py flex/sync_state.py \ + flex/migrations/fix_dex_providers.py \ flex/tools/airdrop.py scripts/verify_algorand_credentials.py tests PYTHON_MODERN_PATHS := \ api/background.py api/nft_lottery.py api/wallet.py core/circuit_breaker.py flex/application \ flex/blockchain/asset_transfers.py \ flex/blockchain/contract_state.py flex/data/asset_prices.py flex/data/lp_prices.py \ - flex/db/asset_transfer_intents.py flex/db/model/airdrop.py flex/db/model/priced.py flex/db/model/transfers.py \ + flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/bson_uint64.py \ + flex/db/lp_projection.py flex/db/sync_coordinator.py \ + flex/db/model/airdrop.py flex/db/model/priced.py flex/db/model/transfers.py \ flex/domain flex/providers/pact.py flex/providers/price_router.py tests/unit PYTHON_FORMAT_PATHS := \ api/background.py api/nft_lottery.py api/wallet.py app.py blockchain/indexer.py bot/log.py \ core/circuit_breaker.py core/cometa.py core/util.py \ - flex/api.py flex/application flex/blockchain/asset_transfers.py flex/blockchain/contract_state.py \ + flex/api.py flex/application flex/blockchain/asset_transfers.py flex/blockchain/contract_state.py flex/blockchain/info.py \ flex/data/asset_prices.py flex/data/lp_prices.py \ flex/data/lp_states.py flex/data/pool_state.py flex/data/tinyman_lps.py \ - flex/data/transactions.py flex/db/asset_transfer_intents.py flex/db/indexes.py \ - flex/db/model/airdrop.py flex/db/model/liquidity_pools.py flex/db/model/priced.py flex/db/model/transfers.py \ + flex/data/transactions.py flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/base_entity.py \ + flex/db/classes/bson_uint64.py flex/db/classes/collection_manager.py flex/db/indexes.py flex/db/lp_projection.py \ + flex/db/sync_coordinator.py flex/db/model/airdrop.py flex/db/model/blockchain.py \ + flex/db/model/liquidity_pools.py flex/db/model/priced.py flex/db/model/transfers.py \ flex/domain flex/providers/pact.py flex/providers/price_router.py \ - flex/providers/vestige.py flex/sync_pools.py telegram_bot.py \ + flex/providers/vestige.py flex/sync_pools.py flex/sync_state.py telegram_bot.py \ + flex/migrations/fix_dex_providers.py \ flex/tools/airdrop.py tests/conftest.py tests/unit .PHONY: sync run lint format format-check typecheck test quality @@ -58,9 +66,12 @@ test: --cov=core.decorators \ --cov=flex.application.asset_transfers \ --cov=flex.db.asset_transfer_intents \ + --cov=flex.db.lp_projection \ + --cov=flex.db.sync_coordinator \ --cov=flex.db.classes.collection_manager \ --cov=flex.blockchain.contract_state \ --cov=flex.domain.allocation \ + --cov=flex.domain.lp_projection \ --cov=flex.domain.pricing \ --cov=flex.domain.transactions \ --cov=flex.providers.pact \ diff --git a/docs/architecture/lp-projection.md b/docs/architecture/lp-projection.md new file mode 100644 index 00000000..2043ca05 --- /dev/null +++ b/docs/architecture/lp-projection.md @@ -0,0 +1,75 @@ +# Crash-Safe LP Projection + +The LP projector treats MongoDB’s LP-state document as the authoritative +idempotency boundary. This is required because the deployed standalone MongoDB +cannot atomically update both `lp_states` and `lp_transactions`. + +## Invariants + +Each scoped transfer has a fixed-width cursor: + +```text +:: +``` + +Events are sorted by that cursor. One `find_one_and_update` atomically applies +the Decimal128 balance delta and advances `last_event_order`. The audit marker +is inserted afterward. If the process dies in that gap, replay sees the cursor, +repairs the missing marker, and does not apply the delta again. + +Before the first write, the complete block batch is checked for its expected +round, deterministic order, duplicate IDs, and conflicting payloads. LP +self-transfers are discarded as net zero. Clawback and close semantics fail +closed only when they affect an LP account. + +The repository fails closed when: + +- an event ID exists with different immutable data; +- an event marker exists while its state cursor is still behind; +- a later cursor has passed an unrecorded event; +- a balance would leave the Algorand uint64 range; +- legacy root aliases make an inner event ambiguous; +- duplicate LP token IDs or addresses exist. + +Reserves, issued LP supply, IDs, rounds, positions, and deltas are Python +integers encoded as BSON Decimal128 where MongoDB int64 is insufficient. +Pricing stays in `Decimal`; floats exist only in legacy API fields. ASA total +supply is read directly from Indexer base units and persisted as a decimal +string. + +## Snapshots and derived prices + +Account balances and `current-round` come from the same Indexer response. An +authoritative snapshot advances the cursor to that round’s end sentinel. A +stale snapshot cannot overwrite a newer event, and conflicting balances for the +same snapshot round require reconciliation. Price writers update only derived +fields when the state cursor still matches; a monotonic observation timestamp +prevents an older calculation from overwriting a newer quote. Tinyman reserve +projection writes only the underlying asset quote; the LP token price has one +canonical writer. + +## Worker ordering + +The singleton `sync_states/main` document provides a Mongo lease and a fenced, +compare-and-set round checkpoint. An expired worker can duplicate work but +cannot regress the checkpoint; per-state cursors make that replay convergent. +Cutover advances only through the minimum round fully covered by every LP +cursor. A mid-round event cursor covers the preceding round, not the unfinished +one, so an Indexer lag cannot skip its remaining events. +Repeated deterministic failure uses bounded exponential backoff and terminates +the worker instead of hot-looping. + +`SYNC_LIQUIDITY_POOLS` remains opt-in until an operational snapshot cutover is +performed. `SYNC_STAKING_POOLS` remains disabled because the legacy classifier +does not validate complete Algorand application groups. Enabling it is rejected +at runtime rather than risking financial corruption. + +Standalone MongoDB still cannot make an LP-to-LP transfer visible in two pool +documents simultaneously. Replay guarantees convergence; strict cross-pool +visibility would require a replica-set transaction or a one-document ledger +aggregate. + +Indexer account balances are authoritative for reconciliation, but not +necessarily for a DEX’s economic reserve accounting: donations and protocol +excess balances may be included. Production pricing therefore remains disabled +until each DEX has a verified app-state adapter. diff --git a/env.py b/env.py index 217df01e..82bc2bd5 100644 --- a/env.py +++ b/env.py @@ -63,6 +63,9 @@ class Settings(BaseSettings): sync_new_pools: bool = True sync_liquidity_pools: bool = False + # Legacy stake projection does not yet validate complete application groups. + # Keep it fail-closed until the authoritative event classifier lands. + sync_staking_pools: bool = False update_contract_caches: bool = True update_contracts_chunk_size: int = Field(default=10, ge=1, le=100) @@ -74,6 +77,8 @@ class Settings(BaseSettings): old_pool_end_date_days_ago: int = 30 sync_lag_max_rounds: int = 1000 # 1 hour sync_behind_seconds_threshold: int = 60 + sync_round_max_attempts: int = Field(default=5, ge=1, le=100) + sync_retry_max_seconds: float = Field(default=30, gt=0, le=300) reset_and_resync_pool_states: bool = False diff --git a/flex/blockchain/info.py b/flex/blockchain/info.py index d196c323..7c494a48 100644 --- a/flex/blockchain/info.py +++ b/flex/blockchain/info.py @@ -1,22 +1,31 @@ import asyncio import logging +from dataclasses import dataclass from aiocache import cached from blockchain.node import get_current_round as _sync_get_current_round -from flex.blockchain.base import indexer_client, algod_client +from flex.blockchain.base import algod_client, indexer_client from flex.db.model.blockchain import Asset logger = logging.getLogger(__name__) + +@dataclass(frozen=True, slots=True) +class AssetBalanceSnapshot: + balances: dict[int, int] + observed_round: int + + ALGO_ASSET = Asset( id=0, decimals=6, - name='Algorand', - unit_name='ALGO', - creator='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ', - reserve='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ', - total_supply=10_000_000_000 + name="Algorand", + unit_name="ALGO", + creator="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ", + reserve="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ", + total_supply=10_000_000_000, + total_supply_micros=10_000_000_000_000_000, ) @@ -31,31 +40,48 @@ async def fetch_asset(asset_id: int) -> Asset: return ALGO_ASSET data = await _run_sync(indexer_client.asset_info, asset_id) - params = data['asset']['params'] + params = data["asset"]["params"] return Asset( id=asset_id, - decimals=params['decimals'], - name=params.get('name', ''), - unit_name=params.get('unit-name', ''), - creator=params.get('creator', ''), - reserve=params.get('reserve', ''), - total_supply=params['total'] / (10 ** params['decimals']) + decimals=params["decimals"], + name=params.get("name", ""), + unit_name=params.get("unit-name", ""), + creator=params.get("creator", ""), + reserve=params.get("reserve", ""), + total_supply=0, + total_supply_micros=params["total"], ) async def get_address_assets(address: str) -> dict: - data = await _run_sync(lambda: indexer_client.lookup_account_assets(address=address)) - return {asset['asset-id']: asset['amount'] for asset in data['assets']} + return (await get_address_asset_snapshot(address, include_algo=False)).balances async def get_address_assets_with_algo(address: str) -> dict: + return (await get_address_asset_snapshot(address, include_algo=True)).balances + + +async def get_address_asset_snapshot( + address: str, + *, + include_algo: bool, +) -> AssetBalanceSnapshot: + """Read balances and their authoritative Indexer round in one response.""" + data = await _run_sync(indexer_client.account_info, address) - asset_balances = {asset['asset-id']: asset['amount'] for asset in data['account']['assets']} - asset_balances[0] = data['account']['amount'] - return asset_balances + asset_balances = {asset["asset-id"]: asset["amount"] for asset in data["account"]["assets"]} + if include_algo: + asset_balances[0] = data["account"]["amount"] + observed_round = data.get("current-round") + if isinstance(observed_round, bool) or not isinstance(observed_round, int) or observed_round < 0: + raise RuntimeError("Indexer account snapshot has no valid current-round") + return AssetBalanceSnapshot( + balances=asset_balances, + observed_round=observed_round, + ) -@cached(ttl=10, namespace='node', key='current_round') +@cached(ttl=10, namespace="node", key="current_round") async def get_current_round(): """Async wrapper around the sync get_current_round (single implementation, shared cache).""" return await _run_sync(_sync_get_current_round) @@ -63,22 +89,22 @@ async def get_current_round(): async def get_app_address(app_id: int) -> str: data = await _run_sync(lambda: indexer_client.application_logs(application_id=app_id, limit=10)) - log_data = data['log-data'] + log_data = data["log-data"] - txid = log_data[0]['txid'] + txid = log_data[0]["txid"] data = await _run_sync(lambda: indexer_client.transaction(txid=txid)) - return data['transaction']['inner-txns'][0]['sender'] + return data["transaction"]["inner-txns"][0]["sender"] async def get_address_app_ids(address: str) -> list[int]: data = await _run_sync(lambda: indexer_client.account_info(address=address)) - return [app_state['id'] for app_state in data['account']['apps-local-state']] + return [app_state["id"] for app_state in data["account"]["apps-local-state"]] def is_opted_in(address: str, asa_id: int) -> bool: account_info = algod_client.account_info(address) - for account in account_info.get('assets', []): - if account['asset-id'] == asa_id: + for account in account_info.get("assets", []): + if account["asset-id"] == asa_id: return True return False diff --git a/flex/data/asset_prices.py b/flex/data/asset_prices.py index cf30c1ba..667edc4e 100644 --- a/flex/data/asset_prices.py +++ b/flex/data/asset_prices.py @@ -134,14 +134,20 @@ def _upsert_asset_price(asset_price: AssetPrice) -> bool: doc.pop("_id", None) created = doc.pop("created", asset_price.created) collection = db.asset_prices.mongodb_collection - selector = { - "id": asset_price.id, - "$or": [ - {"observed_at": {"$exists": False}}, - {"observed_at": None}, - {"observed_at": {"$lte": asset_price.observed_at}}, - ], - } + selector = AssetPrice.encode_query( + { + "id": asset_price.id, + "$or": [ + {"observed_at": {"$exists": False}}, + {"observed_at": None}, + { + "observed_at": { + "$lte": asset_price.observed_at, + } + }, + ], + } + ) result = collection.update_one( selector, {"$set": doc, "$setOnInsert": {"created": created}}, @@ -150,7 +156,13 @@ def _upsert_asset_price(asset_price: AssetPrice) -> bool: if result.matched_count: return True - if collection.find_one({"id": asset_price.id}, {"_id": 1}) is not None: + if ( + collection.find_one( + AssetPrice.encode_query({"id": asset_price.id}), + {"_id": 1}, + ) + is not None + ): logger.info( "Discarding older price observation for asset %s at %s", asset_price.id, diff --git a/flex/data/lp_states.py b/flex/data/lp_states.py index 60fdd731..88cd179a 100644 --- a/flex/data/lp_states.py +++ b/flex/data/lp_states.py @@ -1,15 +1,26 @@ +import asyncio import logging +from datetime import UTC, datetime from aiocache import cached from env import settings from flex import db -from flex.blockchain.info import get_address_assets, get_address_assets_with_algo, get_current_round -from flex.data.assets import get_asset_total_supply, micros_to_amount +from flex.blockchain.info import get_address_asset_snapshot +from flex.data.assets import get_asset_total_supply, get_full_asset from flex.data.lp_tokens import get_lp_token_by_id, lp_token_from_tinyman_pool +from flex.db.lp_projection import ( + LpProjectionPersistenceError, + MongoLpProjectionRepository, +) from flex.db.model.blockchain import LpToken from flex.db.model.liquidity_pools import LpState, LpTransaction -from flex.domain.transactions import event_id_aliases +from flex.domain.lp_projection import lp_round_end_order +from flex.domain.pricing import ( + base_units_to_decimal, + calculate_lp_token_price_from_issued_supply, + decimal_to_legacy_float, +) from flex.meta_error import MetaError from flex.providers.tinyman import fetch_algo_tinyman_pool_by_asset_id from flex.providers.vestige import vestige_full_asset_price @@ -30,40 +41,85 @@ async def get_price_algo(asset_id) -> float: async def recalculate_lp_state_price_algo_with_micros(lp_state: LpState) -> LpState: + # Capture this before external reads. A slower, older calculation must not + # overwrite a newer quote for the same balance cursor. + lp_state.derived_observed_at = datetime.now(UTC) + asset1, asset2, lp_token = await asyncio.gather( + get_full_asset(lp_state.asset1_id), + get_full_asset(lp_state.asset2_id), + get_full_asset(lp_state.token_id), + ) + asset1_reserve = base_units_to_decimal( + lp_state.asset1_reserve_micros, + decimals=asset1.decimals, + field="asset1_reserve_micros", + ) + asset2_reserve = base_units_to_decimal( + lp_state.asset2_reserve_micros, + decimals=asset2.decimals, + field="asset2_reserve_micros", + ) + issued_tokens = base_units_to_decimal( + lp_state.total_tokens_micros, + decimals=lp_token.decimals, + field="issued_lp_supply_micros", + ) + lp_state.asset1_reserve = decimal_to_legacy_float( + asset1_reserve, + field="asset1_reserve", + ) + lp_state.asset2_reserve = decimal_to_legacy_float( + asset2_reserve, + field="asset2_reserve", + ) + lp_state.total_tokens = decimal_to_legacy_float( + issued_tokens, + field="issued_tokens", + ) + if lp_state.total_tokens_micros == 0: - lp_state.total_tokens = 0 lp_state.token_price_algo = 0 return lp_state - lp_state.asset1_reserve = await micros_to_amount(lp_state.asset1_id, lp_state.asset1_reserve_micros) - lp_state.asset2_reserve = await micros_to_amount(lp_state.asset2_id, lp_state.asset2_reserve_micros) - lp_state.total_tokens = await micros_to_amount(lp_state.token_id, lp_state.total_tokens_micros) - asset1_price_algo = await get_price_algo(lp_state.asset1_id) - lp_state.token_price_algo = asset1_price_algo * lp_state.asset1_reserve * 2 / lp_state.total_tokens + exact_price = calculate_lp_token_price_from_issued_supply( + asset1_price_algo=asset1_price_algo, + asset1_reserve_micros=lp_state.asset1_reserve_micros, + asset1_decimals=asset1.decimals, + issued_lp_supply_micros=lp_state.total_tokens_micros, + lp_token_decimals=lp_token.decimals, + ) + lp_state.token_price_algo = decimal_to_legacy_float( + exact_price, + field="token_price_algo", + ) return lp_state -async def create_lp_state_by_lp_token_id(lp_token_id: int, current_round: int | None = None) -> LpState: +async def create_lp_state_by_lp_token_id(lp_token_id: int) -> LpState: lp_token = await get_lp_token_by_id(lp_token_id) if lp_token is None: raise MetaError(f"LP token not found for ID {lp_token_id}") - return await create_lp_state_by_lp_token(lp_token, current_round) - + return await create_lp_state_by_lp_token(lp_token) -async def create_lp_state_by_lp_token(lp_token: LpToken, current_round: int | None = None) -> LpState: - if db.lp_states.exists(token_id=lp_token.id): - raise MetaError(f"LP state already exists for LP token ID {lp_token.id}") +async def create_lp_state_by_lp_token(lp_token: LpToken) -> LpState: # TODO: remove after test if lp_token.asset1_id == 0: raise MetaError(f"ALGO LP token asset1 ID = 0, not id2: {lp_token}") if lp_token.asset2_id == 0: - balances = await get_address_assets_with_algo(lp_token.address) + snapshot = await get_address_asset_snapshot( + lp_token.address, + include_algo=True, + ) else: - balances = await get_address_assets(lp_token.address) + snapshot = await get_address_asset_snapshot( + lp_token.address, + include_algo=False, + ) + balances = snapshot.balances asset1_reserve_micros = balances[lp_token.asset1_id] asset2_reserve_micros = balances[lp_token.asset2_id] @@ -73,13 +129,7 @@ async def create_lp_state_by_lp_token(lp_token: LpToken, current_round: int | No lp_token_total_supply_micros = await get_asset_total_supply(lp_token.id) issued_lp_tokens_micros = lp_token_total_supply_micros - lp_token_reserve_micros - asset1_price_full = await vestige_full_asset_price(lp_token.asset1_id) - asset1_reserve = await micros_to_amount(lp_token.asset1_id, asset1_reserve_micros) - asset2_reserve = await micros_to_amount(lp_token.asset2_id, asset2_reserve_micros) - issued_tokens = await micros_to_amount(lp_token.id, issued_lp_tokens_micros) - lp_token_price_algo = asset1_price_full.algo * asset1_reserve * 2 / issued_tokens - - current_round = current_round or (await get_current_round()) + current_round = snapshot.observed_round lp_state = LpState( id=lp_token.pool_id, token_id=lp_token.id, @@ -90,26 +140,53 @@ async def create_lp_state_by_lp_token(lp_token: LpToken, current_round: int | No asset1_reserve_micros=asset1_reserve_micros, asset2_reserve_micros=asset2_reserve_micros, total_tokens_micros=issued_lp_tokens_micros, - asset1_reserve=asset1_reserve, - asset2_reserve=asset2_reserve, - total_tokens=issued_tokens, - token_price_algo=lp_token_price_algo, + asset1_reserve=0, + asset2_reserve=0, + total_tokens=0, + token_price_algo=0, last_updated_round=current_round, + last_event_order=lp_round_end_order(current_round), is_algo_pool=lp_token.asset2_id == 0, ) + lp_state = await recalculate_lp_state_price_algo_with_micros(lp_state) if lp_state.is_algo_pool: logger.info(f"Created new LP state for ALGO pool, asa_id={lp_token.asset1_id}:\n{lp_state.pretty_str()}") - db.lp_states.create(lp_state) - return lp_state + persisted = db.lp_states.get_or_create(lp_state) + immutable_fields = ( + persisted.id, + persisted.token_id, + persisted.asset1_id, + persisted.asset2_id, + persisted.dex_provider, + persisted.address, + ) + requested_fields = ( + lp_state.id, + lp_state.token_id, + lp_state.asset1_id, + lp_state.asset2_id, + lp_state.dex_provider, + lp_state.address, + ) + if immutable_fields != requested_fields: + raise MetaError(f"LP token ID {lp_token.id} is already mapped to different pool metadata") + return persisted async def update_lp_state(lp_state: LpState) -> LpState: if lp_state.asset1_id == 0 or lp_state.asset2_id == 0: # TODO: take values from DB sync - balances = await get_address_assets_with_algo(lp_state.address) + snapshot = await get_address_asset_snapshot( + lp_state.address, + include_algo=True, + ) else: - balances = await get_address_assets(lp_state.address) + snapshot = await get_address_asset_snapshot( + lp_state.address, + include_algo=False, + ) + balances = snapshot.balances lp_state.asset1_reserve_micros = balances[lp_state.asset1_id] lp_state.asset2_reserve_micros = balances[lp_state.asset2_id] @@ -119,15 +196,22 @@ async def update_lp_state(lp_state: LpState) -> LpState: lp_state.total_tokens_micros = lp_token_total_supply_micros - lp_token_reserve_micros lp_state = await recalculate_lp_state_price_algo_with_micros(lp_state) - lp_state.last_updated_round = await get_current_round() - db.lp_states.update(lp_state) - - return lp_state + current_round = snapshot.observed_round + repository = MongoLpProjectionRepository( + states=db.lp_states.mongodb_collection, + events=db.lp_transactions.mongodb_collection, + ) + return await asyncio.to_thread( + repository.replace_snapshot, + lp_state, + observed_round=current_round, + ) async def create_lp_states_from_all_pools() -> list[LpState]: farming_pools = db.farming_pools.get_all() new_lp_states = [] + failures: list[Exception] = [] for farming_pool in farming_pools: try: if db.lp_states.exists(token_id=farming_pool.stake_token.id): @@ -136,58 +220,114 @@ async def create_lp_states_from_all_pools() -> list[LpState]: lp_state = await create_lp_state_by_lp_token_id(farming_pool.stake_token.id) new_lp_states.append(lp_state) except Exception as e: + failures.append(e) logger.error(f"Failed to create LP state: {e}\n{farming_pool.pretty_str()}", exc_info=True) + if failures: + raise LpProjectionPersistenceError( + f"failed to create {len(failures)} LP state(s) during cutover" + ) from failures[0] return new_lp_states -async def update_lp_states_with_transactions(transactions: list[LpTransaction]) -> list[LpState]: - if len(transactions) == 0: - return [] - - updated_lp_states = {} - applied_transactions = [] - seen_event_ids: set[str] = set() - for tx in transactions: - if tx.id in seen_event_ids or any(db.lp_transactions.exists(id=alias) for alias in event_id_aliases(tx.id)): - logger.debug(f"Transaction {tx.id} already recorded in DB") - continue - - lp_state = updated_lp_states.get(tx.pool_address) - if lp_state is None: - # TODO: cache - lp_state = db.lp_states.get_one(address=tx.pool_address) - if lp_state is None: - logger.error(f"LP state not found for address {tx.pool_address}") - continue - updated_lp_states[tx.pool_address] = lp_state - - if tx.asa_id == lp_state.token_id: - lp_state.total_tokens_micros += -tx.delta_amount_micros - elif tx.asa_id == lp_state.asset1_id: - lp_state.asset1_reserve_micros += tx.delta_amount_micros - elif tx.asa_id == lp_state.asset2_id: - lp_state.asset2_reserve_micros += tx.delta_amount_micros - else: - logger.error(f"Invalid tx {tx.id} ASA ID {tx.asa_id} for LP state {lp_state.id}") +def _preflight_lp_transactions( + transactions: list[LpTransaction], + *, + expected_round: int, +) -> list[LpTransaction]: + """Validate and canonicalize a complete block batch before any writes.""" + + canonical_by_id: dict[str, LpTransaction] = {} + for transaction in transactions: + if transaction.confirmed_round != expected_round: + raise LpProjectionPersistenceError( + f"LP event {transaction.id!r} belongs to round {transaction.confirmed_round}, expected {expected_round}" + ) + if transaction.event_order is None: + raise LpProjectionPersistenceError(f"LP event {transaction.id!r} has no deterministic order") + + existing = canonical_by_id.get(transaction.id) + if existing is None: + canonical_by_id[transaction.id] = transaction continue + if ( + existing.pool_address, + existing.user_address, + existing.asa_id, + existing.delta_amount_micros, + existing.confirmed_round, + existing.event_position, + existing.event_order, + ) != ( + transaction.pool_address, + transaction.user_address, + transaction.asa_id, + transaction.delta_amount_micros, + transaction.confirmed_round, + transaction.event_position, + transaction.event_order, + ): + raise LpProjectionPersistenceError(f"LP event ID {transaction.id!r} has conflicting data in one block") + + return sorted( + canonical_by_id.values(), + key=lambda item: item.event_order or "", + ) - lp_state.last_updated_round = tx.confirmed_round - applied_transactions.append(tx) - seen_event_ids.add(tx.id) - lp_states = [] - for lp_state in updated_lp_states.values(): - # TODO: optimize calculate only changed fields - lp_state = await recalculate_lp_state_price_algo_with_micros(lp_state) - db.lp_states.update(lp_state) - lp_states.append(lp_state) +async def update_lp_states_with_transactions( + transactions: list[LpTransaction], + *, + expected_round: int, +) -> list[LpState]: + if not transactions: + return [] - if applied_transactions: - db.lp_transactions.create_many(applied_transactions) + transactions = _preflight_lp_transactions( + transactions, + expected_round=expected_round, + ) + repository = MongoLpProjectionRepository( + states=db.lp_states.mongodb_collection, + events=db.lp_transactions.mongodb_collection, + ) + changed_addresses: set[str] = set() + applied_event_count = 0 + for transaction in transactions: + outcome = await asyncio.to_thread( + repository.project, + transaction, + ) + if outcome.requires_derived_refresh: + changed_addresses.add(transaction.pool_address) + if outcome.changed_balances: + applied_event_count += 1 + + updated_states: list[LpState] = [] + for pool_address in sorted(changed_addresses): + for _ in range(2): + state = await asyncio.to_thread(repository.get_state, pool_address) + expected_cursor = state.last_event_order + if expected_cursor is None: + raise LpProjectionPersistenceError(f"LP state {state.token_id} has no event cursor") + state = await recalculate_lp_state_price_algo_with_micros(state) + updated = await asyncio.to_thread( + repository.update_derived_fields, + state, + expected_cursor=expected_cursor, + ) + if updated is not None: + updated_states.append(updated) + break + else: + raise LpProjectionPersistenceError(f"LP state {pool_address} kept changing during price persistence") - logger.info(f"Updated {len(lp_states)} LP states with {len(applied_transactions)} transactions") - return lp_states + logger.info( + "Updated %s LP states with %s new transaction(s)", + len(updated_states), + applied_event_count, + ) + return updated_states async def update_all_lp_states_linear() -> list[LpState]: @@ -195,13 +335,19 @@ async def update_all_lp_states_linear() -> list[LpState]: logger.debug(f"Updating {len(lp_states)} LP states...") updated_lp_states = [] + failures: list[Exception] = [] for lp_state in lp_states: try: updated_lp_state = await update_lp_state(lp_state) updated_lp_states.append(updated_lp_state) except Exception as e: + failures.append(e) logger.error(f"Error updating state of LP {lp_state.id}: {e}", exc_info=True) + if failures: + raise LpProjectionPersistenceError( + f"failed to snapshot {len(failures)} LP state(s); sync checkpoint was not advanced" + ) from failures[0] logger.debug(f"Updated {len(updated_lp_states)} LP states") return updated_lp_states diff --git a/flex/data/tinyman_lps.py b/flex/data/tinyman_lps.py index 5e962c49..eb3d0afd 100644 --- a/flex/data/tinyman_lps.py +++ b/flex/data/tinyman_lps.py @@ -5,7 +5,6 @@ from decimal import Decimal, localcontext from env import settings -from flex import db from flex.blockchain.info import ALGO_ASSET from flex.data.asset_prices import _upsert_asset_price from flex.data.assets import get_asset_details @@ -31,20 +30,19 @@ def _pool_observed_at(lp_state: LpState) -> datetime: return observed_at.astimezone(UTC) -async def _tinyman_pool_quotes( +async def _tinyman_asset_quote( lp_state: LpState, *, algo_quote: PriceQuote, -) -> tuple[PriceQuote, PriceQuote, str]: +) -> tuple[PriceQuote, str]: if not lp_state.is_algo_pool or lp_state.asset2_id != ALGO_ASSET.id: raise InvalidLiquidityPoolError("Tinyman price source must be an asset/ALGO pool") - if lp_state.asset1_reserve_micros <= 0 or lp_state.asset2_reserve_micros <= 0 or lp_state.total_tokens_micros <= 0: - raise InvalidLiquidityPoolError("Tinyman reserves and issued LP supply must be positive") + if lp_state.asset1_reserve_micros <= 0 or lp_state.asset2_reserve_micros <= 0: + raise InvalidLiquidityPoolError("Tinyman reserves must be positive") if algo_quote.asset_id != ALGO_ASSET.id: raise InvalidPriceError("Tinyman projection requires an ALGO/USD quote") asset_details = await get_asset_details(lp_state.asset1_id) - lp_details = await get_asset_details(lp_state.token_id) with localcontext() as context: context.prec = PERSISTED_PRICE_PRECISION asset_reserve = Decimal(lp_state.asset1_reserve_micros) / Decimal( @@ -53,11 +51,7 @@ async def _tinyman_pool_quotes( algo_reserve = Decimal(lp_state.asset2_reserve_micros) / Decimal( 10**ALGO_ASSET.decimals, ) - issued_lp_tokens = Decimal(lp_state.total_tokens_micros) / Decimal( - 10**lp_details.decimals, - ) asset_price_algo = +(algo_reserve / asset_reserve) - lp_price_algo = +(algo_reserve * Decimal(2) / issued_lp_tokens) algo_usd = algo_quote.usd stale_after = timedelta(seconds=settings.asset_prices_ttl) @@ -71,30 +65,19 @@ async def _tinyman_pool_quotes( observed_round=lp_state.last_updated_round, observed_at=observed_at, ) - lp_quote = PriceQuote.from_raw( - asset_id=lp_state.token_id, - algo=lp_price_algo, - usd=lp_price_algo * algo_usd, - source=PriceSource.DERIVED_LP, - stale_after=stale_after, - observed_round=lp_state.last_updated_round, - observed_at=observed_at, - ) - return asset_quote, lp_quote, asset_details.name + return asset_quote, asset_details.name -async def update_tinyman_algo_lp_state_and_prices( +async def update_tinyman_algo_asset_price( lp_state: LpState, algo_quote: PriceQuote, ) -> AssetPrice: - """Project one validated Tinyman observation into LP and asset read models.""" + """Project one validated Tinyman observation into the asset read model.""" - asset_quote, lp_quote, asset_name = await _tinyman_pool_quotes( + asset_quote, asset_name = await _tinyman_asset_quote( lp_state, algo_quote=algo_quote, ) - lp_state.token_price_algo = lp_quote.to_legacy_floats()[0] - db.lp_states.update(lp_state) price_algo, price_usd = asset_quote.to_legacy_floats() asset_price = AssetPrice( diff --git a/flex/db/bson.py b/flex/db/bson.py new file mode 100644 index 00000000..4e5403ce --- /dev/null +++ b/flex/db/bson.py @@ -0,0 +1,51 @@ +"""Lossless BSON codecs for Algorand integer domains.""" + +from decimal import Decimal +from typing import Any + +from bson import Decimal128 + +ALGORAND_UINT64_MAX = 2**64 - 1 + + +def _decode_integral_decimal(value: object, *, signed: bool) -> int: + decimal_value = value.to_decimal() if isinstance(value, Decimal128) else Decimal(str(value)) + if not decimal_value.is_finite() or decimal_value != decimal_value.to_integral_value(): + raise ValueError("on-chain amount must be a finite integer") + parsed = int(decimal_value) + minimum = -ALGORAND_UINT64_MAX if signed else 0 + if not minimum <= parsed <= ALGORAND_UINT64_MAX: + raise ValueError("on-chain amount is outside the Algorand uint64 range") + return parsed + + +def decode_bson_uint64(value: object) -> int: + return _decode_integral_decimal(value, signed=False) + + +def decode_bson_delta(value: object) -> int: + return _decode_integral_decimal(value, signed=True) + + +def decode_optional_bson_uint64(value: object | None) -> int | None: + return None if value is None else decode_bson_uint64(value) + + +def encode_bson_integer(value: int) -> Decimal128: + return Decimal128(Decimal(value)) + + +def encode_optional_bson_uint64( + value: int | None, +) -> Decimal128 | None: + return None if value is None else encode_bson_integer(value) + + +def encode_uint64_query_value(value: Any) -> Any: + if isinstance(value, int) and not isinstance(value, bool): + return encode_bson_integer(value) + if isinstance(value, list): + return [encode_uint64_query_value(item) for item in value] + if isinstance(value, dict): + return {operator: encode_uint64_query_value(operand) for operator, operand in value.items()} + return value diff --git a/flex/db/classes/base_entity.py b/flex/db/classes/base_entity.py index 8d25fc1e..0f875a90 100644 --- a/flex/db/classes/base_entity.py +++ b/flex/db/classes/base_entity.py @@ -1,11 +1,10 @@ import json from datetime import datetime - -from typing import Any, TypeVar, Generic +from typing import Any, Generic, TypeVar from flex.db.util import string_to_snake_case -EntityT = TypeVar('EntityT') +EntityT = TypeVar("EntityT") # IMPLEMENTATIONS MUST be dataclass and dataclass_json @@ -16,7 +15,20 @@ class BaseEntity(Generic[EntityT]): @classmethod def primary_key_name(cls) -> str: - return 'id' + return "id" + + @classmethod + def encode_query(cls, query: dict[str, Any]) -> dict[str, Any]: + """Encode entity-specific storage types used in Mongo selectors.""" + return dict(query) + + @classmethod + def encode_storage_fields( + cls, + values: dict[str, Any], + ) -> dict[str, Any]: + """Encode a partial Mongo update using the entity storage schema.""" + return dict(values) @property def primary_key(self) -> Any: diff --git a/flex/db/classes/bson_uint64.py b/flex/db/classes/bson_uint64.py new file mode 100644 index 00000000..fea3dd77 --- /dev/null +++ b/flex/db/classes/bson_uint64.py @@ -0,0 +1,33 @@ +"""Entity mixin for lossless uint64 selectors and partial updates.""" + +from typing import Any, ClassVar + +from flex.db.bson import encode_uint64_query_value + + +class BsonUint64StorageMixin: + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset() + + @classmethod + def encode_query( + cls, + query: dict[str, Any], + ) -> dict[str, Any]: + encoded: dict[str, Any] = {} + for field_name, value in query.items(): + if field_name in cls.BSON_UINT64_FIELDS: + encoded[field_name] = encode_uint64_query_value(value) + elif field_name in {"$and", "$nor", "$or"} and isinstance(value, list): + encoded[field_name] = [ + cls.encode_query(branch) if isinstance(branch, dict) else branch for branch in value + ] + else: + encoded[field_name] = value + return encoded + + @classmethod + def encode_storage_fields( + cls, + values: dict[str, Any], + ) -> dict[str, Any]: + return cls.encode_query(values) diff --git a/flex/db/classes/collection_manager.py b/flex/db/classes/collection_manager.py index f6cc7b69..50ff028d 100644 --- a/flex/db/classes/collection_manager.py +++ b/flex/db/classes/collection_manager.py @@ -17,7 +17,7 @@ class DbError(Exception): message: str -EntityT = TypeVar('EntityT', bound='BaseEntity') +EntityT = TypeVar("EntityT", bound="BaseEntity") @dataclass @@ -40,7 +40,7 @@ def create_with(self, **kwargs) -> EntityT: return self.create(item) def get_one(self, **kwargs) -> EntityT | None: - res = self.mongodb_collection.find_one(kwargs) + res = self.mongodb_collection.find_one(self.elem_type.encode_query(kwargs)) if res is None: return None return self.item_from_dict(dict(res)) @@ -48,16 +48,16 @@ def get_one(self, **kwargs) -> EntityT | None: def get_by_primary_key(self, val: Any, throw_ex: bool = True) -> EntityT | None: res = self.get_one(**{self.primary_key_name: val}) if res is None and throw_ex: - raise DbError(code=404, message=f'No {self.name} found with {self.primary_key_name}={val}') + raise DbError(code=404, message=f"No {self.name} found with {self.primary_key_name}={val}") return res def get_or_create(self, item: EntityT) -> EntityT: """Use one upsert operation; concurrent uniqueness requires a primary-key index.""" - query = {self.primary_key_name: item.primary_key} + query = self.elem_type.encode_query({self.primary_key_name: item.primary_key}) try: document = self.mongodb_collection.find_one_and_update( query, - {'$setOnInsert': item.to_dict()}, + {"$setOnInsert": item.to_dict()}, upsert=True, return_document=ReturnDocument.AFTER, ) @@ -66,34 +66,30 @@ def get_or_create(self, item: EntityT) -> EntityT: if document is None: raise if document is None: - raise DbError(code=500, message=f'Failed to read {self.name} after upsert') + raise DbError(code=500, message=f"Failed to read {self.name} after upsert") return self.item_from_dict(document) def get_or_create_with(self, **kwargs) -> EntityT: return self.get_or_create(self.elem_type(**kwargs)) def get_many(self, **kwargs) -> list[EntityT]: - items = self.mongodb_collection.find(kwargs) + items = self.mongodb_collection.find(self.elem_type.encode_query(kwargs)) return [self.item_from_dict(i) for i in items] def get_many_by_query( - self, - query_dict: dict, - sort_by: str | None = None, - reversed: bool = False, - limit: int| None = None + self, query_dict: dict, sort_by: str | None = None, reversed: bool = False, limit: int | None = None ) -> list[EntityT]: - items = self.mongodb_collection.find(query_dict) + items = self.mongodb_collection.find(self.elem_type.encode_query(query_dict)) if sort_by is not None: items = items.sort(sort_by, DESCENDING if reversed else ASCENDING) elif reversed: - items = items.sort('_id', DESCENDING) + items = items.sort("_id", DESCENDING) if limit is not None: items = items.limit(limit) return [self.item_from_dict(i) for i in items] def get_by_array(self, field_name: str, values: list[Any]) -> list[EntityT]: - return self.get_many(**{field_name: {'$in': values}}) + return self.get_many(**{field_name: {"$in": values}}) def get_all(self) -> list[EntityT]: return self.get_many() @@ -102,42 +98,52 @@ def update(self, item: EntityT) -> EntityT: item.updated = datetime.now() item_dict = item.to_dict() self.mongodb_collection.update_one( - {self.primary_key_name: item.primary_key}, {'$set': item_dict} + self.elem_type.encode_query({self.primary_key_name: item.primary_key}), + {"$set": item_dict}, ) return item def update_with(self, item: EntityT, **kwargs) -> EntityT: - kwargs['updated'] = datetime.now() + kwargs["updated"] = datetime.now() + encoded_fields = self.elem_type.encode_storage_fields(kwargs) self.mongodb_collection.update_one( - {self.primary_key_name: item.primary_key}, {'$set': kwargs} + self.elem_type.encode_query({self.primary_key_name: item.primary_key}), + {"$set": encoded_fields}, ) item_dict = item.to_dict() - item_dict.update(kwargs) + item_dict.update(encoded_fields) return self.item_from_dict(item_dict) def update_many_with(self, filter: dict, **kwargs) -> int: - kwargs['updated'] = datetime.now() - res = self.mongodb_collection.update_many(filter, {'$set': kwargs}) + kwargs["updated"] = datetime.now() + res = self.mongodb_collection.update_many( + self.elem_type.encode_query(filter), + {"$set": self.elem_type.encode_storage_fields(kwargs)}, + ) return res.modified_count def remove(self, item: EntityT) -> bool: - res = self.mongodb_collection.delete_one( - {self.primary_key_name: item.primary_key} - ) + res = self.mongodb_collection.delete_one(self.elem_type.encode_query({self.primary_key_name: item.primary_key})) return res.deleted_count > 0 def remove_by(self, **kwargs) -> int: - res = self.mongodb_collection.delete_many(kwargs) + res = self.mongodb_collection.delete_many(self.elem_type.encode_query(kwargs)) return res.deleted_count def remove_all(self) -> int: return self.remove_by() def count(self, **kwargs) -> int: - return self.mongodb_collection.count_documents(kwargs) + return self.mongodb_collection.count_documents(self.elem_type.encode_query(kwargs)) def exists(self, **kwargs) -> bool: - return self.mongodb_collection.find_one(kwargs, projection={'_id': 1}) is not None + return ( + self.mongodb_collection.find_one( + self.elem_type.encode_query(kwargs), + projection={"_id": 1}, + ) + is not None + ) def item_from_dict(self, item_dict: dict) -> EntityT: return self.elem_type.from_dict(item_dict) @@ -147,7 +153,7 @@ def primary_key_name(self) -> str: return self.elem_type.primary_key_name() @classmethod - def create_for_type(cls, elem_type: Type[EntityT], mongodb_database: MongoDatabase) -> 'CollectionManager[EntityT]': - name = f'{elem_type.type_name_snake_case()}s' + def create_for_type(cls, elem_type: Type[EntityT], mongodb_database: MongoDatabase) -> "CollectionManager[EntityT]": + name = f"{elem_type.type_name_snake_case()}s" collection = mongodb_database[name] return cls(name, elem_type, collection) diff --git a/flex/db/indexes.py b/flex/db/indexes.py index 629d2d45..93c8873a 100644 --- a/flex/db/indexes.py +++ b/flex/db/indexes.py @@ -17,19 +17,21 @@ ) _HOT_INDEXES = ( - ("lp_states", "token_id", "token_id_idx"), + ("lp_states", "token_id", "token_id_unique"), + ("lp_states", "address", "address_unique"), ("pool_states", "pool_id", "pool_id_idx"), ("user_states", "address", "address_idx"), ("lp_tokens", "id", "lp_token_id_idx"), ) -def _duplicate_id_pipeline() -> list[dict[str, Any]]: - """Group duplicate IDs after sorting the newest record first.""" +def _duplicate_field_pipeline(field_name: str) -> list[dict[str, Any]]: + """Group duplicate business keys after sorting the newest record first.""" + return [ { "$sort": { - "id": 1, + field_name: 1, "observed_at": -1, "updated": -1, "_id": -1, @@ -37,7 +39,7 @@ def _duplicate_id_pipeline() -> list[dict[str, Any]]: }, { "$group": { - "_id": "$id", + "_id": f"${field_name}", "count": {"$sum": 1}, "keep_id": {"$first": "$_id"}, "all_ids": {"$push": "$_id"}, @@ -54,7 +56,12 @@ def deduplicate_and_create_unique_id_index( ) -> int: """Keep the newest document per ID, then enforce uniqueness.""" removed = 0 - duplicate_groups: Sequence[dict[str, Any]] = list(collection.aggregate(_duplicate_id_pipeline(), allowDiskUse=True)) + duplicate_groups: Sequence[dict[str, Any]] = list( + collection.aggregate( + _duplicate_field_pipeline("id"), + allowDiskUse=True, + ) + ) for group in duplicate_groups: duplicate_ids = [document_id for document_id in group["all_ids"] if document_id != group["keep_id"]] @@ -80,7 +87,7 @@ def create_unique_id_index_fail_closed( duplicate_groups: Sequence[dict[str, Any]] = list( collection.aggregate( - _duplicate_id_pipeline(), + _duplicate_field_pipeline("id"), allowDiskUse=True, ) ) @@ -92,6 +99,55 @@ def create_unique_id_index_fail_closed( collection.create_index("id", unique=True, name="id_unique") +def create_unique_field_index_fail_closed( + collection: Collection[dict[str, Any]], + *, + collection_name: str, + field_name: str, + index_name: str, +) -> None: + """Enforce aggregate identity without destroying conflicting state.""" + + duplicate_groups: Sequence[dict[str, Any]] = list( + collection.aggregate( + _duplicate_field_pipeline(field_name), + allowDiskUse=True, + ) + ) + if duplicate_groups: + raise RuntimeError( + f"{collection_name} contains {len(duplicate_groups)} duplicate {field_name!r} group(s); " + "reconcile them before startup" + ) + collection.create_index( + field_name, + unique=True, + name=index_name, + ) + + +def ensure_sync_state_singleton(database: CometaDatabase) -> None: + """Migrate one legacy random-ID cursor and reject competing checkpoints.""" + + collection = database.sync_states.mongodb_collection + documents = list( + collection.find( + {}, + projection={"_id": 1, "id": 1, "last_round": 1}, + ) + ) + if len(documents) > 1: + raise RuntimeError( + f"sync_states contains {len(documents)} competing checkpoints; reconcile them before startup" + ) + if documents and documents[0].get("id") != "main": + collection.update_one( + {"_id": documents[0]["_id"]}, + {"$set": {"id": "main"}}, + ) + collection.create_index("id", unique=True, name="id_unique") + + def ensure_database_indexes(database: CometaDatabase) -> dict[str, int]: """Install correctness-critical unique indexes and hot query indexes.""" removed_by_collection: dict[str, int] = {} @@ -110,9 +166,19 @@ def ensure_database_indexes(database: CometaDatabase) -> dict[str, int]: ) removed_by_collection[collection_name] = 0 + ensure_sync_state_singleton(database) + for manager_name, field_name, index_name in _HOT_INDEXES: manager = getattr(database, manager_name) - manager.mongodb_collection.create_index(field_name, name=index_name) + if manager_name == "lp_states": + create_unique_field_index_fail_closed( + manager.mongodb_collection, + collection_name=manager_name, + field_name=field_name, + index_name=index_name, + ) + else: + manager.mongodb_collection.create_index(field_name, name=index_name) database.airdrop_rewards.mongodb_collection.create_index( "operation_id", diff --git a/flex/db/lp_projection.py b/flex/db/lp_projection.py new file mode 100644 index 00000000..183e0b80 --- /dev/null +++ b/flex/db/lp_projection.py @@ -0,0 +1,351 @@ +"""Atomic MongoDB adapter for liquidity-pool event projection.""" + +from dataclasses import dataclass +from datetime import UTC, datetime +from enum import StrEnum +from typing import Any, cast + +from pymongo import ReturnDocument +from pymongo.collection import Collection +from pymongo.errors import DuplicateKeyError + +from flex.db.bson import encode_bson_integer +from flex.db.model.liquidity_pools import LpState, LpTransaction +from flex.domain.lp_projection import ( + MAX_ALGORAND_UINT, + LpBalanceDelta, + lp_balance_delta, + lp_round_end_order, + snapshot_covers_event, +) +from flex.domain.transactions import event_id_aliases + + +class LpProjectionPersistenceError(RuntimeError): + """Raised when replay cannot prove a safe LP projection outcome.""" + + +class LpProjectionResult(StrEnum): + APPLIED = "applied" + ALREADY_APPLIED = "already_applied" + SNAPSHOT_COVERED = "snapshot_covered" + + +@dataclass(frozen=True, slots=True) +class LpProjectionOutcome: + state: LpState + result: LpProjectionResult + + @property + def changed_balances(self) -> bool: + return self.result is LpProjectionResult.APPLIED + + @property + def requires_derived_refresh(self) -> bool: + return self.result is not LpProjectionResult.SNAPSHOT_COVERED + + +@dataclass(slots=True) +class MongoLpProjectionRepository: + states: Collection[dict[str, Any]] + events: Collection[dict[str, Any]] + + def project(self, transaction: LpTransaction) -> LpProjectionOutcome: + """Apply one event atomically or prove that replay is already covered.""" + + while True: + state = self._get_state(transaction.pool_address) + order = transaction.event_order + if order is None: + raise LpProjectionPersistenceError("LP transaction has no deterministic event order") + + canonical_event = self.events.find_one( + {"id": transaction.id}, + ) + if canonical_event is not None: + self._assert_event_document( + canonical_event, + expected=transaction, + ) + legacy_alias = self._legacy_alias(transaction) + if legacy_alias is not None: + raise LpProjectionPersistenceError( + f"legacy LP marker {legacy_alias!r} is ambiguous for {transaction.id!r}; " + "reconcile from an authoritative pool snapshot" + ) + + cursor = state.last_event_order + if cursor is None: + raise LpProjectionPersistenceError( + f"LP state {state.token_id} has no cutover cursor; take an authoritative snapshot before replay" + ) + + if canonical_event is not None: + if cursor < order: + raise LpProjectionPersistenceError( + f"LP event {transaction.id!r} is recorded but state " + f"{state.token_id} is behind it; reconcile from an " + "authoritative pool snapshot" + ) + return LpProjectionOutcome( + state=state, + result=LpProjectionResult.ALREADY_APPLIED, + ) + + if cursor == order: + self._record_event(transaction) + return LpProjectionOutcome( + state=state, + result=LpProjectionResult.ALREADY_APPLIED, + ) + + if cursor > order: + if snapshot_covers_event( + cursor, + confirmed_round=transaction.confirmed_round, + ): + return LpProjectionOutcome( + state=state, + result=LpProjectionResult.SNAPSHOT_COVERED, + ) + raise LpProjectionPersistenceError( + f"LP state {state.token_id} advanced past unrecorded event {transaction.id!r}" + ) + + delta = lp_balance_delta( + token_id=state.token_id, + asset1_id=state.asset1_id, + asset2_id=state.asset2_id, + event_asset_id=transaction.asa_id, + event_pool_delta_micros=transaction.delta_amount_micros, + ) + updated = self._apply_delta( + state=state, + order=order, + confirmed_round=transaction.confirmed_round, + delta=delta, + ) + if updated is None: + latest = self._get_state(transaction.pool_address) + if latest.last_event_order != cursor: + continue + raise LpProjectionPersistenceError( + f"LP event {transaction.id!r} would underflow or overflow {delta.field}" + ) + + # Marker-last is intentional: if this write fails, replay observes + # cursor == order and heals the marker without repeating the delta. + self._record_event(transaction) + return LpProjectionOutcome( + state=updated, + result=LpProjectionResult.APPLIED, + ) + + def update_derived_fields( + self, + state: LpState, + *, + expected_cursor: str, + ) -> LpState | None: + if state.derived_observed_at is None: + raise LpProjectionPersistenceError("derived LP fields have no observation timestamp") + document = self.states.find_one_and_update( + { + "token_id": encode_bson_integer(state.token_id), + "last_event_order": expected_cursor, + "$or": [ + {"derived_observed_at": {"$exists": False}}, + {"derived_observed_at": None}, + { + "derived_observed_at": { + "$lte": state.derived_observed_at, + } + }, + ], + }, + { + "$set": { + "asset1_reserve": state.asset1_reserve, + "asset2_reserve": state.asset2_reserve, + "total_tokens": state.total_tokens, + "token_price_algo": state.token_price_algo, + "derived_observed_at": state.derived_observed_at, + "updated": datetime.now(UTC), + } + }, + return_document=ReturnDocument.AFTER, + ) + return self._from_state_document(document) + + def replace_snapshot( + self, + state: LpState, + *, + observed_round: int, + ) -> LpState: + """Replace balances only when the authoritative snapshot is newer.""" + + snapshot_order = lp_round_end_order(observed_round) + document = self.states.find_one_and_update( + { + "token_id": encode_bson_integer(state.token_id), + "$or": [ + {"last_event_order": {"$exists": False}}, + {"last_event_order": None}, + {"last_event_order": {"$lt": snapshot_order}}, + ], + }, + { + "$set": { + "asset1_reserve_micros": encode_bson_integer( + state.asset1_reserve_micros, + ), + "asset2_reserve_micros": encode_bson_integer( + state.asset2_reserve_micros, + ), + "total_tokens_micros": encode_bson_integer( + state.total_tokens_micros, + ), + "asset1_reserve": state.asset1_reserve, + "asset2_reserve": state.asset2_reserve, + "total_tokens": state.total_tokens, + "token_price_algo": state.token_price_algo, + "derived_observed_at": state.derived_observed_at, + "last_updated_round": encode_bson_integer( + observed_round, + ), + "last_event_order": snapshot_order, + "updated": datetime.now(UTC), + } + }, + return_document=ReturnDocument.AFTER, + ) + replaced = self._from_state_document(document) + if replaced is not None: + return replaced + + current = self._get_state(state.address) + if current.last_event_order == snapshot_order and self._balances(current) != self._balances(state): + raise LpProjectionPersistenceError( + f"LP snapshot {snapshot_order} conflicts with persisted balances for token {state.token_id}" + ) + return current + + def get_state(self, pool_address: str) -> LpState: + return self._get_state(pool_address) + + def _get_state(self, pool_address: str) -> LpState: + document = self.states.find_one({"address": pool_address}) + state = self._from_state_document(document) + if state is None: + raise LpProjectionPersistenceError(f"LP state not found for address {pool_address}") + return state + + def _apply_delta( + self, + *, + state: LpState, + order: str, + confirmed_round: int, + delta: LpBalanceDelta, + ) -> LpState | None: + bounds: dict[str, Any] = { + "$gte": encode_bson_integer(max(0, -delta.amount)), + } + if delta.amount > 0: + bounds["$lte"] = encode_bson_integer( + MAX_ALGORAND_UINT - delta.amount, + ) + document = self.states.find_one_and_update( + { + "token_id": encode_bson_integer(state.token_id), + "last_event_order": state.last_event_order, + delta.field: bounds, + }, + { + "$inc": { + delta.field: encode_bson_integer(delta.amount), + }, + "$set": { + "last_event_order": order, + "last_updated_round": encode_bson_integer( + confirmed_round, + ), + "updated": datetime.now(UTC), + }, + }, + return_document=ReturnDocument.AFTER, + ) + return self._from_state_document(document) + + def _record_event(self, transaction: LpTransaction) -> None: + payload = transaction.to_dict() + try: + document = self.events.find_one_and_update( + {"id": transaction.id}, + {"$setOnInsert": payload}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + except DuplicateKeyError: + document = self.events.find_one({"id": transaction.id}) + if document is None: + raise LpProjectionPersistenceError(f"failed to record LP event {transaction.id!r}") + self._assert_event_document( + document, + expected=transaction, + ) + + @staticmethod + def _assert_event_document( + document: dict[str, Any], + *, + expected: LpTransaction, + ) -> None: + persisted = dict(document) + persisted.pop("_id", None) + persisted_event = LpTransaction.from_dict(persisted) + if ( + persisted_event.pool_address, + persisted_event.asa_id, + persisted_event.delta_amount_micros, + persisted_event.confirmed_round, + persisted_event.event_position, + persisted_event.event_order, + ) != ( + expected.pool_address, + expected.asa_id, + expected.delta_amount_micros, + expected.confirmed_round, + expected.event_position, + expected.event_order, + ): + raise LpProjectionPersistenceError(f"LP event ID {expected.id!r} belongs to different immutable data") + + def _legacy_alias(self, transaction: LpTransaction) -> str | None: + aliases = event_id_aliases(transaction.id) + for alias in aliases: + if alias == transaction.id: + continue + if self.events.find_one({"id": alias}, projection={"_id": 1}) is not None: + return alias + return None + + @staticmethod + def _balances(state: LpState) -> tuple[int, int, int]: + return ( + state.asset1_reserve_micros, + state.asset2_reserve_micros, + state.total_tokens_micros, + ) + + @staticmethod + def _from_state_document(document: dict[str, Any] | None) -> LpState | None: + if document is None: + return None + payload = dict(document) + payload.pop("_id", None) + return cast( + LpState, + LpState.from_dict(payload), + ) diff --git a/flex/db/model/blockchain.py b/flex/db/model/blockchain.py index 640ed984..ceea33bb 100644 --- a/flex/db/model/blockchain.py +++ b/flex/db/model/blockchain.py @@ -1,12 +1,43 @@ from abc import ABC from dataclasses import dataclass, field from datetime import datetime +from decimal import Decimal from functools import cached_property +from typing import ClassVar -from dataclasses_json import dataclass_json +from dataclasses_json import config, dataclass_json +from flex.db.bson import ( + decode_bson_uint64, + decode_optional_bson_uint64, + encode_bson_integer, + encode_optional_bson_uint64, +) from flex.db.classes.base_entity import BaseEntity -from flex.db.util import get_uuid +from flex.db.classes.bson_uint64 import BsonUint64StorageMixin + +UINT64_MAX = (1 << 64) - 1 + + +class _MissingTotalSupply(int): + """Typed sentinel that cannot be confused with an explicit integer.""" + + +_MISSING_TOTAL_SUPPLY = _MissingTotalSupply(-1) + + +def _decode_total_supply(value: object) -> int: + if isinstance(value, _MissingTotalSupply): + return value + return int(value) + + +def _validate_uint64(value: int, *, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{field_name} must be an integer") + if not 0 <= value <= UINT64_MAX: + raise ValueError(f"{field_name} must be between 0 and {UINT64_MAX}") + return value @dataclass_json @@ -18,7 +49,7 @@ class TxInfo: @dataclass_json @dataclass -class PoolTransaction(BaseEntity['PoolTransaction']): +class PoolTransaction(BaseEntity["PoolTransaction"]): id: str pool_id: int pool_address: str @@ -47,13 +78,45 @@ class LpTokenInfo: @dataclass_json @dataclass -class LpToken(BaseEntity['LpToken']): - id: int - asset1_id: int - asset2_id: int # asset1_id > asset2_id +class LpToken( + BsonUint64StorageMixin, + BaseEntity["LpToken"], +): + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "id", + "asset1_id", + "asset2_id", + "pool_id", + } + ) + + id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + asset1_id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + asset2_id: int = field( # asset1_id > asset2_id + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) dex_provider: str address: str - pool_id: int + pool_id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) @@ -65,7 +128,7 @@ def to_info(self) -> LpTokenInfo: asset2_id=self.asset2_id, dex_provider=self.dex_provider, address=self.address, - pool_id=self.pool_id + pool_id=self.pool_id, ) @@ -77,13 +140,19 @@ class AssetBase(ABC): @cached_property def amount_multiplier(self) -> int: - return 10 ** self.decimals + return 10**self.decimals + + def amount_to_micros(self, amount: Decimal | float | int | str) -> int: + """Convert a display amount without introducing binary-float error.""" + return int(Decimal(str(amount)) * self.amount_multiplier) - def amount_to_micros(self, amount: float) -> int: - return int(amount * self.amount_multiplier) + def micros_to_decimal(self, micros: int) -> Decimal: + """Return an exact display-unit value for an on-chain base-unit amount.""" + return Decimal(micros) / self.amount_multiplier def micros_to_amount(self, micros: int) -> float: - return micros / self.amount_multiplier + """Compatibility conversion for API models that still expose floats.""" + return float(self.micros_to_decimal(micros)) @dataclass_json @@ -110,8 +179,19 @@ class AssetDetails(AssetBase): @dataclass_json @dataclass -class Asset(BaseEntity['Asset'], AssetBase): - id: int +class Asset( + BsonUint64StorageMixin, + BaseEntity["Asset"], + AssetBase, +): + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset({"id"}) + + id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) name: str decimals: int unit_name: str @@ -120,21 +200,38 @@ class Asset(BaseEntity['Asset'], AssetBase): total_supply: float logo_url: str | None = None + # ASA amounts occupy the full uint64 range, which does not fit BSON int64. + # Keep an int in memory and serialize it as a decimal string in MongoDB. + total_supply_micros: int = field( + default=_MISSING_TOTAL_SUPPLY, + metadata=config(encoder=str, decoder=_decode_total_supply), + ) created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) - @cached_property - def total_supply_micros(self) -> int: - return self.amount_to_micros(self.total_supply) + def __post_init__(self) -> None: + if isinstance(self.total_supply_micros, _MissingTotalSupply): + # Documents written before the canonical field was introduced only + # contain display units. This is necessarily best-effort because a + # historical float may already have lost precision. + self.total_supply_micros = self.amount_to_micros(self.total_supply) - def to_info(self) -> AssetInfo: - return AssetInfo( - name=self.name, - decimals=self.decimals, - unit_name=self.unit_name, - id=self.id + self.total_supply_micros = _validate_uint64( + self.total_supply_micros, + field_name="total_supply_micros", ) + # The persisted base-unit amount is authoritative. Keep the old float + # field as a presentation-only compatibility value. + self.total_supply = self.micros_to_amount(self.total_supply_micros) + + @property + def total_supply_base_units(self) -> int: + """Canonical on-chain supply; alias clarifies the historical name.""" + return self.total_supply_micros + + def to_info(self) -> AssetInfo: + return AssetInfo(name=self.name, decimals=self.decimals, unit_name=self.unit_name, id=self.id) def to_details(self) -> AssetDetails: return AssetDetails( @@ -145,29 +242,69 @@ def to_details(self) -> AssetDetails: creator=self.creator, reserve=self.reserve, logo_url=self.logo_url, - total_supply=self.total_supply + total_supply=self.total_supply, ) @dataclass_json @dataclass -class SyncState(BaseEntity['SyncState']): - id: str = field(default_factory=get_uuid) - last_round: int | None = None +class SyncState( + BsonUint64StorageMixin, + BaseEntity["SyncState"], +): + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "last_round", + "claimed_round", + } + ) + + id: str = "main" + last_round: int | None = field( + default=None, + metadata=config( + encoder=encode_optional_bson_uint64, + decoder=decode_optional_bson_uint64, + ), + ) + claimed_round: int | None = field( + default=None, + metadata=config( + encoder=encode_optional_bson_uint64, + decoder=decode_optional_bson_uint64, + ), + ) + lease_owner: str | None = None + lease_until: datetime | None = None + last_error: str | None = None created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) def rounds_since_updated(self, current_round: int) -> int | None: - return current_round - self.last_round if self.last_round else None + return current_round - self.last_round if self.last_round is not None else None @dataclass_json @dataclass -class SyncBlock(BaseEntity['SyncBlock']): - round: int +class SyncBlock( + BsonUint64StorageMixin, + BaseEntity["SyncBlock"], +): + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset({"round"}) + + round: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) timestamp: int - id: str = field(default_factory=get_uuid) + id: str | None = None created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) + + def __post_init__(self) -> None: + if self.id is None: + self.id = f"round:{self.round}" diff --git a/flex/db/model/liquidity_pools.py b/flex/db/model/liquidity_pools.py index 49cf81a0..d72b418c 100644 --- a/flex/db/model/liquidity_pools.py +++ b/flex/db/model/liquidity_pools.py @@ -1,9 +1,17 @@ from dataclasses import dataclass, field from datetime import datetime +from typing import ClassVar -from dataclasses_json import dataclass_json +from dataclasses_json import config, dataclass_json +from flex.db.bson import ( + decode_bson_delta, + decode_bson_uint64, + encode_bson_integer, +) from flex.db.classes.base_entity import BaseEntity +from flex.db.classes.bson_uint64 import BsonUint64StorageMixin +from flex.domain.lp_projection import lp_event_order @dataclass_json @@ -34,19 +42,75 @@ class LpStateInfo: @dataclass_json @dataclass -class LpState(BaseEntity["LpState"]): - id: int - token_id: int - asset1_id: int # asset1_id > asset2_id - asset2_id: int +class LpState( + BsonUint64StorageMixin, + BaseEntity["LpState"], +): + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "id", + "token_id", + "asset1_id", + "asset2_id", + "last_updated_round", + "asset1_reserve_micros", + "asset2_reserve_micros", + "total_tokens_micros", + } + ) + + id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + token_id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + asset1_id: int = field( # asset1_id > asset2_id + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + asset2_id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) dex_provider: str address: str - last_updated_round: int + last_updated_round: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) # TODO: could not fit into MongoDB ??? (64 bits) - asset1_reserve_micros: int - asset2_reserve_micros: int - total_tokens_micros: int # TODO: change name to issued_tokens_micros + asset1_reserve_micros: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + asset2_reserve_micros: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + total_tokens_micros: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) asset1_reserve: float asset2_reserve: float @@ -54,6 +118,8 @@ class LpState(BaseEntity["LpState"]): token_price_algo: float is_algo_pool: bool = False + last_event_order: str | None = None + derived_observed_at: datetime | None = None swap_fee_apr: float | None = None updated: datetime = field(default_factory=datetime.now) @@ -92,9 +158,40 @@ class LpTransaction(BaseEntity["LpTransaction"]): id: str pool_address: str user_address: str - asa_id: int - delta_amount_micros: int - confirmed_round: int - + asa_id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + delta_amount_micros: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_delta, + ) + ) + confirmed_round: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + + event_position: int = field( + default=0, + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ), + ) + event_order: str | None = None created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) + + def __post_init__(self) -> None: + if self.event_order is None: + self.event_order = lp_event_order( + self.confirmed_round, + self.id, + self.event_position, + ) diff --git a/flex/db/model/priced.py b/flex/db/model/priced.py index 318f9d0a..27d6c4c9 100644 --- a/flex/db/model/priced.py +++ b/flex/db/model/priced.py @@ -1,9 +1,17 @@ from dataclasses import dataclass, field from datetime import UTC, datetime +from typing import ClassVar -from dataclasses_json import dataclass_json +from dataclasses_json import config, dataclass_json +from flex.db.bson import ( + decode_bson_uint64, + decode_optional_bson_uint64, + encode_bson_integer, + encode_optional_bson_uint64, +) from flex.db.classes.base_entity import BaseEntity +from flex.db.classes.bson_uint64 import BsonUint64StorageMixin from flex.db.model.pool_states import PoolStateInfo from flex.db.model.pools import PoolInfo from flex.db.util import get_uuid @@ -52,7 +60,13 @@ class AirdropReward(BaseEntity["AirdropReward"]): txid: str operation_id: str | None = None - confirmed_round: int | None = None + confirmed_round: int | None = field( + default=None, + metadata=config( + encoder=encode_optional_bson_uint64, + decoder=decode_optional_bson_uint64, + ), + ) id: str = field(default_factory=get_uuid) created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) @@ -83,14 +97,41 @@ class AssetPriceInfo: @dataclass_json @dataclass -class AssetPrice(BaseEntity["AssetPrice"]): - id: int +class AssetPrice( + BsonUint64StorageMixin, + BaseEntity["AssetPrice"], +): + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "id", + "last_update_round", + "tinyman_algo_pool_id", + } + ) + + id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) price_usd: float price_algo: float - last_update_round: int + last_update_round: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) name: str - tinyman_algo_pool_id: int | None = None + tinyman_algo_pool_id: int | None = field( + default=None, + metadata=config( + encoder=encode_optional_bson_uint64, + decoder=decode_optional_bson_uint64, + ), + ) source: str | None = None observed_at: datetime | None = None diff --git a/flex/db/sync_coordinator.py b/flex/db/sync_coordinator.py new file mode 100644 index 00000000..3bfaa0bd --- /dev/null +++ b/flex/db/sync_coordinator.py @@ -0,0 +1,193 @@ +"""Mongo-backed single-writer ordering gate for the block projector.""" + +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any, cast + +from pymongo import ReturnDocument +from pymongo.collection import Collection + +from flex.db.bson import encode_bson_integer +from flex.db.model.blockchain import SyncState + +SYNC_STATE_ID = "main" + + +class SyncCoordinatorError(RuntimeError): + """Raised when a worker loses ownership of a claimed round.""" + + +@dataclass(slots=True) +class MongoSyncCoordinator: + collection: Collection[dict[str, Any]] + lease_duration: timedelta = timedelta(minutes=2) + + def claim_round( + self, + *, + owner: str, + expected_last_round: int | None, + round_number: int, + now: datetime | None = None, + ) -> SyncState | None: + self._validate_next_round( + expected_last_round=expected_last_round, + round_number=round_number, + ) + current_time = now or datetime.now(UTC) + encoded_expected_round = None if expected_last_round is None else encode_bson_integer(expected_last_round) + document = self.collection.find_one_and_update( + { + "id": SYNC_STATE_ID, + "last_round": encoded_expected_round, + "$or": [ + {"lease_until": {"$exists": False}}, + {"lease_until": None}, + {"lease_until": {"$lte": current_time}}, + {"lease_owner": owner}, + ], + }, + { + "$set": { + "claimed_round": encode_bson_integer(round_number), + "lease_owner": owner, + "lease_until": current_time + self.lease_duration, + "last_error": None, + "updated": current_time, + } + }, + return_document=ReturnDocument.AFTER, + ) + return self._from_document(document) + + def complete_round( + self, + *, + owner: str, + expected_last_round: int | None, + round_number: int, + now: datetime | None = None, + ) -> SyncState: + self._validate_next_round( + expected_last_round=expected_last_round, + round_number=round_number, + ) + current_time = now or datetime.now(UTC) + encoded_expected_round = None if expected_last_round is None else encode_bson_integer(expected_last_round) + document = self.collection.find_one_and_update( + { + "id": SYNC_STATE_ID, + "last_round": encoded_expected_round, + "claimed_round": encode_bson_integer(round_number), + "lease_owner": owner, + }, + { + "$set": { + "last_round": encode_bson_integer(round_number), + "claimed_round": None, + "lease_owner": None, + "lease_until": None, + "last_error": None, + "updated": current_time, + } + }, + return_document=ReturnDocument.AFTER, + ) + state = self._from_document(document) + if state is None: + raise SyncCoordinatorError(f"worker {owner!r} lost its claim for round {round_number}") + return state + + def advance_snapshot( + self, + *, + expected_last_round: int | None, + round_number: int, + now: datetime | None = None, + ) -> SyncState: + if ( + isinstance(round_number, bool) + or not isinstance(round_number, int) + or round_number < 0 + or (expected_last_round is not None and round_number < expected_last_round) + ): + raise SyncCoordinatorError("snapshot checkpoint cannot regress") + current_time = now or datetime.now(UTC) + encoded_expected_round = None if expected_last_round is None else encode_bson_integer(expected_last_round) + document = self.collection.find_one_and_update( + { + "id": SYNC_STATE_ID, + "last_round": encoded_expected_round, + "$or": [ + {"lease_until": {"$exists": False}}, + {"lease_until": None}, + {"lease_until": {"$lte": current_time}}, + ], + }, + { + "$set": { + "last_round": encode_bson_integer(round_number), + "claimed_round": None, + "lease_owner": None, + "lease_until": None, + "last_error": None, + "updated": current_time, + } + }, + return_document=ReturnDocument.AFTER, + ) + state = self._from_document(document) + if state is None: + raise SyncCoordinatorError("sync checkpoint changed during authoritative snapshot") + return state + + def release_after_error( + self, + *, + owner: str, + round_number: int, + error: str, + now: datetime | None = None, + ) -> None: + current_time = now or datetime.now(UTC) + self.collection.update_one( + { + "id": SYNC_STATE_ID, + "claimed_round": encode_bson_integer(round_number), + "lease_owner": owner, + }, + { + "$set": { + "claimed_round": None, + "lease_owner": None, + "lease_until": None, + "last_error": error[:500], + "updated": current_time, + } + }, + ) + + @staticmethod + def _from_document(document: dict[str, Any] | None) -> SyncState | None: + if document is None: + return None + payload = dict(document) + payload.pop("_id", None) + return cast( + SyncState, + SyncState.from_dict(payload), + ) + + @staticmethod + def _validate_next_round( + *, + expected_last_round: int | None, + round_number: int, + ) -> None: + if ( + isinstance(round_number, bool) + or not isinstance(round_number, int) + or round_number < 0 + or (expected_last_round is not None and round_number != expected_last_round + 1) + ): + raise SyncCoordinatorError("projected rounds must advance exactly once") diff --git a/flex/domain/lp_projection.py b/flex/domain/lp_projection.py new file mode 100644 index 00000000..7f1be615 --- /dev/null +++ b/flex/domain/lp_projection.py @@ -0,0 +1,119 @@ +"""Pure invariants for ordered liquidity-pool projections.""" + +from dataclasses import dataclass +from typing import Literal + +MAX_ALGORAND_UINT = 2**64 - 1 +EVENT_ROUND_WIDTH = 20 +ROUND_END_SUFFIX = "~" + +type LpBalanceField = Literal[ + "asset1_reserve_micros", + "asset2_reserve_micros", + "total_tokens_micros", +] + + +class InvalidLpProjectionError(ValueError): + """Raised when an LP event cannot safely update its target aggregate.""" + + +@dataclass(frozen=True, slots=True) +class LpBalanceDelta: + field: LpBalanceField + amount: int + + +def lp_event_order( + confirmed_round: int, + event_id: str, + event_position: int = 0, +) -> str: + """Build a lexicographically sortable, deterministic event cursor.""" + + if ( + isinstance(confirmed_round, bool) + or not isinstance(confirmed_round, int) + or not 0 <= confirmed_round <= MAX_ALGORAND_UINT + ): + raise InvalidLpProjectionError("confirmed_round must be an Algorand uint64") + if not isinstance(event_id, str) or not event_id or ":" in event_id: + raise InvalidLpProjectionError("event_id must be non-empty and cannot contain ':'") + if ( + isinstance(event_position, bool) + or not isinstance(event_position, int) + or not 0 <= event_position <= MAX_ALGORAND_UINT + ): + raise InvalidLpProjectionError("event_position must be an Algorand uint64") + return f"{confirmed_round:0{EVENT_ROUND_WIDTH}d}:{event_position:0{EVENT_ROUND_WIDTH}d}:{event_id}" + + +def lp_round_end_order(confirmed_round: int) -> str: + """Represent an authoritative snapshot taken after an entire round.""" + + prefix = lp_event_order(confirmed_round, "snapshot").partition(":")[0] + return f"{prefix}:{ROUND_END_SUFFIX}" + + +def snapshot_covers_event(cursor: str, *, confirmed_round: int) -> bool: + """Return whether a round-end snapshot is authoritative for an event.""" + + round_prefix, separator, suffix = cursor.partition(":") + return ( + separator == ":" + and suffix == ROUND_END_SUFFIX + and len(round_prefix) == EVENT_ROUND_WIDTH + and round_prefix.isdigit() + and int(round_prefix) >= confirmed_round + ) + + +def lp_cursor_complete_through(cursor: str | None) -> int: + """Return the last round fully covered by a persisted LP cursor.""" + + if cursor is None: + raise InvalidLpProjectionError("LP cursor is missing") + round_prefix, separator, suffix = cursor.partition(":") + if separator != ":" or len(round_prefix) != EVENT_ROUND_WIDTH or not round_prefix.isdigit() or not suffix: + raise InvalidLpProjectionError("LP cursor has an invalid format") + round_number = int(round_prefix) + if suffix == ROUND_END_SUFFIX: + return round_number + if round_number == 0: + raise InvalidLpProjectionError("an event cursor in round zero covers no complete round") + return round_number - 1 + + +def lp_balance_delta( + *, + token_id: int, + asset1_id: int, + asset2_id: int, + event_asset_id: int, + event_pool_delta_micros: int, +) -> LpBalanceDelta: + """Map one pool-account transfer to its canonical LP balance field.""" + + if isinstance(event_pool_delta_micros, bool) or not isinstance(event_pool_delta_micros, int): + raise InvalidLpProjectionError("event delta must be an integer") + if not -MAX_ALGORAND_UINT <= event_pool_delta_micros <= MAX_ALGORAND_UINT: + raise InvalidLpProjectionError("event delta is outside the Algorand uint64 range") + + if event_asset_id == token_id: + return LpBalanceDelta( + field="total_tokens_micros", + amount=-event_pool_delta_micros, + ) + if event_asset_id == asset1_id: + return LpBalanceDelta( + field="asset1_reserve_micros", + amount=event_pool_delta_micros, + ) + if event_asset_id == asset2_id: + return LpBalanceDelta( + field="asset2_reserve_micros", + amount=event_pool_delta_micros, + ) + raise InvalidLpProjectionError( + f"asset {event_asset_id} does not belong to LP token {token_id}", + ) diff --git a/flex/domain/pricing.py b/flex/domain/pricing.py index e87c20ea..02c87e56 100644 --- a/flex/domain/pricing.py +++ b/flex/domain/pricing.py @@ -73,6 +73,27 @@ def _legacy_float(value: Decimal, *, field: str) -> float: return converted +def base_units_to_decimal( + amount_micros: int, + *, + decimals: int, + field: str = "amount_micros", +) -> Decimal: + """Convert integer base units without crossing a float boundary.""" + + amount = _non_negative_int(amount_micros, field=field) + precision = _asset_decimals(decimals, field=f"{field}_decimals") + with localcontext() as context: + context.prec = CALCULATION_PRECISION + return Decimal(amount) / Decimal(10**precision) + + +def decimal_to_legacy_float(value: DecimalInput, *, field: str) -> float: + """Convert an exact value only at a legacy float API/storage boundary.""" + + return _legacy_float(_decimal(value, field=field), field=field) + + @dataclass(frozen=True, slots=True) class PriceQuote: """A validated provider observation before legacy float persistence.""" @@ -195,13 +216,62 @@ def calculate_lp_token_price_algo( if pool_balance_micros >= total_supply_micros: raise InvalidLiquidityPoolError("circulating LP supply must be positive") + return calculate_lp_token_price_from_issued_supply( + asset1_price_algo=price, + asset1_reserve_micros=reserve_micros, + asset1_decimals=asset_decimals, + issued_lp_supply_micros=total_supply_micros - pool_balance_micros, + lp_token_decimals=lp_decimals, + ) + + +def calculate_lp_token_price_from_issued_supply( + *, + asset1_price_algo: DecimalInput, + asset1_reserve_micros: int, + asset1_decimals: int, + issued_lp_supply_micros: int, + lp_token_decimals: int, +) -> Decimal: + """Calculate LP price from an already-derived issued token supply.""" + + price = _decimal(asset1_price_algo, field="asset1_price_algo") + if price <= 0: + raise InvalidLiquidityPoolError("asset1_price_algo must be positive") + reserve_micros = _non_negative_int( + asset1_reserve_micros, + field="asset1_reserve_micros", + ) + issued_micros = _non_negative_int( + issued_lp_supply_micros, + field="issued_lp_supply_micros", + ) + asset_decimals = _asset_decimals( + asset1_decimals, + field="asset1_decimals", + ) + lp_decimals = _asset_decimals( + lp_token_decimals, + field="lp_token_decimals", + ) + if reserve_micros == 0: + raise InvalidLiquidityPoolError("asset1 reserve must be positive") + if issued_micros == 0: + raise InvalidLiquidityPoolError("issued LP supply must be positive") + with localcontext() as context: context.prec = CALCULATION_PRECISION - reserve = Decimal(reserve_micros) / Decimal(10**asset_decimals) - circulating_supply = Decimal( - total_supply_micros - pool_balance_micros, - ) / Decimal(10**lp_decimals) - calculated = price * reserve * Decimal(2) / circulating_supply + reserve = base_units_to_decimal( + reserve_micros, + decimals=asset_decimals, + field="asset1_reserve_micros", + ) + issued_supply = base_units_to_decimal( + issued_micros, + decimals=lp_decimals, + field="issued_lp_supply_micros", + ) + calculated = price * reserve * Decimal(2) / issued_supply if not calculated.is_finite() or calculated <= 0: raise InvalidLiquidityPoolError("calculated LP price is invalid") with localcontext() as context: diff --git a/flex/migrations/fix_dex_providers.py b/flex/migrations/fix_dex_providers.py index ef3253d6..84520afe 100644 --- a/flex/migrations/fix_dex_providers.py +++ b/flex/migrations/fix_dex_providers.py @@ -1,34 +1,36 @@ import logging from flex import db -from flex.providers.vestige import is_valid_dex_provider, get_dex_tag_by_name - +from flex.providers.vestige import get_dex_tag_by_name, is_valid_dex_provider logger = logging.getLogger(__name__) def fix_dex_names() -> None: - logger.info('Fixing dex names.') + logger.info("Fixing dex names.") lp_tokens = db.lp_tokens.get_all() for lp_token in lp_tokens: if not is_valid_dex_provider(lp_token.dex_provider): - logger.info(f'Invalid DEX in LP token:\n{lp_token.pretty_str()}') + logger.info(f"Invalid DEX in LP token:\n{lp_token.pretty_str()}") lp_token.dex_provider = get_dex_tag_by_name(lp_token.dex_provider) db.lp_tokens.update(lp_token) farming_pools = db.farming_pools.get_all() for farming_pool in farming_pools: if not is_valid_dex_provider(farming_pool.dex_name): - logger.info(f'Invalid DEX in farming pool:\n{farming_pool.pretty_str()}') + logger.info(f"Invalid DEX in farming pool:\n{farming_pool.pretty_str()}") farming_pool.dex_name = get_dex_tag_by_name(farming_pool.dex_name) db.farming_pools.update(farming_pool) lp_states = db.lp_states.get_all() for lp_state in lp_states: if not is_valid_dex_provider(lp_state.dex_provider): - logger.info(f'Invalid DEX in LP state:\n{lp_state.pretty_str()}') + logger.info(f"Invalid DEX in LP state:\n{lp_state.pretty_str()}") lp_state.dex_provider = get_dex_tag_by_name(lp_state.dex_provider) - db.lp_states.update(lp_state) + db.lp_states.update_with( + lp_state, + dex_provider=lp_state.dex_provider, + ) - logger.info('All DEX names are valid now!') + logger.info("All DEX names are valid now!") diff --git a/flex/sync_pools.py b/flex/sync_pools.py index 6444310b..6ee41c95 100644 --- a/flex/sync_pools.py +++ b/flex/sync_pools.py @@ -1,8 +1,9 @@ import asyncio import logging +import random +from uuid import uuid4 from aiocache import cached as cached_async -from cachetools import TTLCache, cached from env import settings from flex import db @@ -17,15 +18,19 @@ ) from flex.data.pool_state import ( get_or_create_pool_state, - update_all_pool_states_linear, update_pool_state, update_pool_states_with_transactions, ) -from flex.data.tinyman_lps import update_tinyman_algo_lp_state_and_prices +from flex.data.tinyman_lps import update_tinyman_algo_asset_price from flex.db.model.blockchain import PoolTransaction, SyncBlock, SyncState from flex.db.model.liquidity_pools import LpState, LpTransaction from flex.db.model.pool_states import PoolState, UserState from flex.db.model.priced import AssetPrice +from flex.db.sync_coordinator import ( + MongoSyncCoordinator, + SyncCoordinatorError, +) +from flex.domain.lp_projection import lp_cursor_complete_through from flex.domain.pricing import PricingError from flex.domain.transactions import ( ASSET_TRANSFER_TX, @@ -41,7 +46,6 @@ logger = logging.getLogger(__name__) -@cached(cache=TTLCache(maxsize=1, ttl=30)) def get_all_lp_state_addresses() -> set[str]: return set(lp_state.address for lp_state in db.lp_states.get_all()) @@ -63,22 +67,49 @@ async def process_lp_transactions(transactions: list[dict]) -> list[LpTransactio all_lp_addresses = get_all_lp_state_addresses() lp_transactions = [] - for tx in transactions: + for event_position, tx in enumerate(transactions): txid = tx["id"] sender = tx["sender"] confirmed_round = tx["confirmed-round"] if ASSET_TRANSFER_TX in tx: - asa_id = tx[ASSET_TRANSFER_TX]["asset-id"] - receiver = tx[ASSET_TRANSFER_TX]["receiver"] - amount = tx[ASSET_TRANSFER_TX]["amount"] + transfer = tx[ASSET_TRANSFER_TX] + receiver = transfer["receiver"] + affected_addresses = { + sender, + receiver, + transfer.get("sender"), + transfer.get("close-to"), + } + if not (affected_addresses & all_lp_addresses): + continue + if any(field in transfer for field in ("sender", "close-to", "close-amount")): + raise ValueError(f"LP projection does not support clawback/close semantics: {txid}") + asa_id = transfer["asset-id"] + amount = transfer["amount"] elif PAYMENT_TX in tx: + payment = tx[PAYMENT_TX] + receiver = payment["receiver"] + affected_addresses = { + sender, + receiver, + payment.get("close-remainder-to"), + } + if not (affected_addresses & all_lp_addresses): + continue + if any(field in payment for field in ("close-remainder-to", "close-amount")): + raise ValueError(f"LP projection does not support payment close semantics: {txid}") asa_id = 0 - receiver = tx[PAYMENT_TX]["receiver"] - amount = tx[PAYMENT_TX]["amount"] + amount = payment["amount"] else: raise ValueError(f"Invalid transaction type: {tx}") + if sender == receiver and sender in all_lp_addresses: + # A self-transfer has zero net effect on the pool. Emitting both + # legs would create the same scoped event ID twice and make a + # deduplicating projector persist only one side. + continue + if sender in all_lp_addresses: lp_tx = LpTransaction( id=projection_event_id(txid, sender), @@ -87,6 +118,7 @@ async def process_lp_transactions(transactions: list[dict]) -> list[LpTransactio asa_id=asa_id, delta_amount_micros=-amount, confirmed_round=confirmed_round, + event_position=event_position, ) lp_transactions.append(lp_tx) @@ -98,6 +130,7 @@ async def process_lp_transactions(transactions: list[dict]) -> list[LpTransactio asa_id=asa_id, delta_amount_micros=amount, confirmed_round=confirmed_round, + event_position=event_position, ) lp_transactions.append(lp_tx) @@ -151,9 +184,16 @@ async def update_pools(txns: list[dict]) -> [PoolState]: return await update_pool_states_with_transactions(pool_transactions) -async def update_lp_states(txns: list[dict]) -> list[LpState]: +async def update_lp_states( + txns: list[dict], + *, + expected_round: int, +) -> list[LpState]: lp_transactions = await process_lp_transactions(txns) - return await update_lp_states_with_transactions(lp_transactions) + return await update_lp_states_with_transactions( + lp_transactions, + expected_round=expected_round, + ) async def update_asset_prices(updated_lp_states: list[LpState]) -> list[AssetPrice]: @@ -165,7 +205,7 @@ async def update_asset_prices(updated_lp_states: list[LpState]) -> list[AssetPri for lp_state in updated_lp_states: if lp_state.is_algo_pool: try: - updated_asset_price = await update_tinyman_algo_lp_state_and_prices( + updated_asset_price = await update_tinyman_algo_asset_price( lp_state, algo_quote, ) @@ -182,6 +222,28 @@ async def update_asset_prices(updated_lp_states: list[LpState]) -> list[AssetPri return updated_asset_prices +def _snapshot_checkpoint_round( + *, + previous_round: int | None, + node_round: int, + lp_states: list[LpState], +) -> int: + """Choose a checkpoint covered by every persisted LP snapshot.""" + + checkpoint = min( + [ + node_round, + *(lp_cursor_complete_through(state.last_event_order) for state in lp_states), + ], + ) + if previous_round is not None and checkpoint < previous_round: + raise SyncCoordinatorError( + "Indexer snapshots lag behind the persisted sync checkpoint; " + "wait for Indexer catch-up before resuming projection" + ) + return checkpoint + + async def catch_up_the_sync_manually(sync_state: SyncState, current_round: int) -> SyncState: logger.info("\n\nMANUAL roll rock and roll BABE.\n") logger.info( @@ -189,41 +251,82 @@ async def catch_up_the_sync_manually(sync_state: SyncState, current_round: int) ) _ = await load_all_assets_data() - _ = await update_all_pool_states_linear(reset_pool_states=True) + if settings.sync_staking_pools: + raise RuntimeError( + "legacy staking-pool projection is disabled until full Algorand group validation is implemented" + ) current_round = await get_current_round() logger.info(f"\n\nAnother, shorter loop, starting from round {current_round}\n") - await update_all_pool_states_linear() - - logger.info("\n\nSyncing LP states linearly.\n") - _ = await create_lp_states_from_all_pools() - _ = await update_all_lp_states_linear() - _ = await create_and_update_asset_prices() + if settings.sync_liquidity_pools: + logger.info("\n\nSyncing LP states from authoritative account snapshots.\n") + _ = await create_lp_states_from_all_pools() + snapshotted_states = await update_all_lp_states_linear() + _ = await create_and_update_asset_prices() + else: + snapshotted_states = [] + + checkpoint_round = _snapshot_checkpoint_round( + previous_round=sync_state.last_round, + node_round=current_round, + lp_states=snapshotted_states, + ) - sync_state.last_round = current_round - db.sync_states.update(sync_state) - logger.info(f"\n\nALL synced up to round {current_round}.\n") + coordinator = MongoSyncCoordinator( + db.sync_states.mongodb_collection, + ) + sync_state = await asyncio.to_thread( + coordinator.advance_snapshot, + expected_last_round=sync_state.last_round, + round_number=checkpoint_round, + ) + logger.info(f"\n\nALL synced up to round {checkpoint_round}.\n") return sync_state async def sync_pools_loop(): + if settings.sync_staking_pools: + raise RuntimeError( + "legacy staking-pool projection is disabled until full Algorand group validation is implemented" + ) + if not settings.sync_liquidity_pools: + logger.warning( + "Financial sync is disabled; set SYNC_LIQUIDITY_POOLS=true only after reviewing the snapshot cutover" + ) + return + current_round = await get_current_round() logger.info(f"\n\nEnter sync loop. Current round = {current_round}") sync_state = await get_sync_state() - if sync_state.last_round is None or sync_state.rounds_since_updated(current_round) > settings.sync_lag_max_rounds: + previous_round = sync_state.last_round + try: sync_state = await catch_up_the_sync_manually(sync_state, current_round) + except SyncCoordinatorError: + # A competing worker may have completed the same authoritative cutover. + refreshed = await get_sync_state() + if refreshed.last_round == previous_round: + raise + sync_state = refreshed logger.info("\n\nMain BLOCKCHAIN sync loop!\n") + coordinator = MongoSyncCoordinator( + db.sync_states.mongodb_collection, + ) + worker_id = uuid4().hex no_block_seconds = 0 + processing_attempts = 0 MAX_BLOCK_DELAY_SECONDS = 10 while True: next_round = sync_state.last_round + 1 try: - block_dict = indexer_client.block_info(round_num=next_round) + block_dict = await asyncio.to_thread( + indexer_client.block_info, + round_num=next_round, + ) except Exception as e: no_block_seconds += 1 if no_block_seconds > MAX_BLOCK_DELAY_SECONDS: @@ -234,21 +337,68 @@ async def sync_pools_loop(): no_block_seconds = 0 + claimed = await asyncio.to_thread( + coordinator.claim_round, + owner=worker_id, + expected_last_round=sync_state.last_round, + round_number=next_round, + ) + if claimed is None: + await asyncio.sleep(0.25) + sync_state = await get_sync_state() + continue + try: raw_transactions = block_dict["transactions"] logger.debug(f"Fetch #{next_round}: sync {len(raw_transactions)} txns") - updated_pool_states = await update_pools(find_transfer_transactions(raw_transactions)) - updated_lp_states = await update_lp_states(find_transfer_payment_transactions(raw_transactions)) + if settings.sync_staking_pools: + raise RuntimeError( + "legacy staking-pool projection is disabled until full Algorand group validation is implemented" + ) + updated_pool_states: list[PoolState] = [] + updated_lp_states = ( + await update_lp_states( + find_transfer_payment_transactions(raw_transactions), + expected_round=next_round, + ) + if settings.sync_liquidity_pools + else [] + ) _ = await update_asset_prices(updated_lp_states) - sync_state.last_round = next_round - db.sync_states.update(sync_state) - db.sync_blocks.create(SyncBlock(round=next_round, timestamp=block_dict["timestamp"])) + sync_state = await asyncio.to_thread( + coordinator.complete_round, + owner=worker_id, + expected_last_round=sync_state.last_round, + round_number=next_round, + ) + db.sync_blocks.get_or_create( + SyncBlock( + round=next_round, + timestamp=block_dict["timestamp"], + ) + ) + processing_attempts = 0 logger.debug(f"#{next_round} sync OK! Saved {len(updated_pool_states) + len(updated_lp_states)} txns\n") except Exception as e: + processing_attempts += 1 + await asyncio.to_thread( + coordinator.release_after_error, + owner=worker_id, + round_number=next_round, + error=f"{type(e).__name__}: {e}", + ) logger.error(f"Error processing round {next_round}: {e}", exc_info=True) + if processing_attempts >= settings.sync_round_max_attempts: + raise RuntimeError(f"round {next_round} failed {processing_attempts} consecutive times") from e + delay_cap = min( + 2 ** (processing_attempts - 1), + settings.sync_retry_max_seconds, + ) + await asyncio.sleep(random.uniform(0, delay_cap)) + sync_state = await get_sync_state() continue diff --git a/flex/sync_state.py b/flex/sync_state.py index 14ff5ea6..890d345f 100644 --- a/flex/sync_state.py +++ b/flex/sync_state.py @@ -7,10 +7,7 @@ async def get_sync_state() -> SyncState: - sync_state = db.sync_states.get_one() - if sync_state is None: - sync_state = db.sync_states.create(SyncState()) - return sync_state + return db.sync_states.get_or_create(SyncState()) sync_max_delay = timedelta(seconds=settings.sync_behind_seconds_threshold) @@ -20,4 +17,5 @@ async def get_sync_state() -> SyncState: async def is_sync_delayed(current_round: int | None = None) -> bool: current_round = current_round or (await get_current_round()) sync_state = await get_sync_state() - return sync_state.rounds_since_updated(current_round) > sync_max_delay_rounds + rounds_behind = sync_state.rounds_since_updated(current_round) + return rounds_behind is None or rounds_behind > sync_max_delay_rounds diff --git a/pyproject.toml b/pyproject.toml index c61bb702..f195d75a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,10 @@ files = [ "core/circuit_breaker.py", "flex/blockchain/contract_state.py", "flex/application/asset_transfers.py", + "flex/db/lp_projection.py", + "flex/db/sync_coordinator.py", "flex/domain/allocation.py", + "flex/domain/lp_projection.py", "flex/domain/pricing.py", "flex/domain/transactions.py", "flex/providers/pact.py", diff --git a/tests/unit/test_asset_supply.py b/tests/unit/test_asset_supply.py new file mode 100644 index 00000000..f98ebe7b --- /dev/null +++ b/tests/unit/test_asset_supply.py @@ -0,0 +1,94 @@ +from collections.abc import Callable +from typing import Any + +import pytest + +from flex.blockchain import info +from flex.db.model.blockchain import UINT64_MAX, Asset + + +def _asset_response(*, total: int, decimals: int) -> dict[str, Any]: + return { + "asset": { + "params": { + "creator": "CREATOR", + "decimals": decimals, + "name": "Precision Asset", + "reserve": "RESERVE", + "total": total, + "unit-name": "PRECISE", + } + } + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("total", [2**53 + 1, UINT64_MAX]) +async def test_fetch_asset_preserves_full_uint64_supply( + monkeypatch: pytest.MonkeyPatch, + total: int, +) -> None: + async def fake_run_sync(func: Callable[..., Any], *args: Any) -> dict[str, Any]: + del func, args + return _asset_response(total=total, decimals=0) + + monkeypatch.setattr(info, "_run_sync", fake_run_sync) + + asset = await info.fetch_asset(42) + + assert asset.total_supply_micros == total + assert asset.total_supply_base_units == total + + stored = asset.to_dict() + assert stored["total_supply_micros"] == str(total) + assert Asset.from_dict(stored).total_supply_micros == total + + +def test_legacy_asset_document_backfills_canonical_supply() -> None: + legacy_document = { + "id": 42, + "name": "Legacy Asset", + "decimals": 6, + "unit_name": "OLD", + "creator": "CREATOR", + "reserve": "RESERVE", + "total_supply": 123.456789, + } + + asset = Asset.from_dict(legacy_document) + + assert asset.total_supply_micros == 123_456_789 + assert asset.to_dict()["total_supply_micros"] == "123456789" + + +def test_canonical_supply_wins_over_lossy_legacy_float() -> None: + canonical_supply = 2**53 + 1 + + asset = Asset( + id=42, + name="Canonical Asset", + decimals=0, + unit_name="CANON", + creator="CREATOR", + reserve="RESERVE", + total_supply=float(canonical_supply), + total_supply_micros=canonical_supply, + ) + + assert int(asset.total_supply) != canonical_supply + assert asset.total_supply_micros == canonical_supply + + +@pytest.mark.parametrize("invalid_supply", [-1, UINT64_MAX + 1]) +def test_asset_rejects_supply_outside_algorand_uint64(invalid_supply: int) -> None: + with pytest.raises(ValueError, match="total_supply_micros"): + Asset( + id=42, + name="Invalid Asset", + decimals=0, + unit_name="BAD", + creator="CREATOR", + reserve="RESERVE", + total_supply=0, + total_supply_micros=invalid_supply, + ) diff --git a/tests/unit/test_bson_uint64_models.py b/tests/unit/test_bson_uint64_models.py new file mode 100644 index 00000000..8c8284f1 --- /dev/null +++ b/tests/unit/test_bson_uint64_models.py @@ -0,0 +1,78 @@ +from datetime import UTC, datetime + +from bson import BSON, Decimal128 + +from flex.db.model.blockchain import Asset, LpToken +from flex.db.model.priced import AssetPrice + +UINT64_MAX = 2**64 - 1 + + +def test_lp_token_round_trips_full_uint64_identifiers() -> None: + token = LpToken( + id=UINT64_MAX, + asset1_id=UINT64_MAX - 1, + asset2_id=0, + dex_provider="tinyman", + address="POOL", + pool_id=UINT64_MAX, + ) + + restored = LpToken.from_dict( + BSON(BSON.encode(token.to_dict())).decode(), + ) + + assert restored.id == UINT64_MAX + assert restored.asset1_id == UINT64_MAX - 1 + assert restored.pool_id == UINT64_MAX + assert LpToken.encode_query({"id": UINT64_MAX}) == { + "id": Decimal128(str(UINT64_MAX)), + } + + +def test_asset_round_trips_full_uint64_identifier_and_supply() -> None: + asset = Asset( + id=UINT64_MAX, + name="MAX", + decimals=0, + unit_name="MAX", + creator="CREATOR", + reserve="RESERVE", + total_supply=0, + total_supply_micros=UINT64_MAX, + ) + + restored = Asset.from_dict( + BSON(BSON.encode(asset.to_dict())).decode(), + ) + + assert restored.id == UINT64_MAX + assert restored.total_supply_micros == UINT64_MAX + assert Asset.encode_query({"id": UINT64_MAX}) == { + "id": Decimal128(str(UINT64_MAX)), + } + + +def test_asset_price_round_trips_full_uint64_identifiers_and_round() -> None: + observed_at = datetime(2026, 1, 1, tzinfo=UTC) + price = AssetPrice( + id=UINT64_MAX, + price_usd=1, + price_algo=2, + last_update_round=UINT64_MAX, + name="MAX", + tinyman_algo_pool_id=UINT64_MAX, + source="tinyman", + observed_at=observed_at, + ) + + restored = AssetPrice.from_dict( + BSON(BSON.encode(price.to_dict())).decode(), + ) + + assert restored.id == UINT64_MAX + assert restored.last_update_round == UINT64_MAX + assert restored.tinyman_algo_pool_id == UINT64_MAX + assert AssetPrice.encode_query({"id": UINT64_MAX}) == { + "id": Decimal128(str(UINT64_MAX)), + } diff --git a/tests/unit/test_database_indexes.py b/tests/unit/test_database_indexes.py index 20f15c74..90b39af6 100644 --- a/tests/unit/test_database_indexes.py +++ b/tests/unit/test_database_indexes.py @@ -7,6 +7,7 @@ create_unique_id_index_fail_closed, deduplicate_and_create_unique_id_index, ensure_database_indexes, + ensure_sync_state_singleton, ) @@ -26,6 +27,7 @@ def _database(**collections: Mock) -> SimpleNamespace: "user_states", "lp_tokens", "airdrop_rewards", + "sync_states", ) return SimpleNamespace(**{name: _manager(collections.get(name, Mock())) for name in names}) @@ -100,8 +102,16 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: } for collection in unique_collections.values(): collection.aggregate.return_value = [] - - database = _database(**unique_collections) + lp_states = Mock() + lp_states.aggregate.return_value = [] + sync_states = Mock() + sync_states.find.return_value = [] + + database = _database( + **unique_collections, + lp_states=lp_states, + sync_states=sync_states, + ) removed = ensure_database_indexes(database) @@ -115,7 +125,10 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: for collection in unique_collections.values(): collection.create_index.assert_called_once_with("id", unique=True, name="id_unique") - database.lp_states.mongodb_collection.create_index.assert_called_once_with("token_id", name="token_id_idx") + assert database.lp_states.mongodb_collection.create_index.call_args_list == [ + call("token_id", unique=True, name="token_id_unique"), + call("address", unique=True, name="address_unique"), + ] database.pool_states.mongodb_collection.create_index.assert_called_once_with("pool_id", name="pool_id_idx") database.user_states.mongodb_collection.create_index.assert_called_once_with("address", name="address_idx") database.lp_tokens.mongodb_collection.create_index.assert_called_once_with("id", name="lp_token_id_idx") @@ -125,6 +138,51 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: name="operation_id_unique", partialFilterExpression={"operation_id": {"$type": "string"}}, ) + database.sync_states.mongodb_collection.create_index.assert_called_once_with( + "id", + unique=True, + name="id_unique", + ) + + +def test_single_legacy_sync_cursor_is_migrated_without_guessing_between_competitors() -> None: + collection = Mock() + collection.find.return_value = [ + { + "_id": "mongo-id", + "id": "legacy-random-id", + "last_round": 123, + } + ] + database = _database(sync_states=collection) + + ensure_sync_state_singleton(database) + + collection.update_one.assert_called_once_with( + {"_id": "mongo-id"}, + {"$set": {"id": "main"}}, + ) + collection.create_index.assert_called_once_with( + "id", + unique=True, + name="id_unique", + ) + + +def test_competing_sync_cursors_fail_closed() -> None: + collection = Mock() + collection.find.return_value = [ + {"_id": "a", "id": "a", "last_round": 100}, + {"_id": "b", "id": "b", "last_round": 101}, + ] + + with pytest.raises(RuntimeError, match="competing checkpoints"): + ensure_sync_state_singleton( + _database(sync_states=collection), + ) + + collection.update_one.assert_not_called() + collection.create_index.assert_not_called() def test_correctness_critical_index_failure_is_not_swallowed() -> None: diff --git a/tests/unit/test_lp_projection_repository.py b/tests/unit/test_lp_projection_repository.py new file mode 100644 index 00000000..2406a35a --- /dev/null +++ b/tests/unit/test_lp_projection_repository.py @@ -0,0 +1,519 @@ +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from datetime import UTC, datetime, timedelta +from threading import RLock +from types import SimpleNamespace + +import pytest +from bson import BSON, Decimal128 + +from flex.db.lp_projection import ( + LpProjectionPersistenceError, + LpProjectionResult, + MongoLpProjectionRepository, +) +from flex.db.model.blockchain import SyncState +from flex.db.model.liquidity_pools import LpState, LpTransaction +from flex.db.sync_coordinator import ( + MongoSyncCoordinator, + SyncCoordinatorError, +) +from flex.domain.lp_projection import lp_event_order, lp_round_end_order + + +def _numeric(value): + return value.to_decimal() if isinstance(value, Decimal128) else value + + +def _matches(document: dict, query: dict) -> bool: + for field, expected in query.items(): + if field == "$or": + if not any(_matches(document, branch) for branch in expected): + return False + continue + if isinstance(expected, dict): + if "$exists" in expected: + if (field in document) is not expected["$exists"]: + return False + continue + actual = _numeric(document.get(field)) + if "$gte" in expected and actual < _numeric(expected["$gte"]): + return False + if "$lte" in expected and actual > _numeric(expected["$lte"]): + return False + if "$gt" in expected and actual <= _numeric(expected["$gt"]): + return False + if "$lt" in expected and actual >= _numeric(expected["$lt"]): + return False + continue + if document.get(field) != expected: + return False + return True + + +class AtomicCollection: + def __init__(self, documents: list[dict] | None = None) -> None: + self.documents = [deepcopy(document) for document in documents or []] + self.lock = RLock() + self.fail_next_upsert = False + + def find_one(self, query, projection=None): + with self.lock: + document = next((item for item in self.documents if _matches(item, query)), None) + if document is None: + return None + if projection is None: + return deepcopy(document) + return { + field: deepcopy(document[field]) + for field, include in projection.items() + if include and field in document + } + + def find_one_and_update( + self, + query, + update, + *, + upsert=False, + return_document=None, + ): + del return_document + with self.lock: + document = next((item for item in self.documents if _matches(item, query)), None) + if document is None and upsert: + if self.fail_next_upsert: + self.fail_next_upsert = False + raise RuntimeError("injected marker failure") + document = { + field: value + for field, value in query.items() + if not field.startswith("$") and not isinstance(value, dict) + } + document.update(deepcopy(update.get("$setOnInsert", {}))) + self.documents.append(document) + if document is None: + return None + for field, value in update.get("$inc", {}).items(): + document[field] = Decimal128(_numeric(document[field]) + _numeric(value)) + document.update(deepcopy(update.get("$set", {}))) + return deepcopy(document) + + def update_one(self, query, update): + with self.lock: + document = next((item for item in self.documents if _matches(item, query)), None) + if document is not None: + document.update(deepcopy(update.get("$set", {}))) + return SimpleNamespace(matched_count=int(document is not None)) + + +class BsonValidatingCollection(AtomicCollection): + def find_one_and_update(self, query, update, **kwargs): + BSON.encode( + { + "query": query, + "update": update, + } + ) + return super().find_one_and_update( + query, + update, + **kwargs, + ) + + +def _state( + *, + cursor: str | None = None, + last_round: int = 99, + reserve: int = 100, +) -> LpState: + return LpState( + id=1, + token_id=99, + asset1_id=7, + asset2_id=0, + dex_provider="tinyman", + address="POOL", + last_updated_round=last_round, + last_event_order=cursor, + asset1_reserve_micros=reserve, + asset2_reserve_micros=100, + total_tokens_micros=100, + asset1_reserve=0, + asset2_reserve=0, + total_tokens=0, + token_price_algo=0, + ) + + +def _transaction( + event_id: str = "TX@POOL", + *, + amount: int = 10, + round_number: int = 100, + position: int = 1, +) -> LpTransaction: + return LpTransaction( + id=event_id, + pool_address="POOL", + user_address="USER", + asa_id=7, + delta_amount_micros=amount, + confirmed_round=round_number, + event_position=position, + ) + + +def _repository( + state: LpState, + events: list[LpTransaction] | None = None, +) -> tuple[MongoLpProjectionRepository, AtomicCollection, AtomicCollection]: + states = AtomicCollection([state.to_dict()]) + markers = AtomicCollection([event.to_dict() for event in events or []]) + return ( + MongoLpProjectionRepository( + states=states, # type: ignore[arg-type] + events=markers, # type: ignore[arg-type] + ), + states, + markers, + ) + + +def test_crash_after_state_cas_repairs_marker_without_reapplying_delta() -> None: + transaction = _transaction() + repository, states, markers = _repository( + _state(cursor=lp_round_end_order(99)), + ) + markers.fail_next_upsert = True + + with pytest.raises(RuntimeError, match="injected marker"): + repository.project(transaction) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 110 + assert markers.documents == [] + + replay = repository.project(transaction) + + assert replay.result is LpProjectionResult.ALREADY_APPLIED + assert replay.state.asset1_reserve_micros == 110 + assert len(markers.documents) == 1 + + +def test_concurrent_replay_changes_the_balance_once() -> None: + transaction = _transaction() + repository, states, markers = _repository( + _state(cursor=lp_round_end_order(99)), + ) + + with ThreadPoolExecutor(max_workers=8) as executor: + outcomes = list(executor.map(repository.project, [transaction] * 20)) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 110 + assert len(markers.documents) == 1 + assert sum(outcome.result is LpProjectionResult.APPLIED for outcome in outcomes) == 1 + + +def test_later_event_cannot_hide_an_unrecorded_earlier_event() -> None: + later = _transaction("Z@POOL", position=2) + earlier = _transaction("A@POOL", position=1) + repository, states, _ = _repository( + _state(cursor=lp_round_end_order(99)), + ) + + repository.project(later) + + with pytest.raises(LpProjectionPersistenceError, match="advanced past"): + repository.project(earlier) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 110 + + +def test_snapshot_cursor_covers_legacy_events_without_guessing_markers() -> None: + transaction = _transaction(round_number=100) + repository, states, markers = _repository( + _state(cursor=lp_round_end_order(100), last_round=100), + ) + + outcome = repository.project(transaction) + + assert outcome.result is LpProjectionResult.SNAPSHOT_COVERED + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 100 + assert markers.documents == [] + + +def test_legacy_state_without_cutover_snapshot_fails_closed() -> None: + repository, states, markers = _repository( + _state(cursor=None, last_round=100), + ) + + with pytest.raises(LpProjectionPersistenceError, match="authoritative snapshot"): + repository.project(_transaction(round_number=101)) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 100 + assert markers.documents == [] + + +def test_projection_rejects_underflow_and_preserves_cursor() -> None: + initial_cursor = lp_round_end_order(99) + transaction = _transaction(amount=-101) + repository, states, _ = _repository( + _state(cursor=initial_cursor), + ) + + with pytest.raises(LpProjectionPersistenceError, match="underflow or overflow"): + repository.project(transaction) + + stored = LpState.from_dict(states.documents[0]) + assert stored.asset1_reserve_micros == 100 + assert stored.last_event_order == initial_cursor + + +def test_existing_event_id_with_different_payload_fails_closed() -> None: + expected = _transaction(amount=10) + conflicting = _transaction(amount=11) + repository, states, _ = _repository( + _state(cursor=lp_round_end_order(99)), + events=[conflicting], + ) + + with pytest.raises(LpProjectionPersistenceError, match="different immutable data"): + repository.project(expected) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 100 + + +def test_existing_marker_cannot_advance_a_state_without_its_delta() -> None: + transaction = _transaction() + initial_cursor = lp_round_end_order(99) + repository, states, _ = _repository( + _state(cursor=initial_cursor), + events=[transaction], + ) + + with pytest.raises( + LpProjectionPersistenceError, + match="recorded but state", + ): + repository.project(transaction) + + stored = LpState.from_dict(states.documents[0]) + assert stored.asset1_reserve_micros == 100 + assert stored.last_event_order == initial_cursor + + +def test_uint64_amounts_use_decimal128_without_precision_loss() -> None: + amount = 2**64 - 2 + repository, states, _ = _repository( + _state( + cursor=lp_round_end_order(99), + reserve=1, + ), + ) + + outcome = repository.project( + _transaction(amount=amount), + ) + + assert outcome.state.asset1_reserve_micros == 2**64 - 1 + assert isinstance(states.documents[0]["asset1_reserve_micros"], Decimal128) + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 2**64 - 1 + + +def test_uint64_identifiers_and_cursors_are_bson_safe() -> None: + maximum = 2**64 - 1 + transaction = LpTransaction( + id="TX@POOL", + pool_address="POOL", + user_address="USER", + asa_id=maximum, + delta_amount_micros=maximum, + confirmed_round=maximum, + event_position=maximum, + ) + + encoded = BSON.encode(transaction.to_dict()) + restored = LpTransaction.from_dict(BSON(encoded).decode()) + + assert restored.asa_id == maximum + assert restored.delta_amount_micros == maximum + assert restored.confirmed_round == maximum + assert restored.event_position == maximum + + +def test_uint64_repository_queries_and_updates_are_bson_safe() -> None: + maximum = 2**64 - 1 + state = _state( + cursor=lp_round_end_order(maximum - 1), + last_round=maximum - 1, + ) + state.token_id = maximum + states = BsonValidatingCollection([state.to_dict()]) + markers = BsonValidatingCollection() + repository = MongoLpProjectionRepository( + states=states, # type: ignore[arg-type] + events=markers, # type: ignore[arg-type] + ) + + outcome = repository.project( + _transaction( + amount=1, + round_number=maximum, + position=maximum, + ), + ) + + assert outcome.result is LpProjectionResult.APPLIED + assert outcome.state.last_updated_round == maximum + BSON.encode( + LpState.encode_query( + { + "token_id": { + "$in": [maximum], + } + } + ) + ) + + +def test_event_position_is_sorted_numerically_within_a_round() -> None: + assert lp_event_order(100, "TX-2", 2) < lp_event_order(100, "TX-10", 10) + + +def test_snapshot_cannot_overwrite_a_newer_event_cursor() -> None: + newer_cursor = lp_event_order(101, "TX@POOL", 1) + repository, states, _ = _repository( + _state(cursor=newer_cursor, last_round=101, reserve=150), + ) + + result = repository.replace_snapshot( + _state(cursor=None, last_round=100, reserve=90), + observed_round=100, + ) + + assert result.last_event_order == newer_cursor + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 150 + + +def test_same_round_snapshot_with_different_balances_fails_closed() -> None: + snapshot_cursor = lp_round_end_order(100) + repository, states, _ = _repository( + _state(cursor=snapshot_cursor, last_round=100, reserve=100), + ) + + with pytest.raises(LpProjectionPersistenceError, match="conflicts"): + repository.replace_snapshot( + _state(cursor=None, last_round=100, reserve=101), + observed_round=100, + ) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 100 + + +def test_older_derived_calculation_cannot_overwrite_newer_price() -> None: + cursor = lp_round_end_order(100) + newer_time = datetime(2026, 1, 1, 0, 1, tzinfo=UTC) + repository, states, _ = _repository( + _state(cursor=cursor, last_round=100), + ) + states.documents[0]["derived_observed_at"] = newer_time + states.documents[0]["token_price_algo"] = 2.0 + stale = _state(cursor=cursor, last_round=100) + stale.derived_observed_at = newer_time - timedelta(minutes=1) + stale.token_price_algo = 1.0 + + updated = repository.update_derived_fields( + stale, + expected_cursor=cursor, + ) + + assert updated is None + assert states.documents[0]["token_price_algo"] == 2.0 + assert states.documents[0]["derived_observed_at"] == newer_time + + +def test_expired_round_lease_uses_fencing_owner_and_checkpoint_cas() -> None: + now = datetime(2026, 1, 1, tzinfo=UTC) + collection = AtomicCollection( + [ + SyncState( + last_round=99, + ).to_dict() + ] + ) + coordinator = MongoSyncCoordinator( + collection=collection, # type: ignore[arg-type] + lease_duration=timedelta(seconds=30), + ) + + first = coordinator.claim_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now, + ) + blocked = coordinator.claim_round( + owner="worker-b", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=1), + ) + takeover = coordinator.claim_round( + owner="worker-b", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=31), + ) + + assert first is not None + assert blocked is None + assert takeover is not None + with pytest.raises(SyncCoordinatorError, match="lost its claim"): + coordinator.complete_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=32), + ) + + completed = coordinator.complete_round( + owner="worker-b", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=32), + ) + + assert completed.last_round == 100 + + +def test_uint64_sync_checkpoint_operations_are_bson_safe() -> None: + maximum = 2**64 - 1 + now = datetime(2026, 1, 1, tzinfo=UTC) + collection = BsonValidatingCollection( + [ + SyncState( + last_round=maximum - 1, + ).to_dict() + ] + ) + coordinator = MongoSyncCoordinator( + collection=collection, # type: ignore[arg-type] + ) + + claimed = coordinator.claim_round( + owner="worker", + expected_last_round=maximum - 1, + round_number=maximum, + now=now, + ) + completed = coordinator.complete_round( + owner="worker", + expected_last_round=maximum - 1, + round_number=maximum, + now=now, + ) + + assert claimed is not None + assert claimed.claimed_round == maximum + assert completed.last_round == maximum diff --git a/tests/unit/test_lp_transaction_projection.py b/tests/unit/test_lp_transaction_projection.py index 10def713..8077e115 100644 --- a/tests/unit/test_lp_transaction_projection.py +++ b/tests/unit/test_lp_transaction_projection.py @@ -1,8 +1,15 @@ import asyncio from types import SimpleNamespace +import pytest + from flex import sync_pools from flex.data import lp_states +from flex.db.lp_projection import ( + LpProjectionOutcome, + LpProjectionResult, +) +from flex.db.model.liquidity_pools import LpTransaction from flex.domain.transactions import ASSET_TRANSFER_TX @@ -43,6 +50,7 @@ def test_lp_to_lp_transfer_updates_both_scoped_projections(monkeypatch) -> None: asset2_reserve_micros=100, total_tokens_micros=100, last_updated_round=0, + last_event_order=None, ), "POOL-B": SimpleNamespace( id=2, @@ -54,24 +62,45 @@ def test_lp_to_lp_transfer_updates_both_scoped_projections(monkeypatch) -> None: asset2_reserve_micros=200, total_tokens_micros=200, last_updated_round=0, + last_event_order=None, ), } - created = [] - updated = [] + + class FakeProjectionRepository: + def __init__(self) -> None: + self.projected = [] + + def project(self, transaction): + self.projected.append(transaction) + state = states[transaction.pool_address] + state.asset1_reserve_micros += transaction.delta_amount_micros + state.last_event_order = transaction.event_order + return LpProjectionOutcome( + state=state, + result=LpProjectionResult.APPLIED, + ) + + def get_state(self, pool_address): + return states[pool_address] + + def update_derived_fields(self, state, *, expected_cursor): + assert state.last_event_order == expected_cursor + return state + + repository = FakeProjectionRepository() monkeypatch.setattr( lp_states, "db", SimpleNamespace( - lp_transactions=SimpleNamespace( - exists=lambda **kwargs: False, - create_many=lambda values: created.extend(values), - ), - lp_states=SimpleNamespace( - get_one=lambda **kwargs: states[kwargs["address"]], - update=updated.append, - ), + lp_transactions=SimpleNamespace(mongodb_collection=object()), + lp_states=SimpleNamespace(mongodb_collection=object()), ), ) + monkeypatch.setattr( + lp_states, + "MongoLpProjectionRepository", + lambda **kwargs: repository, + ) async def unchanged(state): return state @@ -83,10 +112,202 @@ async def unchanged(state): ) result = asyncio.run( - lp_states.update_lp_states_with_transactions(projections), + lp_states.update_lp_states_with_transactions( + projections, + expected_round=123, + ), ) assert states["POOL-A"].asset1_reserve_micros == 95 assert states["POOL-B"].asset1_reserve_micros == 205 assert {state.address for state in result} == {"POOL-A", "POOL-B"} - assert [tx.id for tx in created] == ["TX@POOL-A", "TX@POOL-B"] + assert [tx.id for tx in repository.projected] == ["TX@POOL-A", "TX@POOL-B"] + + +def test_lp_self_transfer_has_zero_projection(monkeypatch) -> None: + monkeypatch.setattr( + sync_pools, + "get_all_lp_state_addresses", + lambda: {"POOL"}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "POOL", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert projections == [] + + +def test_lp_projection_rejects_clawback_semantics(monkeypatch) -> None: + monkeypatch.setattr( + sync_pools, + "get_all_lp_state_addresses", + lambda: {"POOL"}, + ) + raw_transaction = { + "id": "TX", + "sender": "CLAWBACK", + "confirmed-round": 123, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "POOL", + "sender": "VICTIM", + }, + } + + with pytest.raises(ValueError, match="clawback/close"): + asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + +def test_lp_projection_ignores_unrelated_clawback(monkeypatch) -> None: + monkeypatch.setattr( + sync_pools, + "get_all_lp_state_addresses", + lambda: {"POOL"}, + ) + raw_transaction = { + "id": "TX", + "sender": "CLAWBACK", + "confirmed-round": 123, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "OTHER", + "sender": "VICTIM", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert projections == [] + + +def test_lp_batch_preflight_rejects_conflicts_before_projection() -> None: + original = LpTransaction( + id="TX@POOL", + pool_address="POOL", + user_address="USER", + asa_id=7, + delta_amount_micros=5, + confirmed_round=123, + event_position=1, + ) + conflicting = LpTransaction( + id="TX@POOL", + pool_address="POOL", + user_address="USER", + asa_id=7, + delta_amount_micros=6, + confirmed_round=123, + event_position=1, + ) + + with pytest.raises( + lp_states.LpProjectionPersistenceError, + match="conflicting data", + ): + lp_states._preflight_lp_transactions( + [original, conflicting], + expected_round=123, + ) + + +def test_lp_batch_preflight_deduplicates_exact_replay() -> None: + transaction = LpTransaction( + id="TX@POOL", + pool_address="POOL", + user_address="USER", + asa_id=7, + delta_amount_micros=5, + confirmed_round=123, + event_position=1, + ) + + canonical = lp_states._preflight_lp_transactions( + [transaction, transaction], + expected_round=123, + ) + + assert canonical == [transaction] + + +def test_snapshot_checkpoint_uses_slowest_indexer_observation() -> None: + states = [ + SimpleNamespace( + last_updated_round=105, + last_event_order="00000000000000000105:~", + ), + SimpleNamespace( + last_updated_round=102, + last_event_order="00000000000000000102:~", + ), + ] + + checkpoint = sync_pools._snapshot_checkpoint_round( + previous_round=100, + node_round=110, + lp_states=states, + ) + + assert checkpoint == 102 + + +def test_snapshot_checkpoint_without_lp_states_uses_node_round() -> None: + checkpoint = sync_pools._snapshot_checkpoint_round( + previous_round=100, + node_round=110, + lp_states=[], + ) + + assert checkpoint == 110 + + +def test_snapshot_checkpoint_rejects_indexer_regression() -> None: + states = [ + SimpleNamespace( + last_updated_round=99, + last_event_order="00000000000000000099:~", + ) + ] + + with pytest.raises( + sync_pools.SyncCoordinatorError, + match="Indexer snapshots lag", + ): + sync_pools._snapshot_checkpoint_round( + previous_round=100, + node_round=110, + lp_states=states, + ) + + +def test_snapshot_checkpoint_does_not_skip_rest_of_partial_round() -> None: + states = [ + SimpleNamespace( + last_updated_round=101, + last_event_order=("00000000000000000101:00000000000000000002:TX@POOL"), + ) + ] + + checkpoint = sync_pools._snapshot_checkpoint_round( + previous_round=100, + node_round=110, + lp_states=states, + ) + + assert checkpoint == 100 diff --git a/tests/unit/test_pricing_domain.py b/tests/unit/test_pricing_domain.py index 3bc7d6e2..c75d92de 100644 --- a/tests/unit/test_pricing_domain.py +++ b/tests/unit/test_pricing_domain.py @@ -342,14 +342,16 @@ async def fetch_quote(*args, **kwargs) -> PriceQuote: update_one.assert_called_once() selector, update = update_one.call_args.args - assert selector == { - "id": stored.id, - "$or": [ - {"observed_at": {"$exists": False}}, - {"observed_at": None}, - {"observed_at": {"$lte": observed_at}}, - ], - } + assert selector == AssetPrice.encode_query( + { + "id": stored.id, + "$or": [ + {"observed_at": {"$exists": False}}, + {"observed_at": None}, + {"observed_at": {"$lte": observed_at}}, + ], + } + ) assert update["$set"]["source"] == PriceSource.VESTIGE.value assert update["$set"]["observed_at"] == observed_at assert update_one.call_args.kwargs == {"upsert": False} @@ -383,7 +385,10 @@ def test_older_asset_price_observation_cannot_replace_newer_record( persisted = asset_price_data._upsert_asset_price(older) assert persisted is False - collection.find_one.assert_called_once_with({"id": older.id}, {"_id": 1}) + collection.find_one.assert_called_once_with( + AssetPrice.encode_query({"id": older.id}), + {"_id": 1}, + ) collection.insert_one.assert_not_called() diff --git a/tests/unit/test_tinyman_price_projection.py b/tests/unit/test_tinyman_price_projection.py index a5f64fc9..1340717d 100644 --- a/tests/unit/test_tinyman_price_projection.py +++ b/tests/unit/test_tinyman_price_projection.py @@ -28,6 +28,7 @@ def _lp_state( total_tokens_micros=2_000_000, token_price_algo=0, last_updated_round=123, + last_event_order="00000000000000000123:~", updated=observed_at or datetime(2026, 1, 1, tzinfo=UTC), ) @@ -46,21 +47,13 @@ def _algo_quote(*, observed_at: datetime | None = None) -> PriceQuote: def test_tinyman_projection_persists_validated_provenance(monkeypatch) -> None: pool_observed_at = datetime(2026, 1, 1, tzinfo=UTC) state = _lp_state(observed_at=pool_observed_at) - lp_updates = [] persisted = [] async def asset_details(asset_id: int): - if asset_id == 7: - return SimpleNamespace(decimals=6, name="ASSET") - assert asset_id == 99 - return SimpleNamespace(decimals=6, name="LP") + assert asset_id == 7 + return SimpleNamespace(decimals=6, name="ASSET") monkeypatch.setattr(tinyman_lps, "get_asset_details", asset_details) - monkeypatch.setattr( - tinyman_lps, - "db", - SimpleNamespace(lp_states=SimpleNamespace(update=lp_updates.append)), - ) monkeypatch.setattr( tinyman_lps, "_upsert_asset_price", @@ -68,7 +61,7 @@ async def asset_details(asset_id: int): ) result = asyncio.run( - tinyman_lps.update_tinyman_algo_lp_state_and_prices( + tinyman_lps.update_tinyman_algo_asset_price( state, algo_quote=_algo_quote( observed_at=pool_observed_at + timedelta(minutes=1), @@ -76,8 +69,7 @@ async def asset_details(asset_id: int): ), ) - assert state.token_price_algo == 2.0 - assert lp_updates == [state] + assert state.token_price_algo == 0 assert persisted == [result] assert result.price_algo == 0.5 assert result.price_usd == 0.25 @@ -98,7 +90,7 @@ def test_tinyman_projection_rejects_empty_reserve_without_writing( with pytest.raises(InvalidLiquidityPoolError, match="must be positive"): asyncio.run( - tinyman_lps.update_tinyman_algo_lp_state_and_prices( + tinyman_lps.update_tinyman_algo_asset_price( state, algo_quote=_algo_quote(), ), From cfa4033e2839d507ddd145047142034dcd140705 Mon Sep 17 00:00:00 2001 From: wackloner Date: Sun, 19 Jul 2026 15:33:05 +0700 Subject: [PATCH 03/10] fix financial read model regressions --- BOARD.md | 3 +- Makefile | 5 +- app.py | 4 +- flex/db/model/pools.py | 20 +++-- .../unit/test_financial_model_regressions.py | 83 +++++++++++++++++++ 5 files changed, 102 insertions(+), 13 deletions(-) create mode 100644 tests/unit/test_financial_model_regressions.py diff --git a/BOARD.md b/BOARD.md index 3723762a..17985709 100644 --- a/BOARD.md +++ b/BOARD.md @@ -8,7 +8,7 @@ - **Statuses**: `todo` | `in_progress` | `blocked` | `done` - **Priorities**: `critical` | `high` | `medium` | `low` - **Tags**: `security` | `backend` | `infra` | `dx` | `arch` | `perf` -- Next available ID: **CB-080** +- Next available ID: **CB-081** ## Active @@ -20,6 +20,7 @@ | CB-076 | Async persistence boundary | todo | high | backend, perf | Storage outages cannot block the event loop; timeouts and readiness covered | | CB-078 | Replay-safe outbound asset payouts | done | critical | security, backend, arch | Exact allocations, immutable airdrop manifests, persisted signed intents, on-chain reconciliation, and regression tests | | CB-079 | Crash-safe LP projection | done | critical | backend, arch | Decimal128 balances, ordered per-state CAS cursor, fenced round checkpoint, snapshot guards, and crash/concurrency tests | +| CB-080 | Financial read-model regressions | done | medium | backend | Reward decimals use the reward asset and request ordering never mutates cached contract state | ## Completed milestones diff --git a/Makefile b/Makefile index c243cd04..ebe1293c 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,8 @@ PYTHON_LINT_PATHS := \ flex/db/classes/bson_uint64.py flex/db/classes/collection_manager.py \ flex/db/indexes.py flex/db/lp_projection.py \ flex/db/sync_coordinator.py flex/db/model/airdrop.py flex/db/model/blockchain.py flex/db/model/liquidity_pools.py \ - flex/db/model/priced.py flex/db/model/transfers.py flex/domain flex/providers/pact.py flex/providers/price_router.py \ + flex/db/model/pools.py flex/db/model/priced.py flex/db/model/transfers.py flex/domain flex/providers/pact.py \ + flex/providers/price_router.py \ flex/providers/vestige.py flex/sync_pools.py flex/sync_state.py \ flex/migrations/fix_dex_providers.py \ flex/tools/airdrop.py scripts/verify_algorand_credentials.py tests @@ -32,7 +33,7 @@ PYTHON_FORMAT_PATHS := \ flex/data/transactions.py flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/base_entity.py \ flex/db/classes/bson_uint64.py flex/db/classes/collection_manager.py flex/db/indexes.py flex/db/lp_projection.py \ flex/db/sync_coordinator.py flex/db/model/airdrop.py flex/db/model/blockchain.py \ - flex/db/model/liquidity_pools.py flex/db/model/priced.py flex/db/model/transfers.py \ + flex/db/model/liquidity_pools.py flex/db/model/pools.py flex/db/model/priced.py flex/db/model/transfers.py \ flex/domain flex/providers/pact.py flex/providers/price_router.py \ flex/providers/vestige.py flex/sync_pools.py flex/sync_state.py telegram_bot.py \ flex/migrations/fix_dex_providers.py \ diff --git a/app.py b/app.py index a8c9b477..f38cc54e 100644 --- a/app.py +++ b/app.py @@ -423,7 +423,9 @@ async def get_contracts( without_old_pools: bool = True, include_address_pools: Optional[str] = None, ) -> List[ContractInfo]: - contracts = get_contracts_by_type(type) + # The repository result is TTL-cached. Work on a copy so `new_first` + # cannot reverse shared cache state for later requests. + contracts = list(get_contracts_by_type(type)) if include_address_pools and ( include_address_pools in settings.special_addresses diff --git a/flex/db/model/pools.py b/flex/db/model/pools.py index 17895a08..8d0fdfaf 100644 --- a/flex/db/model/pools.py +++ b/flex/db/model/pools.py @@ -5,14 +5,14 @@ from dataclasses_json import dataclass_json from flex.blockchain.info import ALGO_ASSET -from flex.db.model.blockchain import AssetInfo from flex.db.classes.base_entity import BaseEntity +from flex.db.model.blockchain import AssetInfo class PoolType(str, Enum): - STAKING = 'staking' - FARMING = 'farming' - ANY = 'any' + STAKING = "staking" + FARMING = "farming" + ANY = "any" @dataclass_json @@ -46,7 +46,7 @@ def length_blocks(self) -> int: @dataclass_json @dataclass -class StakingPool(BaseEntity['StakingPool']): +class StakingPool(BaseEntity["StakingPool"]): description: str address: str @@ -80,7 +80,9 @@ def to_info(self) -> PoolInfo: stake_token=self.stake_token, reward_token=self.reward_token, reward_amount_micros=self.reward_amount_micros, - reward_amount=self.stake_token.micros_to_amount(self.reward_amount_micros), + reward_amount=self.reward_token.micros_to_amount( + self.reward_amount_micros, + ), algo_reward_amount_micros=self.algo_reward_amount_micros, algo_reward_amount=ALGO_ASSET.micros_to_amount(self.algo_reward_amount_micros), begin_block=self.begin_block, @@ -88,13 +90,13 @@ def to_info(self) -> PoolInfo: lock_length_blocks=self.lock_length_blocks, deploy_date=self.deploy_date, begin_date=self.begin_date, - end_date=self.end_date + end_date=self.end_date, ) @dataclass_json @dataclass -class FarmingPool(BaseEntity['FarmingPool']): +class FarmingPool(BaseEntity["FarmingPool"]): description: str address: str @@ -141,7 +143,7 @@ def to_info(self) -> PoolInfo: lock_length_blocks=self.lock_length_blocks, deploy_date=self.deploy_date, begin_date=self.begin_date, - end_date=self.end_date + end_date=self.end_date, ) diff --git a/tests/unit/test_financial_model_regressions.py b/tests/unit/test_financial_model_regressions.py new file mode 100644 index 00000000..e47f510a --- /dev/null +++ b/tests/unit/test_financial_model_regressions.py @@ -0,0 +1,83 @@ +import asyncio +from datetime import UTC, datetime +from types import SimpleNamespace + +import app as api_module +from flex.db.model.blockchain import AssetInfo +from flex.db.model.pools import StakingPool + + +def test_staking_pool_uses_reward_token_decimals() -> None: + now = datetime(2026, 1, 1, tzinfo=UTC) + pool = StakingPool( + description="different decimals", + address="POOL", + stake_token=AssetInfo( + id=1, + name="STAKE", + unit_name="STK", + decimals=6, + ), + reward_token=AssetInfo( + id=2, + name="REWARD", + unit_name="RWD", + decimals=2, + ), + reward_amount_micros=123, + algo_reward_amount_micros=0, + begin_block=1, + end_block=2, + lock_length_blocks=0, + deploy_date=now, + begin_date=now, + end_date=now, + id=3, + ) + + assert pool.to_info().reward_amount == 1.23 + + +def test_contract_sorting_does_not_mutate_cached_repository_list( + monkeypatch, +) -> None: + cached = [ + SimpleNamespace( + id=1, + end_date=None, + metadata={"cache": {"global": {"totalStaked": "0x1"}}}, + ), + SimpleNamespace( + id=2, + end_date=None, + metadata={"cache": {"global": {"totalStaked": "0x1"}}}, + ), + ] + monkeypatch.setattr( + api_module, + "get_contracts_by_type", + lambda contract_type: cached, + ) + + newest_first = asyncio.run( + api_module.get_contracts( + type=None, + max_count=None, + new_first=True, + without_old_pools=False, + include_address_pools=None, + ) + ) + original_order = asyncio.run( + api_module.get_contracts( + type=None, + max_count=None, + new_first=False, + without_old_pools=False, + include_address_pools=None, + ) + ) + + assert [contract.id for contract in newest_first] == [2, 1] + assert [contract.id for contract in original_order] == [1, 2] + assert [contract.id for contract in cached] == [1, 2] From cf34ed1bf1a74449379395d27f058c99dda77493 Mon Sep 17 00:00:00 2001 From: wackloner Date: Sun, 19 Jul 2026 16:44:30 +0700 Subject: [PATCH 04/10] fix remaining financial race boundaries --- .env.example | 2 + api/background.py | 14 +++-- docs/architecture/lp-projection.md | 6 +- env.py | 4 ++ flex/db/lp_projection.py | 17 ++++-- flex/db/sync_coordinator.py | 1 + tests/unit/test_lp_projection_repository.py | 65 +++++++++++++++++++++ tests/unit/test_price_background.py | 40 +++++++++++++ 8 files changed, 137 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index 0a949e06..3ccebe70 100644 --- a/.env.example +++ b/.env.example @@ -37,5 +37,7 @@ FARM_CREATION_FEE=0 FARM_FLAT_ALGO_CREATION_FEE=0 # Pricing resilience +BACKGROUND_ASSET_PRICES_UPDATE=true +BACKGROUND_LP_PRICES_UPDATE=false ASSET_PRICES_TTL=120 ASSET_PRICES_MAX_STALE=3600 diff --git a/api/background.py b/api/background.py index bde3f626..2c195352 100644 --- a/api/background.py +++ b/api/background.py @@ -240,11 +240,15 @@ async def update_asset_prices_background(): failures, ) - # LP token prices: calculate from on-chain reserves - try: - await update_lp_token_prices(current_round) - except Exception as e: - logger.error(f"LP token price update failed: {e}") + if settings.background_lp_prices_update: + # This legacy worker prices raw account balances. It is opt-in until + # every supported DEX has a verified economic-reserve adapter. + try: + await update_lp_token_prices(current_round) + except Exception: + logger.exception("LP token price update failed") + else: + logger.info("Background LP price updates are disabled") def run_background(): diff --git a/docs/architecture/lp-projection.md b/docs/architecture/lp-projection.md index 2043ca05..487907c9 100644 --- a/docs/architecture/lp-projection.md +++ b/docs/architecture/lp-projection.md @@ -71,5 +71,7 @@ aggregate. Indexer account balances are authoritative for reconciliation, but not necessarily for a DEX’s economic reserve accounting: donations and protocol -excess balances may be included. Production pricing therefore remains disabled -until each DEX has a verified app-state adapter. +excess balances may be included. The legacy account-balance price worker is +therefore independently disabled by default with +`BACKGROUND_LP_PRICES_UPDATE=false` until each DEX has a verified app-state +adapter. diff --git a/env.py b/env.py index 82bc2bd5..86ff49c6 100644 --- a/env.py +++ b/env.py @@ -57,6 +57,10 @@ class Settings(BaseSettings): background_user_pools_update: bool = False background_pools_update: bool = False background_asset_prices_update: bool = True # Enable background update of asset prices + # Legacy LP pricing uses raw pool-account balances, which can include + # donations or protocol excess. Keep it off until each DEX has a verified + # economic-reserve adapter. + background_lp_prices_update: bool = False asset_price_update_batch_size: int = Field(default=10, gt=0) asset_price_api_call_delay: float = Field(default=1, ge=0) asset_price_batch_delay: float = Field(default=2.0, ge=0) diff --git a/flex/db/lp_projection.py b/flex/db/lp_projection.py index 183e0b80..5aa597e7 100644 --- a/flex/db/lp_projection.py +++ b/flex/db/lp_projection.py @@ -82,11 +82,18 @@ def project(self, transaction: LpTransaction) -> LpProjectionOutcome: if canonical_event is not None: if cursor < order: - raise LpProjectionPersistenceError( - f"LP event {transaction.id!r} is recorded but state " - f"{state.token_id} is behind it; reconcile from an " - "authoritative pool snapshot" - ) + # Another worker may have advanced the state and recorded + # the marker after this worker read its initial snapshot. + # Re-read before classifying marker/state divergence as + # persistent corruption. + state = self._get_state(transaction.pool_address) + cursor = state.last_event_order + if cursor is None or cursor < order: + raise LpProjectionPersistenceError( + f"LP event {transaction.id!r} is recorded but state " + f"{state.token_id} is behind it; reconcile from an " + "authoritative pool snapshot" + ) return LpProjectionOutcome( state=state, result=LpProjectionResult.ALREADY_APPLIED, diff --git a/flex/db/sync_coordinator.py b/flex/db/sync_coordinator.py index 3bfaa0bd..44cb9ec0 100644 --- a/flex/db/sync_coordinator.py +++ b/flex/db/sync_coordinator.py @@ -80,6 +80,7 @@ def complete_round( "last_round": encoded_expected_round, "claimed_round": encode_bson_integer(round_number), "lease_owner": owner, + "lease_until": {"$gt": current_time}, }, { "$set": { diff --git a/tests/unit/test_lp_projection_repository.py b/tests/unit/test_lp_projection_repository.py index 2406a35a..9d948a37 100644 --- a/tests/unit/test_lp_projection_repository.py +++ b/tests/unit/test_lp_projection_repository.py @@ -215,6 +215,45 @@ def test_concurrent_replay_changes_the_balance_once() -> None: assert sum(outcome.result is LpProjectionResult.APPLIED for outcome in outcomes) == 1 +def test_replay_refreshes_stale_state_before_declaring_marker_divergence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + transaction = _transaction() + stale_state = _state(cursor=lp_round_end_order(99)) + repository, _, _ = _repository( + _state( + cursor=transaction.event_order, + last_round=100, + reserve=110, + ), + events=[transaction], + ) + original_get_state = MongoLpProjectionRepository._get_state + read_count = 0 + + def stale_then_current( + current_repository: MongoLpProjectionRepository, + pool_address: str, + ) -> LpState: + nonlocal read_count + read_count += 1 + if read_count == 1: + return stale_state + return original_get_state(current_repository, pool_address) + + monkeypatch.setattr( + MongoLpProjectionRepository, + "_get_state", + stale_then_current, + ) + + outcome = repository.project(transaction) + + assert outcome.result is LpProjectionResult.ALREADY_APPLIED + assert outcome.state.asset1_reserve_micros == 110 + assert read_count == 2 + + def test_later_event_cannot_hide_an_unrecorded_earlier_event() -> None: later = _transaction("Z@POOL", position=2) earlier = _transaction("A@POOL", position=1) @@ -487,6 +526,32 @@ def test_expired_round_lease_uses_fencing_owner_and_checkpoint_cas() -> None: assert completed.last_round == 100 +def test_expired_round_lease_cannot_complete_without_takeover() -> None: + now = datetime(2026, 1, 1, tzinfo=UTC) + collection = AtomicCollection([SyncState(last_round=99).to_dict()]) + coordinator = MongoSyncCoordinator( + collection=collection, # type: ignore[arg-type] + lease_duration=timedelta(seconds=30), + ) + + claimed = coordinator.claim_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now, + ) + + assert claimed is not None + with pytest.raises(SyncCoordinatorError, match="lost its claim"): + coordinator.complete_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=31), + ) + assert SyncState.from_dict(collection.documents[0]).last_round == 99 + + def test_uint64_sync_checkpoint_operations_are_bson_safe() -> None: maximum = 2**64 - 1 now = datetime(2026, 1, 1, tzinfo=UTC) diff --git a/tests/unit/test_price_background.py b/tests/unit/test_price_background.py index 85ad41df..f5f2a295 100644 --- a/tests/unit/test_price_background.py +++ b/tests/unit/test_price_background.py @@ -9,6 +9,7 @@ def test_empty_asset_catalog_still_updates_lp_prices(monkeypatch) -> None: monkeypatch.setattr(background.settings, "background_asset_prices_update", True) + monkeypatch.setattr(background.settings, "background_lp_prices_update", True) monkeypatch.setattr( background, "db", @@ -44,6 +45,7 @@ def test_lp_registry_failure_cannot_overwrite_lp_with_external_price( monkeypatch, ) -> None: monkeypatch.setattr(background.settings, "background_asset_prices_update", True) + monkeypatch.setattr(background.settings, "background_lp_prices_update", True) monkeypatch.setattr( background, "db", @@ -85,6 +87,44 @@ async def record_lp_update(current_round: int) -> None: assert updated_rounds == [321] +def test_lp_price_worker_is_disabled_independently( + monkeypatch, +) -> None: + monkeypatch.setattr(background.settings, "background_asset_prices_update", True) + monkeypatch.setattr(background.settings, "background_lp_prices_update", False) + monkeypatch.setattr( + background, + "db", + SimpleNamespace( + assets=SimpleNamespace(get_all=lambda: []), + asset_prices=SimpleNamespace(get_all=lambda: []), + ), + ) + monkeypatch.setattr(background, "get_current_round", lambda: 321) + + async def no_lp_definitions() -> list[dict]: + return [] + + async def unexpected_lp_update(current_round: int) -> None: + raise AssertionError( + f"disabled LP price worker received round {current_round}", + ) + + monkeypatch.setattr( + background, + "get_lp_token_definitions", + no_lp_definitions, + ) + monkeypatch.setattr( + background, + "update_lp_token_prices", + unexpected_lp_update, + ) + + one_shot = background.update_asset_prices_background.__wrapped__.__wrapped__ + asyncio.run(one_shot()) + + def test_incomplete_lp_registry_fails_closed(monkeypatch) -> None: contract = SimpleNamespace( metadata={ From df1c6f75a98d2771130f8a19cd426a9e2e3aec54 Mon Sep 17 00:00:00 2001 From: wackloner Date: Sun, 19 Jul 2026 16:45:02 +0700 Subject: [PATCH 05/10] add real mongo compatibility gate --- .github/workflows/ci.yml | 55 ++- Makefile | 9 +- pyproject.toml | 3 + .../test_mongo_financial_projection.py | 340 ++++++++++++++++++ 4 files changed, 401 insertions(+), 6 deletions(-) create mode 100644 tests/integration/test_mongo_financial_projection.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e8b02cd..bfc3b5a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,18 +13,23 @@ permissions: contents: read jobs: - python: + python-matrix: + name: Python ${{ matrix.python-version }} runs-on: ubuntu-latest timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.14"] steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: - python-version: "3.12" + python-version: ${{ matrix.python-version }} cache: pipenv - run: python -m pip install --disable-pip-version-check pipenv==2024.4.0 - run: pipenv verify - - run: pipenv sync --dev + - run: pipenv sync --dev --python "${{ matrix.python-version }}" - name: Lint maintained modules run: make lint - name: Check formatting @@ -75,3 +80,47 @@ jobs: --severity HIGH,CRITICAL \ --exit-code 1 \ "cometa-backend:${COMETA_IMAGE_TAG:-local}" + + mongo-integration: + name: MongoDB financial invariants + runs-on: ubuntu-latest + timeout-minutes: 10 + services: + mongodb: + image: mongo:7.0.28@sha256:4510cf3d7050003e958745adb25d2deb3fb907430716162d9cc1a92eda2a6047 + ports: + - 27017:27017 + options: >- + --health-cmd "mongosh --quiet --eval 'quit(db.adminCommand(\"ping\").ok ? 0 : 2)'" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + env: + MONGODB_TEST_URI: mongodb://127.0.0.1:27017 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + cache: pipenv + - run: python -m pip install --disable-pip-version-check pipenv==2024.4.0 + - run: pipenv sync --dev --python "3.12" + - name: Test financial repositories against MongoDB + run: pipenv run pytest tests/integration -m integration -v + + python: + name: python + if: ${{ always() }} + needs: + - python-matrix + - mongo-integration + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: Require Python and MongoDB checks + env: + MATRIX_RESULT: ${{ needs.python-matrix.result }} + MONGO_RESULT: ${{ needs.mongo-integration.result }} + run: | + test "$MATRIX_RESULT" = "success" + test "$MONGO_RESULT" = "success" diff --git a/Makefile b/Makefile index ebe1293c..54caa8c4 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ PYTHON_MODERN_PATHS := \ flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/bson_uint64.py \ flex/db/lp_projection.py flex/db/sync_coordinator.py \ flex/db/model/airdrop.py flex/db/model/priced.py flex/db/model/transfers.py \ - flex/domain flex/providers/pact.py flex/providers/price_router.py tests/unit + flex/domain flex/providers/pact.py flex/providers/price_router.py tests/unit tests/integration PYTHON_FORMAT_PATHS := \ api/background.py api/nft_lottery.py api/wallet.py app.py blockchain/indexer.py bot/log.py \ @@ -37,15 +37,18 @@ PYTHON_FORMAT_PATHS := \ flex/domain flex/providers/pact.py flex/providers/price_router.py \ flex/providers/vestige.py flex/sync_pools.py flex/sync_state.py telegram_bot.py \ flex/migrations/fix_dex_providers.py \ - flex/tools/airdrop.py tests/conftest.py tests/unit + flex/tools/airdrop.py tests/conftest.py tests/unit tests/integration -.PHONY: sync run lint format format-check typecheck test quality +.PHONY: sync run run-api lint format format-check typecheck test quality sync: pipenv verify pipenv sync --dev run: + pipenv run python app.py + +run-api: pipenv run uvicorn app:app --reload --port 8000 lint: diff --git a/pyproject.toml b/pyproject.toml index f195d75a..6a9448cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,9 @@ addopts = "-ra --strict-config --strict-markers" pythonpath = ["."] testpaths = ["tests"] +markers = [ + "integration: requires an external service such as MongoDB", +] [tool.coverage.run] branch = true diff --git a/tests/integration/test_mongo_financial_projection.py b/tests/integration/test_mongo_financial_projection.py new file mode 100644 index 00000000..2a6e15be --- /dev/null +++ b/tests/integration/test_mongo_financial_projection.py @@ -0,0 +1,340 @@ +import os +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime, timedelta +from threading import Barrier, Event +from uuid import uuid4 + +import pytest +from bson import Decimal128 +from pymongo import MongoClient +from pymongo.database import Database + +from flex.db.indexes import create_unique_field_index_fail_closed +from flex.db.lp_projection import ( + LpProjectionResult, + MongoLpProjectionRepository, +) +from flex.db.model.blockchain import SyncState +from flex.db.model.liquidity_pools import LpState, LpTransaction +from flex.db.sync_coordinator import ( + MongoSyncCoordinator, + SyncCoordinatorError, +) +from flex.domain.lp_projection import lp_round_end_order + +pytestmark = pytest.mark.integration + + +class _CoordinatedRepository(MongoLpProjectionRepository): + """Hold stale readers until one worker persists both state and marker.""" + + def __init__( + self, + *, + states, + events, + first_read_barrier: Barrier, + winner_committed: Event, + wait_for_winner: bool, + ) -> None: + super().__init__(states=states, events=events) + self._first_read_barrier = first_read_barrier + self._winner_committed = winner_committed + self._wait_for_winner = wait_for_winner + self._first_read = True + + def _get_state(self, pool_address: str) -> LpState: + state = super()._get_state(pool_address) + if self._first_read: + self._first_read = False + self._first_read_barrier.wait(timeout=5) + if self._wait_for_winner and not self._winner_committed.wait(timeout=5): + raise TimeoutError("winning replay did not persist its marker") + return state + + def _record_event(self, transaction: LpTransaction) -> None: + super()._record_event(transaction) + if not self._wait_for_winner: + self._winner_committed.set() + + +@pytest.fixture +def mongo_database() -> Database: + uri = os.getenv("MONGODB_TEST_URI") + if not uri: + pytest.skip("MONGODB_TEST_URI is not configured") + + client = MongoClient( + uri, + serverSelectionTimeoutMS=2_000, + tz_aware=True, + ) + client.admin.command("ping") + database_name = f"cometa_integration_{uuid4().hex}" + database = client[database_name] + try: + yield database + finally: + client.drop_database(database_name) + client.close() + + +def _state(*, reserve: int = 100) -> LpState: + return LpState( + id=1, + token_id=99, + asset1_id=7, + asset2_id=0, + dex_provider="tinyman", + address="POOL", + last_updated_round=99, + last_event_order=lp_round_end_order(99), + asset1_reserve_micros=reserve, + asset2_reserve_micros=100, + total_tokens_micros=100, + asset1_reserve=0, + asset2_reserve=0, + total_tokens=0, + token_price_algo=0, + ) + + +def _transaction(*, amount: int = 10) -> LpTransaction: + return LpTransaction( + id="TX@POOL", + pool_address="POOL", + user_address="USER", + asa_id=7, + delta_amount_micros=amount, + confirmed_round=100, + event_position=1, + ) + + +def _repository( + database: Database, + state: LpState, +) -> MongoLpProjectionRepository: + states = database["lp_states"] + events = database["lp_transactions"] + states.create_index("token_id", unique=True) + states.create_index("address", unique=True) + events.create_index("id", unique=True) + states.insert_one(state.to_dict()) + return MongoLpProjectionRepository( + states=states, + events=events, + ) + + +def test_concurrent_lp_replay_applies_real_mongo_delta_once( + mongo_database: Database, +) -> None: + repository = _repository(mongo_database, _state()) + transaction = _transaction() + first_read_barrier = Barrier(8) + winner_committed = Event() + workers = [ + _CoordinatedRepository( + states=repository.states, + events=repository.events, + first_read_barrier=first_read_barrier, + winner_committed=winner_committed, + wait_for_winner=index != 0, + ) + for index in range(8) + ] + + with ThreadPoolExecutor(max_workers=8) as executor: + futures = [executor.submit(worker.project, transaction) for worker in workers] + outcomes = [future.result(timeout=10) for future in futures] + + persisted = repository.get_state("POOL") + assert persisted.asset1_reserve_micros == 110 + assert mongo_database["lp_transactions"].count_documents({}) == 1 + assert sum(outcome.result is LpProjectionResult.APPLIED for outcome in outcomes) == 1 + + +def test_real_mongo_repairs_marker_gap_without_repeating_delta( + mongo_database: Database, +) -> None: + state = _state() + repository = _repository(mongo_database, state) + transaction = _transaction() + mongo_database["lp_states"].update_one( + {"address": state.address}, + { + "$inc": { + "asset1_reserve_micros": Decimal128("10"), + }, + "$set": { + "last_event_order": transaction.event_order, + "last_updated_round": Decimal128("100"), + }, + }, + ) + + outcome = repository.project(transaction) + + assert outcome.result is LpProjectionResult.ALREADY_APPLIED + assert outcome.state.asset1_reserve_micros == 110 + assert mongo_database["lp_transactions"].count_documents({}) == 1 + + +def test_real_mongo_decimal128_increment_reaches_uint64_max( + mongo_database: Database, +) -> None: + maximum = 2**64 - 1 + repository = _repository( + mongo_database, + _state(reserve=1), + ) + + outcome = repository.project( + _transaction(amount=maximum - 1), + ) + + raw = mongo_database["lp_states"].find_one({"address": "POOL"}) + assert outcome.state.asset1_reserve_micros == maximum + assert raw is not None + assert raw["asset1_reserve_micros"] == Decimal128(str(maximum)) + + +def test_real_mongo_promotes_legacy_int64_balance_to_decimal128( + mongo_database: Database, +) -> None: + state = _state() + payload = state.to_dict() + for field_name in ( + "id", + "token_id", + "asset1_id", + "asset2_id", + "last_updated_round", + "asset1_reserve_micros", + "asset2_reserve_micros", + "total_tokens_micros", + ): + payload[field_name] = int(payload[field_name].to_decimal()) + + states = mongo_database["lp_states"] + events = mongo_database["lp_transactions"] + states.create_index("token_id", unique=True) + states.create_index("address", unique=True) + events.create_index("id", unique=True) + states.insert_one(payload) + repository = MongoLpProjectionRepository(states=states, events=events) + + outcome = repository.project(_transaction()) + + raw = states.find_one({"address": "POOL"}) + assert outcome.state.asset1_reserve_micros == 110 + assert raw is not None + assert raw["asset1_reserve_micros"] == Decimal128("110") + + +def test_duplicate_financial_business_key_fails_without_deletion( + mongo_database: Database, +) -> None: + collection = mongo_database["lp_states"] + collection.insert_many( + [ + {"id": "a", "token_id": Decimal128("99")}, + {"id": "b", "token_id": Decimal128("99")}, + ] + ) + + with pytest.raises(RuntimeError, match="duplicate"): + create_unique_field_index_fail_closed( + collection, + collection_name="lp_states", + field_name="token_id", + index_name="token_id_unique", + ) + + assert collection.count_documents({}) == 2 + + +def test_expired_real_mongo_sync_lease_fences_stale_owner( + mongo_database: Database, +) -> None: + collection = mongo_database["sync_states"] + collection.create_index("id", unique=True) + collection.insert_one(SyncState(last_round=99).to_dict()) + coordinator = MongoSyncCoordinator( + collection=collection, + lease_duration=timedelta(seconds=30), + ) + now = datetime(2026, 1, 1, tzinfo=UTC) + + assert ( + coordinator.claim_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now, + ) + is not None + ) + assert ( + coordinator.claim_round( + owner="worker-b", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=31), + ) + is not None + ) + + with pytest.raises(SyncCoordinatorError, match="lost its claim"): + coordinator.complete_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=32), + ) + + completed = coordinator.complete_round( + owner="worker-b", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=32), + ) + assert completed.last_round == 100 + + +def test_expired_real_mongo_sync_lease_cannot_commit_without_takeover( + mongo_database: Database, +) -> None: + collection = mongo_database["sync_states"] + collection.create_index("id", unique=True) + collection.insert_one(SyncState(last_round=99).to_dict()) + coordinator = MongoSyncCoordinator( + collection=collection, + lease_duration=timedelta(seconds=30), + ) + now = datetime(2026, 1, 1, tzinfo=UTC) + + assert ( + coordinator.claim_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now, + ) + is not None + ) + + with pytest.raises(SyncCoordinatorError, match="lost its claim"): + coordinator.complete_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=31), + ) + + persisted_document = collection.find_one({"id": "main"}) + assert persisted_document is not None + persisted_document.pop("_id") + persisted = SyncState.from_dict(persisted_document) + assert persisted.last_round == 99 From b386903ec39194eda5ae19cc56555a69b2f67041 Mon Sep 17 00:00:00 2001 From: wackloner Date: Sun, 19 Jul 2026 16:45:48 +0700 Subject: [PATCH 06/10] update public engineering guide --- .claude/AGENTS.md | 14 +- BOARD.md | 8 +- CLAUDE.md | 56 +++--- CONTRIBUTING.md | 23 ++- README.md | 47 +++-- ...audit-architecture-financial-2026-07-19.md | 164 ++++++++++++++++++ 6 files changed, 269 insertions(+), 43 deletions(-) create mode 100644 docs/audit/01-audit-architecture-financial-2026-07-19.md diff --git a/.claude/AGENTS.md b/.claude/AGENTS.md index 3cac4d4e..ae098ecf 100644 --- a/.claude/AGENTS.md +++ b/.claude/AGENTS.md @@ -41,8 +41,18 @@ make quality ``` CI validates Compose but does not certify the image as immutable-deploy ready: -the private Node sidecar package-auth blocker is documented in `README.md`. Keep the -focused lint, type, and coverage ratchets honest. +it runs the full quality gate on Python 3.12 and 3.14, builds the digest-pinned +Python-only image, smoke-tests the non-root runtime, scans it with Trivy, and +checks financial persistence invariants against a disposable MongoDB service. +Keep the focused lint, type, and coverage ratchets honest. + +`make run` uses the production-equivalent `python app.py` entrypoint, including +critical indexes, migrations only when `MIGRATE=true`, and configured workers. +Use `make run-api` only for API-only hot reload. + +Financial projector design and known standalone-Mongo limits are documented in +`docs/architecture/lp-projection.md`; outbound payout recovery is documented in +`docs/architecture/outbound-asset-transfers.md`. ## Cross-Project Changes diff --git a/BOARD.md b/BOARD.md index 17985709..b071f9d9 100644 --- a/BOARD.md +++ b/BOARD.md @@ -8,7 +8,7 @@ - **Statuses**: `todo` | `in_progress` | `blocked` | `done` - **Priorities**: `critical` | `high` | `medium` | `low` - **Tags**: `security` | `backend` | `infra` | `dx` | `arch` | `perf` -- Next available ID: **CB-081** +- Next available ID: **CB-085** ## Active @@ -21,6 +21,10 @@ | CB-078 | Replay-safe outbound asset payouts | done | critical | security, backend, arch | Exact allocations, immutable airdrop manifests, persisted signed intents, on-chain reconciliation, and regression tests | | CB-079 | Crash-safe LP projection | done | critical | backend, arch | Decimal128 balances, ordered per-state CAS cursor, fenced round checkpoint, snapshot guards, and crash/concurrency tests | | CB-080 | Financial read-model regressions | done | medium | backend | Reward decimals use the reward asset and request ordering never mutates cached contract state | +| CB-081 | Fence expired sync workers | done | critical | backend, arch | A worker cannot commit a financial round after its lease expires, with unit and real-Mongo regressions | +| CB-082 | Verify Mongo financial invariants | done | high | backend, infra | CI proves CAS replay, marker repair, BSON promotion, uniqueness, and lease fencing against a disposable pinned MongoDB | +| CB-083 | Refresh public engineering docs | done | medium | dx, arch | Runtime commands, Python support, architecture boundaries, API shapes, and cross-project contract are current | +| CB-084 | Disable unverified LP pricing | done | critical | backend, arch | Legacy raw-account-balance LP pricing is independently default-off until DEX economic reserves are verified | ## Completed milestones @@ -32,7 +36,7 @@ | Replay identity | done | Deterministic nested event IDs and collection-level uniqueness constraints | | LP financial ledger | done | Marker-gap recovery, uint64-safe BSON operations, full-block preflight, snapshot coverage guards, and fenced round CAS | | Container baseline | done | Digest-pinned Alpine base, multi-stage non-root runtime, healthcheck, image exclusions, Trivy CI gate | -| API hardening | done | Fail-closed header authentication, trusted hosts, explicit CORS policy, bounded LP/asset/wallet requests | +| API request hardening | done | Fail-closed configured header checks, trusted hosts, explicit CORS policy, bounded selectors and wallet expansion | | Native Reach decoding | done | Versioned global/local codecs, exact-width integers, deterministic layout tests, no private npm runtime | | Legacy runtime removal | done | CB-077: Node/Reach sidecar and production source bind mount removed | diff --git a/CLAUDE.md b/CLAUDE.md index 145386d5..85d255b1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ Backend for Cometa — an Algorand DeFi platform handling liquidity pools, token ## Stack -- **Language**: Python 3.12, Pipenv +- **Language**: Python 3.12 production runtime; Python 3.14 compatibility CI; Pipenv - **Framework**: FastAPI + Uvicorn - **Database**: MongoDB (pymongo) - **Cache**: process-local TTL caches (Redis migration is roadmap work) @@ -16,19 +16,20 @@ Backend for Cometa — an Algorand DeFi platform handling liquidity pools, token ## Project Structure ``` -app.py — FastAPI application, routes, startup +app.py — FastAPI composition, routes, startup, and workers env.py — Settings via pydantic-settings (from .env) -api/ — Core API: background tasks, DB models, stats, swaps, wallets, NFT lottery -blockchain/ — Algorand node/indexer interaction utilities -core/ — Shared core logic -dexes/ — DEX integrations (HumbleSwap, Vestige, etc.) -flex/ — Flex module (API + data) +api/ — Product API, background work, wallets, and disabled lottery surface +blockchain/ — Algorand node/indexer adapters +core/ — Shared authentication, persistence, and resilience +dexes/ — DEX-specific integrations +flex/application/ — Financial use-case orchestration +flex/domain/ — Pure allocation, pricing, projection, and identity rules +flex/db/ — Mongo models, BSON codecs, repositories, and indexes +flex/tools/ — Operator tools such as manifest-driven airdrops bot/ — Telegram bot logic -farcaster/ — Farcaster integration -scripts/ — Deployment & management shell scripts -airdrop/ — Airdrop tooling -marketplaces/ — NFT marketplace integrations -metapunks/ — MetaPunks-specific logic +scripts/ — Deployment and management shell scripts +tests/ — Unit tests plus opt-in real-service integration tests +docs/ — Architecture decisions, operations, and audit reports ``` ## Key Commands @@ -37,7 +38,8 @@ metapunks/ — MetaPunks-specific logic # Local development pipenv verify pipenv sync --dev -pipenv run uvicorn app:app --reload --port 8000 +make run # production-equivalent startup, including indexes/workers +make run-api # API-only Uvicorn reload; no migrations or workers # Quality gate make quality @@ -57,9 +59,17 @@ scripts/redeploy.sh # pull + rebuild + restart the backend service - Background tasks in `api/background.py` — use exponential backoff for retries - Decode only explicitly supported Reach contract versions in `flex/blockchain/contract_state.py` - Route registration and process orchestration stay in `app.py`; Flex routes live in `flex/api.py` -- MongoDB models in `api/db_model.py` +- Legacy contract persistence models live in `core/db/model.py`; `api/db_model.py` + contains only the public contract-type enum. Maintained Flex models and + repositories live under `flex/db/`. - Asset prices use process-local TTL caches — see `dexes/` for provider calls - Preserve financial values as `Decimal` or integer base units until an explicit compatibility boundary +- Persist maintained Flex financial `uint64` fields through the BSON codecs in + `flex/db/bson.py`; do not copy legacy int64/float compatibility shapes +- Outbound transfers must persist immutable signed intent before broadcast and reconcile on-chain before completion +- LP events must enter through the complete-round preflight and `MongoLpProjectionRepository` +- Keep `SYNC_STAKING_POOLS=false` until full Algorand application-group validation is implemented +- Keep `BACKGROUND_LP_PRICES_UPDATE=false` until DEX-specific economic reserves are verified - New pricing and transaction invariants belong in pure modules under `flex/domain/` - Run the strict mypy target before changing `core/circuit_breaker.py` or `flex/domain/` @@ -91,8 +101,8 @@ See parent `~/dev/cometa/CLAUDE.md` for the canonical API contract table. That f ## Testing -Tests are organized by boundary under `tests/unit/`. Run the same fast checks used -by CI before committing: +Fast tests are organized by boundary under `tests/unit/`; real-service checks live +under `tests/integration/`. Run the same local quality gate used by CI before committing: ```bash make sync @@ -103,7 +113,10 @@ make quality - Use `httpx.AsyncClient` with ASGI transport for endpoint integration tests - Keep domain tests pure and deterministic; inject clocks and provider functions - Priority: event replay/idempotency, price freshness, contract CRUD, authorization +- Exercise crash boundaries, concurrent replay, BSON `uint64` limits, and + Indexer-lag cutovers for financial projections - Use a dedicated test database only for integration tests; never point tests at production +- Set `MONGODB_TEST_URI` only to run the opt-in MongoDB integration suite - Every bug fix should include a regression test when practical ## Commit Discipline @@ -118,9 +131,10 @@ make quality Tasks in `BOARD.md`. Format: pantheon. -## Available MCP Tools +## Optional Diagnostic Tools -- **MongoDB MCP** — direct database queries in Claude sessions. Use for debugging: `db.contracts.find({active: true})`, inspecting collections, verifying data integrity. Connection: `mongodb://localhost:27017/cometa` -- **Algorand MCP** — on-chain state verification, account info, asset lookups on mainnet -- **Vestige MCP** — DEX price data, pool states, trading pairs for Algorand DeFi -- **Codex MCP** — second-opinion code review via GPT-5.x. Run `review` after writing significant code +MongoDB, Algorand, and DEX inspection connectors may be available in some agent +sessions. Treat them as optional and read-only by default. Derive database and +network targets from the active environment; never assume localhost is a safe +database and never sign or broadcast a transaction without explicit task-scoped +authorization. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 076d98df..207079bd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,16 +13,21 @@ make sync make run ``` +`make run` uses the production-equivalent entrypoint. Use `make run-api` for +API-only hot reload when migrations and background workers are intentionally +out of scope. + Before opening a pull request, run the same gate used by CI: ```bash make quality ``` -Use `make format` to apply Ruff formatting. New domain code should use Python -3.12 type hints and pass strict mypy checks. Keep I/O in adapters or application -services; prefer pure functions and immutable value objects for pricing, -transaction parsing, and other financial rules. +Use `make format` to apply Ruff formatting. New domain code targets Python 3.12, +passes strict mypy, and must remain green under the Python 3.14 compatibility +job. Keep I/O in adapters or application services; prefer pure functions and +immutable value objects for pricing, transaction parsing, and other financial +rules. ## Tests @@ -31,7 +36,9 @@ using the `test_.py` naming pattern. Test both the successful result and the failure boundary, especially for: - integer or `Decimal` amount handling; +- BSON round trips across the full Algorand `uint64` range; - duplicate, replayed, reordered, or nested chain events; +- crashes between durable intent, broadcast, reconciliation, marker, and cursor writes; - stale prices and provider fallback exhaustion; - retries, timeouts, and circuit-breaker transitions; - authentication and storage uniqueness. @@ -39,6 +46,14 @@ and the failure boundary, especially for: Mock external networks at the adapter boundary. Never use production credentials, wallets, or mutable production data in tests. +MongoDB semantics that mocks cannot prove belong under `tests/integration/`. +Run them only against a disposable database: + +```bash +MONGODB_TEST_URI=mongodb://127.0.0.1:27017 \ + pipenv run pytest tests/integration -m integration -v +``` + ## API compatibility The frontend lives in `../metafarm-frontend` and calls this service through diff --git a/README.md b/README.md index 7013dfad..a25f0bd4 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@

CI - Python 3.12 + Python 3.12 and 3.14 FastAPI Algorand mainnet API status @@ -37,10 +37,11 @@ MongoDB—into stable, query-oriented API models for the product frontend. | Engineering concern | Implementation | | --- | --- | -| **Financial precision** | Prices use validated `Decimal` value objects, carry source and observation time, and cross the legacy `float` boundary only explicitly. | -| **Deterministic replay identity** | Nested Algorand transfers receive stable, projection-scoped event IDs; unique indexes guard price and event-marker collections. Crash-atomic projection remains roadmap work. | +| **Financial precision** | Prices use validated `Decimal` value objects; maintained LP projection and sync fields retain the full Algorand `uint64` domain through BSON-safe codecs. | +| **Crash-safe projections** | Scoped event cursors, per-state compare-and-set writes, immutable markers, and a fenced round checkpoint make LP replay convergent across crashes and competing workers. | +| **Replay-safe payouts** | Airdrops and NFT transfers persist immutable signed intent before broadcast, reconcile on-chain outcomes, and allocate integer base units exactly. | | **Resilient price routing** | Vestige and Tinyman payloads are validated with provenance and bounded staleness; retry classification and a guarded Vestige refresh prevent failure storms. | -| **Operational boundaries** | Selected latency-sensitive SDK calls leave the event loop through executors; wallet fan-out is cached and bounded; background workers reconcile chain state without coupling reads to refresh latency. | +| **Operational boundaries** | Selected blocking chain calls leave the event loop; deterministic failures use bounded retry; unverified staking, LP projection, and legacy LP pricing paths are disabled by default. | | **Versioned chain decoding** | Reach 0.1.11 state is decoded natively from Algorand with explicit per-version layouts, exact-width integers, and fail-closed schema validation. | | **Supply-chain hardening** | The digest-pinned Alpine image is multi-stage, non-root, and Python-only; CI smoke-tests it and rejects high/critical vulnerabilities or embedded secrets. | @@ -70,6 +71,12 @@ farm and distribution versions map packed Algorand global and local state into the existing API view shape without private npm packages or a second runtime. The decision and extension rules are documented in [`docs/architecture/native-reach-state.md`](docs/architecture/native-reach-state.md). +Financial write paths are documented separately in +[`docs/architecture/outbound-asset-transfers.md`](docs/architecture/outbound-asset-transfers.md) +and [`docs/architecture/lp-projection.md`](docs/architecture/lp-projection.md). +The latest independent risk review, including resolved and intentionally open +items, is in +[`docs/audit/01-audit-architecture-financial-2026-07-19.md`](docs/audit/01-audit-architecture-financial-2026-07-19.md). ### Reliability boundaries @@ -77,8 +84,9 @@ The decision and extension rules are documented in | --- | --- | | Provider quote → stored price | Positive, finite decimal values with source and observation timestamp | | Cached price → API response | Explicit freshness window; expired data is rejected instead of silently relabelled | -| Chain event → read model | Deterministic identity plus unique persistence constraints | -| Sync SDK → async request path | Bounded executor hand-off | +| Chain event → LP read model | Full-block preflight, uint64-safe CAS cursor, marker repair, and round fencing | +| Asset payout → Algorand | Persist signed intent first; rebroadcast identical bytes; reconcile before completion | +| Selected sync chain SDK → async request path | Bounded executor hand-off | | Permanent provider error → retry loop | Typed classification prevents pointless retries | | Half-open circuit → provider | A single probe prevents a recovery stampede | | Reach bytes → public view | Exact key, step, size, tag, and contract-version validation | @@ -87,7 +95,8 @@ The decision and extension rules are documented in ### Requirements -- Python 3.12 and [Pipenv](https://pipenv.pypa.io/) +- Python 3.12 and [Pipenv](https://pipenv.pypa.io/) for the production-equivalent environment +- Python 3.14 is also exercised by CI as a forward-compatibility gate - MongoDB - access to an Algorand node/indexer @@ -100,10 +109,13 @@ pipenv run python -c \ 'from algosdk import account, mnemonic; key, _ = account.generate_account(); print(mnemonic.from_private_key(key))' # Put this throwaway phrase in ALGO_MNEMONIC. Never fund or reuse the account. # Point MONGODB_HOST, ALGOD_ADDRESS, and ALGO_INDEXER_ADDRESS at dev services. -make run +make run-api ``` -`make run` starts Uvicorn with reload on port `8000`. A development mnemonic is +`make run-api` is the safe API-only development loop. `make run` executes the +production-equivalent entrypoint: critical indexes, +optional migrations when `MIGRATE=true`, configured workers, and Uvicorn. For +the full entrypoint, review every worker flag first. A development mnemonic is still required by legacy Python transaction adapters; use a generated, unfunded account only. @@ -121,6 +133,10 @@ docker compose up -d --build docker compose logs -f app ``` +This Compose stack starts persistent MongoDB and a full Algorand node. Inspect +the network, volume paths, and validated `MONGODB_IMAGE`/`ALGOD_IMAGE` values +before using it outside an isolated development host. + ## Quality gate ```bash @@ -134,9 +150,11 @@ This single command runs: - the complete Python test suite with branch coverage, including deterministic Reach state-codec and security-boundary tests. -CI repeats those checks on every pull request and every push to `main`, verifies -the lockfile and Compose configuration, builds and smoke-tests the production -image, and scans it with Trivy. The focused coverage ratchet is currently 75%; +CI repeats those checks on Python 3.12 and 3.14 for every pull request and every +push to `main`, verifies the lockfile and Compose configuration, builds and +smoke-tests the production image, scans it with Trivy, and exercises financial +repository invariants against a digest-pinned MongoDB service. The focused +coverage ratchet is currently 75%; it measures maintained domain and infrastructure modules rather than presenting a misleading whole-repository number. @@ -151,8 +169,8 @@ Useful individual targets are `make lint`, `make format-check`, | `GET /contracts` | Farm and distribution catalog | | `GET /contracts/user/{address}` | Contracts associated with a wallet | | `GET /contracts/farm/enriched` | Contracts enriched with asset metadata and prices | -| `POST /assets/price` | Batch asset pricing | -| `POST /lp/state/priced` | Batched LP reserve and price data | +| `POST /assets/price` | Read stored bounded-fresh prices; optional `ids` accepts up to 250 values, while omission returns all stored projections | +| `POST /lp/state/priced` | Read LP token prices; missing or stale batch entries are returned as `null` | | `GET /stats/tvl` | Protocol TVL snapshot | The production API intentionally disables interactive OpenAPI pages. Endpoint @@ -173,6 +191,7 @@ flex/domain/ Pure pricing and transaction invariants flex/providers/ Market-data provider adapters flex/db/ MongoDB models, repositories, and indexes tests/unit/ Fast regression and boundary tests +tests/integration/ Opt-in tests against disposable real services scripts/ Deployment and safe operational utilities ``` diff --git a/docs/audit/01-audit-architecture-financial-2026-07-19.md b/docs/audit/01-audit-architecture-financial-2026-07-19.md new file mode 100644 index 00000000..b5b25b14 --- /dev/null +++ b/docs/audit/01-audit-architecture-financial-2026-07-19.md @@ -0,0 +1,164 @@ +# Аудит архитектуры и финансовой корректности + +**Дата:** 2026-07-19 + +**База аудита:** `main@2c1cdad` + +**Ветка исправлений:** `audit/financial-correctness` + +## Резюме + +Аудит проводился с позиции production fintech backend, а не косметической +подготовки GitHub-профиля. На исходной базе были риски повторной выплаты, +потери точности и некорректного replay при конкурирующих воркерах. На ветке +исправлений денежные операции переведены на integer base units, устойчивые +business keys, immutable intents и compare-and-set проекции. + +Оценка критических границ до/после: + +| Область | До | После | Комментарий | +| --- | ---: | ---: | --- | +| Финансовая корректность | 3/10 | 9/10 | replay-safe выплаты, точное распределение, fail-closed legacy | +| Конкурентность и recovery | 3/10 | 8/10 | CAS, marker repair, round leases, реальные Mongo-тесты | +| Security boundaries | 5/10 | 7/10 | секреты из кода вынесены; auth/signing требуют следующего milestone | +| Инженерная проверяемость | 6/10 | 9/10 | Python 3.12/3.14, strict typing, 262 теста, Mongo integration CI | + +MongoDB гарантирует атомарность одной операции над одним документом, поэтому +проектор строится вокруг conditional update, а не blind read-modify-write. +Междокументная одновременная видимость остаётся отдельной задачей для +replica-set transactions. См. [MongoDB atomicity](https://www.mongodb.com/docs/manual/core/write-operations-atomicity/) +и [transactions](https://www.mongodb.com/docs/manual/core/transactions/). + +## Ранжированные находки + +### 1. Critical — повторная выплата и неточное распределение airdrop — исправлено + +**Почему:** сбой после broadcast, но до записи результата, позволял повторно +отправить актив; float-доли не гарантировали сохранение целого бюджета. + +**Исправление:** immutable signed intent сохраняется до broadcast и сверяется +по `operation_id`; неопределённый результат reconciled по txid +(`flex/application/asset_transfers.py:179-298`). Airdrop резервирует неизменяемый +manifest до первой отправки (`flex/tools/airdrop.py:332-405`), а largest-remainder +allocation сохраняет бюджет до последней base unit +(`flex/domain/allocation.py:36-80`). + +### 2. Critical — LP double-apply, stale-read race и истёкший lease — исправлено + +**Почему:** blind read-modify-write терял обновления; конкурентный replay мог +принять свежий marker за corruption из-за старого snapshot; истёкший worker +мог завершить round. + +**Исправление:** per-state CAS cursor, marker-last recovery и повторное чтение +при конкурентном marker (`flex/db/lp_projection.py:53-149`). Завершение round +требует неистёкший lease (`flex/db/sync_coordinator.py:63-100`). Управляемая +гонка на настоящем Mongo доказывает exactly-once delta +(`tests/integration/test_mongo_financial_projection.py:130-155`). + +### 3. Critical — LP price manipulation через raw account balance — исправлено + +**Почему:** donation или protocol excess на адресе пула мог попасть в +«экономический резерв» и исказить цену. + +**Исправление:** legacy worker отделён от обычного price refresh и default-off +через `BACKGROUND_LP_PRICES_UPDATE=false` (`env.py:57-64`, +`api/background.py:243-251`). Включать только после DEX-specific проверки +app state и economic reserves. + +### 4. High — staking classifier принимал непроверенные переводы — mitigated + +**Почему:** перевод рядом с application call не доказывает stake; без проверки +полной transaction group можно создать ложное состояние. + +**Исправление сейчас:** `SYNC_STAKING_POOLS=false`, а попытка включения +завершается fail-closed (`flex/sync_pools.py:351-367`). **Следующий фикс:** +типизированный parser полной Algorand group, проверка app ID, selector, +sender/receiver, asset и group order, затем adversarial fixtures. + +### 5. High — browser-visible shared key не является авторизацией — открыто + +**Почему:** `X-API-Key` сравнивается корректно, но один общий клиентский token +не подтверждает пользователя и не разделяет права +(`core/auth.py:8-13`, `app.py:323-369`). + +**Конкретный фикс:** registration авторизовать wallet-signature challenge с +nonce, expiry и replay table; `/contracts/refresh-cache` оставить только +server-to-server роли с отдельным secret и audit log. До этого считать текущий +token compatibility/rate-control механизмом, не security boundary. + +### 6. High — blocking persistence остаётся в async routes — открыто + +**Почему:** sync PyMongo/provider вызовы в event loop увеличивают tail latency +всех запросов при деградации Mongo или DEX (`app.py:418-428`, +`app.py:529-535`, `app.py:559-561`). + +**Конкретный фикс:** ввести async repository ports с timeout/cancellation; +переходно — `asyncio.to_thread` вокруг целого repository call и +thread-safe cache, плюс saturation/load test. + +### 7. High — Mongo invariants раньше проверялись только fake-коллекциями — исправлено + +**Почему:** mocks не проверяют BSON numeric comparison, unique-index races, +`$inc` promotion и реальные `find_one_and_update` semantics. + +**Исправление:** отдельный digest-pinned MongoDB CI job +(`.github/workflows/ci.yml:84-109`) проверяет concurrent CAS, marker repair, +legacy int64→Decimal128, `uint64` max, fail-closed duplicates и fencing +(`tests/integration/test_mongo_financial_projection.py:130-336`). Стабильный +required context `python` агрегирует matrix и Mongo job +(`.github/workflows/ci.yml:111-126`). + +### 8. Medium — mutable stateful Docker images — открыто + +**Почему:** `mongo` и `algorand/algod:latest` могут поменять major/runtime при +обычном rebuild поверх persistent volumes (`docker-compose.yml:29-61`). + +**Конкретный фикс:** после проверки реального VPS зафиксировать оба образа как +`tag@sha256`, описать backup/restore и downgrade, а CI должен отклонять bare +tags и `latest`. Не подменять production digest без data-format rehearsal. + +### 9. Medium — container smoke не запускает production entrypoint — открыто + +**Почему:** CI заменяет entrypoint на shell и импортирует `app`, поэтому не +доказывает запуск `scripts/run.sh`, Uvicorn, healthcheck и graceful shutdown +(`.github/workflows/ci.yml:55-70`). + +**Конкретный фикс:** поднять disposable Mongo, запустить образ штатно с +безопасными feature flags, дождаться healthy, запросить `/status`, затем +проверить SIGTERM и вывести logs при сбое. + +### 10. Medium — contract registration не атомарен по business key — открыто + +**Почему:** check-then-insert допускает конкурентные дубликаты, а notification +после записи является незарегистрированным side effect +(`app.py:327-365`, `core/db/contracts.py:16-25`). + +**Конкретный фикс:** unique index по contract `id`, atomic upsert с +immutable-field conflict check и transactional outbox для уведомления. + +## Milestones + +1. **M0 — money safety (готово):** findings 1–3 исправлены; finding 4 + переведён в fail-closed. Добавлены NFT idempotency + (`api/wallet.py:14-37`), on-chain reconciliation и regressions. +2. **M1 — persistence proof (готово):** finding 7, Python matrix, BSON boundary, + stable required check. Production остаётся на 3.12; 3.14 — compatibility + gate. Python 3.14.6 является актуальным maintenance release + ([Python.org](https://www.python.org/downloads/release/python-3146/)). +3. **M2 — authority and atomic workflows (следующий):** findings 5 и 10. +4. **M3 — runtime hardening:** findings 6, 8 и 9; затем SLO/metrics и controlled + deploy rehearsal. + +## Вклад независимых агентов + +- три financial-review потока независимо подтвердили payout/LP классы ошибок; +- adversarial Mongo/BSON review воспроизвёл stale-read race, пропущенный + первоначальным concurrency-тестом; +- CI/GitHub review обнаружил drift обязательного status context после matrix; +- dependency/container review проверил lock, image posture и runtime smoke; +- docs/API review сравнил публичные обещания с фактическими route shapes и + cross-project consumer contract. + +Форматирование и стиль намеренно не включались в findings: их обеспечивает +Ruff. Приоритет аудита — correctness, security, data integrity и доказуемое +recovery-поведение. From 94467b1102ec8f36f232624ccb9c1aacd94caa5b Mon Sep 17 00:00:00 2001 From: wackloner Date: Sun, 19 Jul 2026 17:00:39 +0700 Subject: [PATCH 07/10] remove public repository credibility drift --- BOARD.md | 3 +- Makefile | 2 +- README.md | 6 +- StakingRewards.sol | 144 ------------------ ...audit-architecture-financial-2026-07-19.md | 43 +++++- 5 files changed, 41 insertions(+), 157 deletions(-) delete mode 100644 StakingRewards.sol diff --git a/BOARD.md b/BOARD.md index b071f9d9..778d971e 100644 --- a/BOARD.md +++ b/BOARD.md @@ -8,7 +8,7 @@ - **Statuses**: `todo` | `in_progress` | `blocked` | `done` - **Priorities**: `critical` | `high` | `medium` | `low` - **Tags**: `security` | `backend` | `infra` | `dx` | `arch` | `perf` -- Next available ID: **CB-085** +- Next available ID: **CB-086** ## Active @@ -25,6 +25,7 @@ | CB-082 | Verify Mongo financial invariants | done | high | backend, infra | CI proves CAS replay, marker repair, BSON promotion, uniqueness, and lease fencing against a disposable pinned MongoDB | | CB-083 | Refresh public engineering docs | done | medium | dx, arch | Runtime commands, Python support, architecture boundaries, API shapes, and cross-project contract are current | | CB-084 | Disable unverified LP pricing | done | critical | backend, arch | Legacy raw-account-balance LP pricing is independently default-off until DEX economic reserves are verified | +| CB-085 | Remove repository credibility drift | done | medium | dx, arch | Public claims match verified behavior, local sync selects Python 3.12, and the unrelated EVM sample is removed | ## Completed milestones diff --git a/Makefile b/Makefile index 54caa8c4..2abd4d73 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ PYTHON_FORMAT_PATHS := \ sync: pipenv verify - pipenv sync --dev + pipenv sync --dev --python 3.12 run: pipenv run python app.py diff --git a/README.md b/README.md index a25f0bd4..7170eb45 100644 --- a/README.md +++ b/README.md @@ -74,8 +74,8 @@ The decision and extension rules are documented in Financial write paths are documented separately in [`docs/architecture/outbound-asset-transfers.md`](docs/architecture/outbound-asset-transfers.md) and [`docs/architecture/lp-projection.md`](docs/architecture/lp-projection.md). -The latest independent risk review, including resolved and intentionally open -items, is in +The latest internal multi-agent engineering review, including resolved and +intentionally open items, is in [`docs/audit/01-audit-architecture-financial-2026-07-19.md`](docs/audit/01-audit-architecture-financial-2026-07-19.md). ### Reliability boundaries @@ -192,7 +192,7 @@ flex/providers/ Market-data provider adapters flex/db/ MongoDB models, repositories, and indexes tests/unit/ Fast regression and boundary tests tests/integration/ Opt-in tests against disposable real services -scripts/ Deployment and safe operational utilities +scripts/ Deployment and legacy operational utilities ``` ## Configuration and security diff --git a/StakingRewards.sol b/StakingRewards.sol deleted file mode 100644 index 3d80d147..00000000 --- a/StakingRewards.sol +++ /dev/null @@ -1,144 +0,0 @@ -// SPDX-License-Identifier: MIT -pragma solidity ^0.8.24; - -contract StakingRewards { - IERC20 public immutable stakingToken; - IERC20 public immutable rewardsToken; - - address public owner; - - // Duration of rewards to be paid out (in seconds) - uint256 public duration; - // Timestamp of when the rewards finish - uint256 public finishAt; - // Minimum of last updated time and reward finish time - uint256 public updatedAt; - // Reward to be paid out per second - uint256 public rewardRate; - // Sum of (reward rate * dt * 1e18 / total supply) - uint256 public rewardPerTokenStored; - // User address => rewardPerTokenStored - mapping(address => uint256) public userRewardPerTokenPaid; - // User address => rewards to be claimed - mapping(address => uint256) public rewards; - - // Total staked - uint256 public totalSupply; - // User address => staked amount - mapping(address => uint256) public balanceOf; - - constructor(address _stakingToken, address _rewardToken) { - owner = msg.sender; - stakingToken = IERC20(_stakingToken); - rewardsToken = IERC20(_rewardToken); - } - - modifier onlyOwner() { - require(msg.sender == owner, "not authorized"); - _; - } - - modifier updateReward(address _account) { - rewardPerTokenStored = rewardPerToken(); - updatedAt = lastTimeRewardApplicable(); - - if (_account != address(0)) { - rewards[_account] = earned(_account); - userRewardPerTokenPaid[_account] = rewardPerTokenStored; - } - - _; - } - - function lastTimeRewardApplicable() public view returns (uint256) { - return _min(finishAt, block.timestamp); - } - - function rewardPerToken() public view returns (uint256) { - if (totalSupply == 0) { - return rewardPerTokenStored; - } - - return rewardPerTokenStored - + (rewardRate * (lastTimeRewardApplicable() - updatedAt) * 1e18) - / totalSupply; - } - - function stake(uint256 _amount) external updateReward(msg.sender) { - require(_amount > 0, "amount = 0"); - stakingToken.transferFrom(msg.sender, address(this), _amount); - balanceOf[msg.sender] += _amount; - totalSupply += _amount; - } - - function withdraw(uint256 _amount) external updateReward(msg.sender) { - require(_amount > 0, "amount = 0"); - balanceOf[msg.sender] -= _amount; - totalSupply -= _amount; - stakingToken.transfer(msg.sender, _amount); - } - - function earned(address _account) public view returns (uint256) { - return ( - ( - balanceOf[_account] - * (rewardPerToken() - userRewardPerTokenPaid[_account]) - ) / 1e18 - ) + rewards[_account]; - } - - function getReward() external updateReward(msg.sender) { - uint256 reward = rewards[msg.sender]; - if (reward > 0) { - rewards[msg.sender] = 0; - rewardsToken.transfer(msg.sender, reward); - } - } - - function setRewardsDuration(uint256 _duration) external onlyOwner { - require(finishAt < block.timestamp, "reward duration not finished"); - duration = _duration; - } - - function notifyRewardAmount(uint256 _amount) - external - onlyOwner - updateReward(address(0)) - { - if (block.timestamp >= finishAt) { - rewardRate = _amount / duration; - } else { - uint256 remainingRewards = (finishAt - block.timestamp) * rewardRate; - rewardRate = (_amount + remainingRewards) / duration; - } - - require(rewardRate > 0, "reward rate = 0"); - require( - rewardRate * duration <= rewardsToken.balanceOf(address(this)), - "reward amount > balance" - ); - - finishAt = block.timestamp + duration; - updatedAt = block.timestamp; - } - - function _min(uint256 x, uint256 y) private pure returns (uint256) { - return x <= y ? x : y; - } -} - -interface IERC20 { - function totalSupply() external view returns (uint256); - function balanceOf(address account) external view returns (uint256); - function transfer(address recipient, uint256 amount) - external - returns (bool); - function allowance(address owner, address spender) - external - view - returns (uint256); - function approve(address spender, uint256 amount) external returns (bool); - function transferFrom(address sender, address recipient, uint256 amount) - external - returns (bool); -} diff --git a/docs/audit/01-audit-architecture-financial-2026-07-19.md b/docs/audit/01-audit-architecture-financial-2026-07-19.md index b5b25b14..191cf19e 100644 --- a/docs/audit/01-audit-architecture-financial-2026-07-19.md +++ b/docs/audit/01-audit-architecture-financial-2026-07-19.md @@ -1,4 +1,4 @@ -# Аудит архитектуры и финансовой корректности +# Внутренний мультиагентный аудит архитектуры и финансовой корректности **Дата:** 2026-07-19 @@ -14,14 +14,25 @@ исправлений денежные операции переведены на integer base units, устойчивые business keys, immutable intents и compare-and-set проекции. -Оценка критических границ до/после: +## Executive summary -| Область | До | После | Комментарий | -| --- | ---: | ---: | --- | -| Финансовая корректность | 3/10 | 9/10 | replay-safe выплаты, точное распределение, fail-closed legacy | -| Конкурентность и recovery | 3/10 | 8/10 | CAS, marker repair, round leases, реальные Mongo-тесты | -| Security boundaries | 5/10 | 7/10 | секреты из кода вынесены; auth/signing требуют следующего milestone | -| Инженерная проверяемость | 6/10 | 9/10 | Python 3.12/3.14, strict typing, 262 теста, Mongo integration CI | +This internal review used separate agent roles for financial correctness, +MongoDB concurrency, CI and supply-chain posture, security and history, and +public API documentation. The branch replaces unsafe payout and LP state +transitions with durable intents, exact base-unit allocation, conditional +writes, fencing, and real-Mongo race tests. It deliberately keeps unverified +staking projection and legacy LP pricing disabled. Remaining authority, +async-persistence, and runtime-hardening work is listed below instead of hidden +behind a maturity score. + +Качественная сводка критических границ: + +| Область | Исходный риск | Текущий контроль | +| --- | --- | --- | +| Финансовая корректность | double-pay и float allocation | replay-safe выплаты, exact allocation, fail-closed legacy | +| Конкурентность и recovery | blind RMW и недоказанный replay | CAS, marker repair, round leases, реальные Mongo-тесты | +| Security boundaries | смешанные read/signing полномочия | секреты вне кода; auth/signing вынесены в следующий milestone | +| Инженерная проверяемость | mocks не доказывали BSON/Mongo | Python 3.12/3.14, strict typing, 262 теста, Mongo integration CI | MongoDB гарантирует атомарность одной операции над одним документом, поэтому проектор строится вокруг conditional update, а не blind read-modify-write. @@ -162,3 +173,19 @@ immutable-field conflict check и transactional outbox для уведомлен Форматирование и стиль намеренно не включались в findings: их обеспечивает Ruff. Приоритет аудита — correctness, security, data integrity и доказуемое recovery-поведение. + +## Воспроизводимость + +```bash +git log --oneline 2c1cdad..HEAD +make sync +make quality + +# Только disposable MongoDB, никогда production: +MONGODB_TEST_URI=mongodb://127.0.0.1:27017 \ + pipenv run pytest tests/integration -m integration -v +``` + +`make quality` проверен в чистом окружении Python 3.14.6; +production-equivalent `make sync` явно выбирает Python 3.12. CI повторяет обе +версии и включает real-Mongo job в стабильный required context `python`. From 022e26bbe2bd32db5244c06b1feb430440d0f85d Mon Sep 17 00:00:00 2001 From: wackloner Date: Sun, 19 Jul 2026 22:25:26 +0700 Subject: [PATCH 08/10] fix terminal payout index races --- .env.example | 1 + api/nft_lottery.py | 753 ++++++++++++++++-- docs/architecture/outbound-asset-transfers.md | 61 +- env.py | 11 +- flex/application/asset_transfers.py | 97 ++- flex/application/transfer_runtime.py | 3 + flex/blockchain/asset_transfers.py | 102 ++- flex/db/asset_transfer_intents.py | 69 +- flex/db/indexes.py | 73 +- flex/db/model/transfers.py | 61 +- flex/tools/airdrop.py | 36 +- .../test_mongo_asset_transfer_races.py | 245 ++++++ .../test_mongo_lottery_index_upgrade.py | 311 ++++++++ tests/unit/test_airdrop.py | 63 +- .../test_algorand_asset_transfer_gateway.py | 118 ++- tests/unit/test_asset_transfer_repository.py | 112 ++- tests/unit/test_asset_transfers.py | 145 ++++ tests/unit/test_database_indexes.py | 124 ++- tests/unit/test_nft_lottery_payouts.py | 639 ++++++++++++++- 19 files changed, 2888 insertions(+), 136 deletions(-) create mode 100644 tests/integration/test_mongo_asset_transfer_races.py create mode 100644 tests/integration/test_mongo_lottery_index_upgrade.py diff --git a/.env.example b/.env.example index 3ccebe70..1722a701 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,7 @@ REKEYED_MNEMONIC= ALGOD_ADDRESS=http://algod:8080 ALGOD_TOKEN= ALGO_INDEXER_ADDRESS=https://mainnet-idx.algonode.cloud +OUTBOUND_ASSET_TRANSFER_MAX_FEE_MICROALGOS=1000 # API authentication API_PASSWORD= diff --git a/api/nft_lottery.py b/api/nft_lottery.py index c2aab2f0..785b5402 100644 --- a/api/nft_lottery.py +++ b/api/nft_lottery.py @@ -1,8 +1,9 @@ import logging +import math import random import time from dataclasses import dataclass -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from enum import StrEnum from hashlib import sha256 from typing import Any @@ -10,7 +11,8 @@ from algosdk.v2client import indexer from dataclasses_json import dataclass_json -from pymongo import ReturnDocument +from pymongo import ASCENDING, ReturnDocument +from pymongo.errors import DuplicateKeyError, OperationFailure import flex from api.swaps import SwapInfo @@ -18,10 +20,19 @@ from blockchain.nfts import get_nft_info from blockchain.node import init_algod_client from core.db.db_manager import DbManager +from core.db.mongodb import get_db_collection from env import settings from flex.blockchain.base import cometa_public_key MIN_DRAW_INTERVAL = 60 * 60 * 24 # 24 hours +LOTTERY_DRAW_ID_INDEX_NAME = "id_nonempty_unique" +LOTTERY_DRAW_ID_FILTER = { + "id": { + "$type": "string", + "$gt": "", + } +} +LEGACY_LOTTERY_DRAW_ID_FILTER = {"id": {"$type": "string"}} class LotteryType(StrEnum): @@ -32,6 +43,23 @@ def __str__(self): return self.value +class LotteryPayoutStatus(StrEnum): + PENDING = "pending" + PREPARED = "prepared" + CONFIRMED = "confirmed" + UNRESOLVED = "unresolved" + RECONCILIATION_REQUIRED = "reconciliation_required" + + +class StakingEntitlementStatus(StrEnum): + RESERVED = "reserved" + PREPARED = "prepared" + MATERIALIZED = "materialized" + CONFIRMED = "confirmed" + NO_PRIZE = "no_prize" + UNRESOLVED = "unresolved" + + @dataclass_json @dataclass class NftLottery: @@ -68,6 +96,9 @@ class LotteryDraw: payout_operation_id: str | None = None payout_txid: str | None = None confirmed_round: int | None = None + payout_status: str | None = None + entitlement_id: str | None = None + entitlement_generation: int | None = None def __post_init__(self): if self.timestamp: @@ -87,6 +118,10 @@ class LotteryParticipant: lottery_participants = DbManager[LotteryParticipant]( settings.db_name, "lottery_participants", "address", LotteryParticipant ) +lottery_entitlements = get_db_collection( + settings.db_name, + "lottery_entitlements", +) algod_client = init_algod_client() indexer_client = indexer.IndexerClient( @@ -96,15 +131,127 @@ class LotteryParticipant: logger = logging.getLogger(__name__) -def ensure_lottery_indexes() -> None: - lottery_draws.collection.create_index( - "id", - unique=True, - name="id_unique", - partialFilterExpression={"id": {"$type": "string"}}, +def _is_lottery_draw_id_index( + index: dict[str, Any], + *, + partial_filter: dict[str, Any], +) -> bool: + return ( + list(index.get("key", [])) == [("id", ASCENDING)] + and index.get("unique") is True + and index.get("partialFilterExpression") == partial_filter + ) + + +def _duplicate_lottery_draw_ids() -> list[dict[str, Any]]: + return list( + lottery_draws.collection.aggregate( + [ + {"$match": LOTTERY_DRAW_ID_FILTER}, + { + "$group": { + "_id": "$id", + "count": {"$sum": 1}, + "document_ids": {"$push": "$_id"}, + } + }, + {"$match": {"count": {"$gt": 1}}}, + {"$limit": 10}, + ], + allowDiskUse=True, + ) ) +def ensure_lottery_indexes() -> None: + """Upgrade the draw identity index without a uniqueness gap or data loss.""" + + collection = lottery_draws.collection + indexes = collection.index_information() + desired_index = indexes.get(LOTTERY_DRAW_ID_INDEX_NAME) + legacy_index = indexes.get("id_unique") + + if desired_index is not None and not _is_lottery_draw_id_index( + desired_index, + partial_filter=LOTTERY_DRAW_ID_FILTER, + ): + raise RuntimeError( + f"lottery_draws index {LOTTERY_DRAW_ID_INDEX_NAME!r} has unexpected options; " + "inspect it manually before startup" + ) + + if legacy_index is not None and not ( + _is_lottery_draw_id_index( + legacy_index, + partial_filter=LEGACY_LOTTERY_DRAW_ID_FILTER, + ) + or _is_lottery_draw_id_index( + legacy_index, + partial_filter=LOTTERY_DRAW_ID_FILTER, + ) + ): + raise RuntimeError("lottery_draws index 'id_unique' has unexpected options; inspect it manually before startup") + + if desired_index is None and not ( + legacy_index is not None + and _is_lottery_draw_id_index( + legacy_index, + partial_filter=LOTTERY_DRAW_ID_FILTER, + ) + ): + duplicate_groups = _duplicate_lottery_draw_ids() + if duplicate_groups: + raise RuntimeError( + "lottery_draws contains duplicate non-empty draw IDs; " + f"preserved {len(duplicate_groups)} duplicate group(s) for explicit reconciliation" + ) + try: + collection.create_index( + "id", + unique=True, + name=LOTTERY_DRAW_ID_INDEX_NAME, + partialFilterExpression=LOTTERY_DRAW_ID_FILTER, + ) + except DuplicateKeyError as exc: + raise RuntimeError( + "lottery_draws uniqueness changed while building its index; " + "all conflicting records were preserved for explicit reconciliation" + ) from exc + except OperationFailure as exc: + if exc.code != 11000: + raise + raise RuntimeError( + "lottery_draws uniqueness changed while building its index; " + "all conflicting records were preserved for explicit reconciliation" + ) from exc + desired_index = collection.index_information().get(LOTTERY_DRAW_ID_INDEX_NAME) + if desired_index is None or not _is_lottery_draw_id_index( + desired_index, + partial_filter=LOTTERY_DRAW_ID_FILTER, + ): + raise RuntimeError("lottery_draws unique draw ID index was not installed") + + if legacy_index is not None and _is_lottery_draw_id_index( + legacy_index, + partial_filter=LEGACY_LOTTERY_DRAW_ID_FILTER, + ): + # The replacement already protects every non-empty ID. Dropping the + # narrower legacy definition now cannot introduce a uniqueness gap. + try: + collection.drop_index("id_unique") + except OperationFailure as exc: + if exc.code != 27: + raise + else: + logger.info("Upgraded lottery_draws.id unique index without modifying draw records") + replacement = collection.index_information().get(LOTTERY_DRAW_ID_INDEX_NAME) + if replacement is None or not _is_lottery_draw_id_index( + replacement, + partial_filter=LOTTERY_DRAW_ID_FILTER, + ): + raise RuntimeError("lottery_draws replacement identity index disappeared during the legacy index upgrade") + + def _create_draw( *, lottery_name: str, @@ -120,38 +267,465 @@ def _create_draw( prize=prize, wallet=wallet, timestamp=timestamp, + payout_status=LotteryPayoutStatus.PENDING, ) ) +def _staking_entitlement_id(lottery_name: str, wallet: str) -> str: + identity = f"staking-lottery:{lottery_name}:{wallet}" + return sha256(identity.encode()).hexdigest() + + +def _valid_draw_timestamp(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + return None + return float(value) + + +def _seed_staking_entitlement( + *, + lottery_name: str, + wallet: str, + now: datetime, +) -> str: + """Initialize the CAS guard from any pre-migration draw.""" + + entitlement_id = _staking_entitlement_id(lottery_name, wallet) + seed: dict[str, Any] = { + "_id": entitlement_id, + "lottery_name": lottery_name, + "wallet": wallet, + "generation": 0, + "next_eligible_at": datetime.fromtimestamp(0, tz=UTC), + "created": now, + "updated": now, + } + latest = lottery_draws.collection.find_one( + { + "wallet": wallet, + "lottery_name": lottery_name, + }, + sort=[("timestamp", -1)], + ) + outstanding = lottery_draws.collection.find_one( + { + "wallet": wallet, + "lottery_name": lottery_name, + "prize": {"$ne": None}, + "claimed": {"$in": [False, None]}, + }, + sort=[("timestamp", -1)], + ) + active_document = outstanding or latest + if active_document is not None: + draw, _ = _backfill_draw_id(active_document) + active_timestamp = _valid_draw_timestamp(draw.timestamp) + latest_timestamp = _valid_draw_timestamp(latest.get("timestamp")) if latest is not None else active_timestamp + if active_timestamp is None or latest_timestamp is None: + draw_timestamp = now.timestamp() + status = StakingEntitlementStatus.UNRESOLVED + next_eligible_at = now + timedelta(seconds=MIN_DRAW_INTERVAL) + else: + draw_timestamp = active_timestamp + status = ( + StakingEntitlementStatus.NO_PRIZE + if draw.prize is None + else (StakingEntitlementStatus.CONFIRMED if draw.claimed else StakingEntitlementStatus.UNRESOLVED) + ) + try: + next_eligible_at = datetime.fromtimestamp( + latest_timestamp, + tz=UTC, + ) + timedelta(seconds=MIN_DRAW_INTERVAL) + except (OverflowError, OSError, ValueError): + draw_timestamp = now.timestamp() + status = StakingEntitlementStatus.UNRESOLVED + next_eligible_at = now + timedelta(seconds=MIN_DRAW_INTERVAL) + seed["next_eligible_at"] = next_eligible_at + seed["active"] = { + "draw_id": draw.id, + "draw_timestamp": draw_timestamp, + "prize": draw.prize, + "status": status, + } + + try: + lottery_entitlements.update_one( + {"_id": entitlement_id}, + {"$setOnInsert": seed}, + upsert=True, + ) + except DuplicateKeyError: + # Another worker installed the same single-document guard. + pass + return entitlement_id + + +def _claim_staking_entitlement( + *, + lottery_name: str, + wallet: str, + now: datetime, +) -> dict[str, Any] | None: + """Claim one rolling-window liability or recover its unfinished draw.""" + + entitlement_id = _seed_staking_entitlement( + lottery_name=lottery_name, + wallet=wallet, + now=now, + ) + draw_id_value = uuid4().hex + draw_timestamp = now.timestamp() + claimed = lottery_entitlements.find_one_and_update( + { + "_id": entitlement_id, + "next_eligible_at": {"$lte": now}, + "$or": [ + {"active": {"$exists": False}}, + { + "active.status": { + "$in": [ + StakingEntitlementStatus.CONFIRMED, + StakingEntitlementStatus.NO_PRIZE, + ] + } + }, + ], + }, + { + "$inc": {"generation": 1}, + "$set": { + "next_eligible_at": now + timedelta(seconds=MIN_DRAW_INTERVAL), + "active": { + "draw_id": draw_id_value, + "draw_timestamp": draw_timestamp, + "reserved_at": now, + "status": StakingEntitlementStatus.RESERVED, + }, + "updated": now, + }, + }, + return_document=ReturnDocument.AFTER, + ) + if claimed is not None: + return claimed + + current = lottery_entitlements.find_one({"_id": entitlement_id}) + if not isinstance(current, dict): + raise RuntimeError("staking lottery entitlement disappeared") + active = current.get("active") + if isinstance(active, dict) and active.get("status") in { + StakingEntitlementStatus.MATERIALIZED, + StakingEntitlementStatus.UNRESOLVED, + }: + active_draw_id = active.get("draw_id") + draw_document = ( + lottery_draws.collection.find_one({"id": active_draw_id}) if isinstance(active_draw_id, str) else None + ) + if ( + isinstance(draw_document, dict) + and draw_document.get("claimed") is True + and draw_document.get("payout_status") == LotteryPayoutStatus.CONFIRMED + ): + repaired = lottery_entitlements.find_one_and_update( + { + "_id": entitlement_id, + "generation": current["generation"], + "active.draw_id": active_draw_id, + "active.status": active["status"], + }, + { + "$set": { + "active.status": StakingEntitlementStatus.CONFIRMED, + "updated": now, + } + }, + return_document=ReturnDocument.AFTER, + ) + if repaired is not None: + return _claim_staking_entitlement( + lottery_name=lottery_name, + wallet=wallet, + now=now, + ) + if isinstance(active, dict) and active.get("status") in { + StakingEntitlementStatus.RESERVED, + StakingEntitlementStatus.PREPARED, + }: + # A crash or competing request may leave the one winning reservation + # unfinished. Helping it cannot create a second business entitlement. + return current + return None + + +def _prepare_staking_prize( + entitlement: dict[str, Any], + lottery: NftLottery, +) -> dict[str, Any]: + active = entitlement.get("active") + if not isinstance(active, dict) or not isinstance(active.get("draw_id"), str): + raise RuntimeError("staking lottery entitlement has no active draw") + if active.get("status") == StakingEntitlementStatus.RESERVED: + prize_id = draw_id(lottery) + prepared = lottery_entitlements.find_one_and_update( + { + "_id": entitlement["_id"], + "generation": entitlement["generation"], + "active.draw_id": active["draw_id"], + "active.status": StakingEntitlementStatus.RESERVED, + }, + { + "$set": { + "active.prize": prize_id, + "active.status": StakingEntitlementStatus.PREPARED, + "updated": datetime.now(UTC), + } + }, + return_document=ReturnDocument.AFTER, + ) + if prepared is not None: + return prepared + entitlement = lottery_entitlements.find_one({"_id": entitlement["_id"]}) + if not isinstance(entitlement, dict): + raise RuntimeError("staking lottery entitlement disappeared during prize reservation") + + active = entitlement.get("active") + if not isinstance(active, dict) or active.get("status") not in { + StakingEntitlementStatus.PREPARED, + StakingEntitlementStatus.MATERIALIZED, + StakingEntitlementStatus.NO_PRIZE, + }: + raise RuntimeError("staking lottery prize reservation is not recoverable") + return entitlement + + +def _materialize_staking_draw( + entitlement: dict[str, Any], + lottery: NftLottery, + wallet: str, +) -> LotteryDraw: + active = entitlement["active"] + draw_id_value = active["draw_id"] + draw_timestamp = active["draw_timestamp"] + prize_id = active.get("prize") + generation = entitlement["generation"] + ensure_lottery_indexes() + payload = LotteryDraw( + id=draw_id_value, + lottery_name=lottery.name, + prize=prize_id, + wallet=wallet, + timestamp=draw_timestamp, + payout_status=LotteryPayoutStatus.PENDING, + entitlement_id=entitlement["_id"], + entitlement_generation=generation, + ).to_dict() + try: + document = lottery_draws.collection.find_one_and_update( + {"id": draw_id_value}, + {"$setOnInsert": payload}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + except DuplicateKeyError: + document = lottery_draws.collection.find_one({"id": draw_id_value}) + if not isinstance(document, dict): + raise RuntimeError("staking lottery draw could not be materialized") + immutable = ( + document.get("id"), + document.get("lottery_name"), + document.get("wallet"), + document.get("prize"), + document.get("timestamp"), + document.get("entitlement_id"), + document.get("entitlement_generation"), + ) + expected = ( + draw_id_value, + lottery.name, + wallet, + prize_id, + draw_timestamp, + entitlement["_id"], + generation, + ) + if immutable != expected: + raise RuntimeError("staking lottery draw ID belongs to different immutable data") + + status = StakingEntitlementStatus.NO_PRIZE if prize_id is None else StakingEntitlementStatus.MATERIALIZED + updated = lottery_entitlements.update_one( + { + "_id": entitlement["_id"], + "generation": generation, + "active.draw_id": draw_id_value, + "active.status": { + "$in": [ + StakingEntitlementStatus.PREPARED, + status, + ] + }, + }, + { + "$set": { + "active.status": status, + "updated": datetime.now(UTC), + } + }, + ) + if updated.matched_count != 1: + current = lottery_entitlements.find_one({"_id": entitlement["_id"]}) + current_active = current.get("active") if isinstance(current, dict) else None + if not ( + isinstance(current_active, dict) + and current_active.get("draw_id") == draw_id_value + and current_active.get("status") + in { + status, + StakingEntitlementStatus.CONFIRMED, + StakingEntitlementStatus.UNRESOLVED, + } + ): + raise RuntimeError("staking lottery entitlement changed during draw materialization") + return LotteryDraw.from_dict(document) + + +def _get_or_create_staking_draw( + lottery: NftLottery, + wallet: str, + *, + now: datetime, +) -> LotteryDraw | None: + entitlement = _claim_staking_entitlement( + lottery_name=lottery.name, + wallet=wallet, + now=now, + ) + if entitlement is None: + return None + prepared = _prepare_staking_prize(entitlement, lottery) + return _materialize_staking_draw(prepared, lottery, wallet) + + +def _update_staking_entitlement_status( + draw: LotteryDraw, + status: StakingEntitlementStatus, +) -> None: + if draw.entitlement_id is None or draw.entitlement_generation is None or draw.id is None: + return + selector: dict[str, Any] = { + "_id": draw.entitlement_id, + "generation": draw.entitlement_generation, + "active.draw_id": draw.id, + } + if status != StakingEntitlementStatus.CONFIRMED: + selector["active.status"] = { + "$ne": StakingEntitlementStatus.CONFIRMED, + } + try: + result = lottery_entitlements.update_one( + selector, + { + "$set": { + "active.status": status, + "updated": datetime.now(UTC), + } + }, + ) + except Exception: + logger.exception( + "Could not update staking entitlement %s generation %s to %s; future draws remain fail-closed", + draw.entitlement_id, + draw.entitlement_generation, + status, + ) + return + if result.matched_count != 1: + current = lottery_entitlements.find_one( + { + "_id": draw.entitlement_id, + "generation": draw.entitlement_generation, + "active.draw_id": draw.id, + } + ) + current_active = current.get("active") if isinstance(current, dict) else None + if isinstance(current_active, dict) and current_active.get("status") in { + status, + StakingEntitlementStatus.CONFIRMED, + }: + return + logger.error( + "Could not update staking entitlement %s generation %s to %s; future draws remain fail-closed", + draw.entitlement_id, + draw.entitlement_generation, + status, + ) + + def _backfill_draw_id(document: dict[str, Any]) -> tuple[LotteryDraw, Any]: document_id = document.get("_id") if document_id is None: raise RuntimeError("lottery draw is missing its MongoDB identity") - draw_id_value = document.get("id") - if not isinstance(draw_id_value, str) or not draw_id_value: - draw_id_value = f"legacy-{sha256(f'lottery-draw:{document_id}'.encode()).hexdigest()}" - updated = lottery_draws.collection.find_one_and_update( + for attempt in range(4): + original_id = document.get("id") + missing_draw_id = not isinstance(original_id, str) or not original_id + draw_id_value = ( + f"legacy-{sha256(f'lottery-draw:{document_id}'.encode()).hexdigest()}" if missing_draw_id else original_id + ) + original_status = document.get("payout_status") + missing_payout_status = not isinstance(original_status, str) or not original_status + claimed_value = document.get("claimed") + missing_claimed = "claimed" not in document or claimed_value is None + if claimed_value is not None and not isinstance(claimed_value, bool): + raise RuntimeError("lottery draw has an invalid claimed state") + if not missing_draw_id and not missing_payout_status and not missing_claimed: + break + if attempt == 3: + raise RuntimeError("lottery draw kept changing during migration") + + expected_operation_id = f"nft:lottery:{draw_id_value}" + payout_status = ( + LotteryPayoutStatus.CONFIRMED + if claimed_value is True + else ( + LotteryPayoutStatus.PREPARED + if document.get("payout_operation_id") == expected_operation_id + else LotteryPayoutStatus.RECONCILIATION_REQUIRED + ) + ) + migration_query: dict[str, Any] = {"_id": document_id} + for field_name in ( + "id", + "payout_status", + "claimed", + "payout_operation_id", + ): + migration_query[field_name] = ( + {"$eq": document[field_name]} if field_name in document else {"$exists": False} + ) + migrated = lottery_draws.collection.find_one_and_update( + migration_query, { - "_id": document_id, - "$or": [ - {"id": {"$exists": False}}, - {"id": None}, - ], + "$set": { + "id": draw_id_value, + "payout_status": payout_status, + **({"claimed": False} if missing_claimed else {}), + } }, - {"$set": {"id": draw_id_value}}, return_document=ReturnDocument.AFTER, ) + if migrated is not None: + document = migrated + break + updated = lottery_draws.collection.find_one({"_id": document_id}) if updated is None: - updated = lottery_draws.collection.find_one({"_id": document_id}) - if updated is None or updated.get("id") != draw_id_value: - raise RuntimeError("lottery draw identity changed during migration") + raise RuntimeError("lottery draw disappeared during migration") document = updated draw = LotteryDraw.from_dict(document) - if draw.id is None: - raise RuntimeError("lottery draw migration did not persist an ID") + if not isinstance(draw.id, str) or not draw.id or not draw.payout_status or not isinstance(draw.claimed, bool): + raise RuntimeError("lottery draw migration did not persist its identity and payout state") return draw, document_id @@ -260,22 +834,15 @@ async def lottery_for_staking(pool_id: int, address: str) -> NftPrize | None: logger.info(f"Lottery {lottery.name} for pool {pool_id} and address {address} started") - address_draws = lottery_draws.get_many({"wallet": address, "lottery_name": lottery.name}) - now_timestamp = time.time() - # TODO: optimize the check, get only last timestamp - if len(address_draws) > 0: - last_draw_timestamp = max([d.timestamp for d in address_draws]) - if now_timestamp - last_draw_timestamp < MIN_DRAW_INTERVAL: - logger.info(f"Lottery {lottery.name} for pool {pool_id} and address {address} already drawn recently") - return None - - prize_id = draw_id(lottery) - _create_draw( - lottery_name=lottery.name, - prize=prize_id, - wallet=address, - timestamp=now_timestamp, + draw = _get_or_create_staking_draw( + lottery, + address, + now=datetime.now(UTC), ) + if draw is None: + logger.info(f"Lottery {lottery.name} for pool {pool_id} and address {address} already drawn recently") + return None + prize_id = draw.prize logger.info(f"The prize is {prize_id}") @@ -294,26 +861,96 @@ def send_all_prizes(): res = [] sent_count = 0 error_count = 0 - documents = list(lottery_draws.collection.find({"claimed": False, "prize": {"$ne": None}})) + documents = list( + lottery_draws.collection.find( + { + "claimed": {"$in": [False, None]}, + "prize": {"$ne": None}, + } + ) + ) for document in documents: - draw, document_id = _backfill_draw_id(document) + try: + draw, document_id = _backfill_draw_id(document) + except Exception as exc: + document_id = document.get("_id") + info = { + "wallet": document.get("wallet"), + "prize": document.get("prize"), + "lottery": document.get("lottery_name"), + "error": f"lottery draw migration failed: {exc}", + } + if document_id is not None: + lottery_draws.collection.update_one( + {"_id": document_id, "claimed": False}, + {"$set": {"send_error": str(info["error"])[:500]}}, + ) + error_count += 1 + logger.exception("Skipped malformed lottery draw") + res.append(info) + continue if draw.claimed or draw.prize is None: continue info = {"wallet": draw.wallet, "prize": draw.prize, "lottery": draw.lottery_name} + payable_statuses = ( + LotteryPayoutStatus.PENDING, + LotteryPayoutStatus.PREPARED, + LotteryPayoutStatus.UNRESOLVED, + ) + if draw.payout_status not in payable_statuses: + error = ( + "legacy lottery payout requires manual reconciliation" + if draw.payout_status == LotteryPayoutStatus.RECONCILIATION_REQUIRED + else f"lottery payout has non-payable status {draw.payout_status!r}" + ) + info["error"] = error + lottery_draws.collection.update_one( + { + "_id": document_id, + "claimed": False, + "payout_status": draw.payout_status, + }, + {"$set": {"send_error": error}}, + ) + error_count += 1 + logger.warning("Skipped non-payable NFT payout: %s", info) + res.append(info) + continue try: idempotency_key = f"lottery:{draw.id}" payout_operation_id = f"nft:{idempotency_key}" if draw.payout_operation_id not in (None, payout_operation_id): raise RuntimeError("lottery draw belongs to a different payout operation") - lottery_draws.collection.update_one( - {"_id": document_id, "claimed": False}, + prepared = lottery_draws.collection.update_one( + { + "_id": document_id, + "claimed": False, + "payout_operation_id": {"$in": [None, payout_operation_id]}, + "payout_status": { + "$in": list(payable_statuses), + }, + }, { "$set": { "payout_operation_id": payout_operation_id, + "payout_status": LotteryPayoutStatus.PREPARED, "send_error": None, } }, ) + if prepared.matched_count != 1: + current = lottery_draws.collection.find_one({"_id": document_id}) + if current is not None and current.get("claimed") is True: + info["already_claimed"] = True + info["txid"] = current.get("payout_txid") + _update_staking_entitlement_status( + LotteryDraw.from_dict(current), + StakingEntitlementStatus.CONFIRMED, + ) + logger.info("NFT payout completed by another worker: %s", info) + res.append(info) + continue + raise RuntimeError("lottery draw changed before payout reservation") receipt = send_nft( draw.wallet, draw.prize, @@ -322,13 +959,21 @@ def send_all_prizes(): info["txid"] = receipt.txid info["sent"] = datetime.now(UTC) claimed = lottery_draws.collection.find_one_and_update( - {"_id": document_id, "claimed": False}, + { + "_id": document_id, + "claimed": False, + "payout_operation_id": receipt.operation_id, + "payout_status": { + "$ne": LotteryPayoutStatus.RECONCILIATION_REQUIRED, + }, + }, { "$set": { "claimed": True, "payout_operation_id": receipt.operation_id, "payout_txid": receipt.txid, "confirmed_round": receipt.confirmed_round, + "payout_status": LotteryPayoutStatus.CONFIRMED, "send_error": None, } }, @@ -341,11 +986,29 @@ def send_all_prizes(): info["already_claimed"] = True else: sent_count += 1 + _update_staking_entitlement_status( + LotteryDraw.from_dict(claimed), + StakingEntitlementStatus.CONFIRMED, + ) except Exception as e: info["error"] = str(e) + _update_staking_entitlement_status( + draw, + StakingEntitlementStatus.UNRESOLVED, + ) lottery_draws.collection.update_one( - {"_id": document_id, "claimed": False}, - {"$set": {"send_error": str(e)[:500]}}, + { + "_id": document_id, + "claimed": False, + "payout_operation_id": payout_operation_id, + "payout_status": {"$ne": LotteryPayoutStatus.RECONCILIATION_REQUIRED}, + }, + { + "$set": { + "payout_status": LotteryPayoutStatus.UNRESOLVED, + "send_error": str(e)[:500], + } + }, ) error_count += 1 diff --git a/docs/architecture/outbound-asset-transfers.md b/docs/architecture/outbound-asset-transfers.md index 18751117..59a29c16 100644 --- a/docs/architecture/outbound-asset-transfers.md +++ b/docs/architecture/outbound-asset-transfers.md @@ -12,14 +12,20 @@ Before the first broadcast, the service stores: A retry loads and rebroadcasts the same signed payload. Before network I/O, the adapter verifies its signature, transaction ID, sender, receiver, ASA, amount, -note, lease, validity window, and absence of close, clawback, group, or rekey -fields. This closes the crash window where Algorand accepted a transaction but -MongoDB did not record the result and prevents a corrupted intent from sending -another valid treasury transaction. +note, lease, validity window, fee ceiling, configured network genesis, and +absence of close, clawback, group, or rekey fields. Suggested parameters are +normalized to a flat protocol-minimum fee only after Algod's reported minimum +is at least the protocol floor and both fee ceiling and genesis match policy. +This closes the crash window where Algorand accepted a +transaction but MongoDB did not record the result, prevents node-provided fees +from draining the signer, and stops a corrupted intent from sending another +valid treasury transaction. -Algorand uint64 values are stored as decimal strings because BSON integers are -signed int64. Conflicting immutable intent or manifest IDs abort index setup; -the application never deletes financial evidence automatically. +ASA IDs and amounts remain decimal strings for compatibility; validity and +confirmation rounds use BSON Decimal128 codecs. Both representations preserve +the full Algorand uint64 domain rather than relying on signed BSON int64. +Conflicting immutable intent or manifest IDs abort index setup; the application +never deletes financial evidence automatically. A deterministic Algorand lease adds defense in depth, but persistence of the exact signed transaction is the primary idempotency mechanism. If the @@ -34,6 +40,12 @@ reserves an immutable SHA-256 manifest covering the asset, total, complete recipient set, allocations, and selected notes. Reusing an `airdrop_id` with a different manifest fails before any broadcast. +`send_airdrop` is also safe when invoked directly by an operator script rather +than through FastAPI startup: before any reward read or signing, it fail-closes +on duplicate manifest IDs or reward operation IDs and installs their unique +indexes. Concurrent workers therefore converge on one reward record as well as +one transfer intent. + Legacy campaigns have no trustworthy complete-recipient manifest. They therefore fail closed until explicitly reviewed and migrated. Legacy reward transactions must be confirmed on-chain and match the stored sender, receiver, @@ -42,11 +54,36 @@ ASA, and amount before they can be marked complete. Operational retries must always reuse the original operation or airdrop ID. `AirdropIncompleteError` reports unresolved recipients and transaction IDs; never invent a replacement ID to bypass reconciliation. +`complete` is terminal: a slower worker cannot overwrite it with `partial`. +Likewise, attempt, submitted, and error updates cannot regress a confirmed +transfer intent. ## Lottery payouts -Every lottery draw receives an immutable ID. Legacy draws are assigned a stable -ID derived from their MongoDB identity before payment. The payout intent is -stored against that ID, and the exact draw is marked claimed with a conditional -update only after confirmation. Multiple workers may race safely: they resolve -to the same persisted transaction and cannot update another draw. +Every new lottery draw receives an immutable ID and starts in `pending`. The +payout moves through `prepared` to `confirmed`; uncertain attempts remain +`unresolved` and reuse the same operation. + +Staking lotteries first claim a single entitlement document keyed by +`(lottery_name, wallet)`. A compare-and-set advances its rolling 24-hour window, +generation, and active draw ID atomically. Prize selection and draw insertion +are replay-repaired from that reservation, so crashes and competing workers +converge on one liability. An unresolved prize blocks the next generation until +reconciliation; no-prize and confirmed generations are terminal. + +Pre-intent legacy draws are different: an old worker may have broadcast and +crashed before recording `claimed`. They receive a deterministic ID but move to +`reconciliation_required`, never directly to payment. Automatic resume is +allowed only when the draw already references the exact durable operation ID, +which proves that any broadcast used the persisted-intent path. Manual +reconciliation must otherwise attach verified on-chain evidence or explicitly +authorize an unpaid draw. + +The exact draw is marked claimed with a conditional update only after +confirmation. Multiple workers may race safely: they resolve to the same +persisted transaction and cannot update another draw. + +Lottery inventory itself is still a separate authority boundary. A future +re-enabled lottery must reserve each one-of-one NFT atomically before exposing +the feature; the public lottery routes remain disabled until that control and +product reconciliation are reviewed. diff --git a/env.py b/env.py index 86ff49c6..60dde2cc 100644 --- a/env.py +++ b/env.py @@ -16,6 +16,11 @@ class Settings(BaseSettings): algod_address: str algod_token: str algo_indexer_address: str + outbound_asset_transfer_max_fee_microalgos: int = Field( + default=1_000, + ge=1_000, + le=1_000_000, + ) server_port: int workers_num: int @@ -57,9 +62,8 @@ class Settings(BaseSettings): background_user_pools_update: bool = False background_pools_update: bool = False background_asset_prices_update: bool = True # Enable background update of asset prices - # Legacy LP pricing uses raw pool-account balances, which can include - # donations or protocol excess. Keep it off until each DEX has a verified - # economic-reserve adapter. + # Retained only so existing deployments can roll forward. The raw-balance + # publisher has been removed and this flag cannot enable LP pricing. background_lp_prices_update: bool = False asset_price_update_batch_size: int = Field(default=10, gt=0) asset_price_api_call_delay: float = Field(default=1, ge=0) @@ -91,7 +95,6 @@ class Settings(BaseSettings): asset_prices_ttl: int = Field(default=120, gt=0) # 2 minutes. asset_prices_max_stale: int = Field(default=3600, gt=0) asset_prices_update_interval: int = 60 # Run the background update every 60 seconds - lp_prices_update_interval: int = 300 # LP pricing via algod every 5 minutes lp_token_prices_ttl: int = 30 total_tvl_ttl: int = 30 diff --git a/flex/application/asset_transfers.py b/flex/application/asset_transfers.py index e17859df..e8774eea 100644 --- a/flex/application/asset_transfers.py +++ b/flex/application/asset_transfers.py @@ -4,8 +4,11 @@ from typing import Protocol from flex.db.model.transfers import AssetTransferIntent +from flex.domain.algorand import ( + MAX_ALGORAND_UINT, + require_algorand_uint64, +) -MAX_ALGORAND_UINT = 2**64 - 1 MAX_NOTE_BYTES = 1_000 MAX_OPERATION_ID_BYTES = 200 @@ -85,6 +88,23 @@ class PreparedAssetTransfer: first_valid_round: int last_valid_round: int + def __post_init__(self) -> None: + try: + require_algorand_uint64( + self.first_valid_round, + "first_valid_round", + ) + require_algorand_uint64( + self.last_valid_round, + "last_valid_round", + ) + except ValueError as exc: + raise InvalidAssetTransferError(str(exc)) from exc + if self.first_valid_round > self.last_valid_round: + raise InvalidAssetTransferError( + "first_valid_round cannot exceed last_valid_round", + ) + @dataclass(frozen=True, slots=True) class AssetTransferReceipt: @@ -93,6 +113,15 @@ class AssetTransferReceipt: confirmed_round: int already_confirmed: bool + def __post_init__(self) -> None: + try: + require_algorand_uint64( + self.confirmed_round, + "confirmed_round", + ) + except ValueError as exc: + raise InvalidAssetTransferError(str(exc)) from exc + @dataclass(frozen=True, slots=True) class ConfirmedAssetTransfer: @@ -103,6 +132,25 @@ class ConfirmedAssetTransfer: amount_micros: int confirmed_round: int + def __post_init__(self) -> None: + try: + require_algorand_uint64( + self.asset_id, + "asset_id", + positive=True, + ) + require_algorand_uint64( + self.amount_micros, + "amount_micros", + positive=True, + ) + require_algorand_uint64( + self.confirmed_round, + "confirmed_round", + ) + except ValueError as exc: + raise InvalidAssetTransferError(str(exc)) from exc + @dataclass(frozen=True, slots=True) class TransferReconciliation: @@ -203,11 +251,13 @@ def execute(self, request: AssetTransferRequest) -> AssetTransferReceipt: try: confirmed_round = self.gateway.lookup_confirmed_round(intent.txid) except Exception as exc: - self.repository.record_error( + intent = self.repository.record_error( intent.id, intent.txid, f"expired transaction lookup unavailable after {type(exc).__name__}", ) + if intent.status == TransferStatus.CONFIRMED: + return self._confirmed_receipt(intent, already_confirmed=True) raise AssetTransferExpiredError(intent.id, intent.txid) from exc if confirmed_round is not None: intent = self.repository.mark_confirmed( @@ -217,14 +267,18 @@ def execute(self, request: AssetTransferRequest) -> AssetTransferReceipt: ) return self._confirmed_receipt(intent, already_confirmed=True) - self.repository.record_error( + intent = self.repository.record_error( intent.id, intent.txid, "signed transaction expired before confirmation; manual reconciliation required", ) + if intent.status == TransferStatus.CONFIRMED: + return self._confirmed_receipt(intent, already_confirmed=True) raise AssetTransferExpiredError(intent.id, intent.txid) - self.repository.record_attempt(intent.id, intent.txid) + intent = self.repository.record_attempt(intent.id, intent.txid) + if intent.status == TransferStatus.CONFIRMED: + return self._confirmed_receipt(intent, already_confirmed=True) broadcast_error: Exception | None = None try: returned_txid = self.gateway.broadcast( @@ -238,7 +292,9 @@ def execute(self, request: AssetTransferRequest) -> AssetTransferReceipt: ) if returned_txid != intent.txid: raise AssetTransferError("Algorand node returned a different transaction ID") - self.repository.mark_submitted(intent.id, intent.txid) + intent = self.repository.mark_submitted(intent.id, intent.txid) + if intent.status == TransferStatus.CONFIRMED: + return self._confirmed_receipt(intent, already_confirmed=True) except Exception as exc: # A transport failure can happen after the node accepted the # transaction. Confirmation of the persisted txid is authoritative. @@ -248,11 +304,13 @@ def execute(self, request: AssetTransferRequest) -> AssetTransferReceipt: confirmed_round = self.gateway.wait_for_confirmation(intent.txid) except Exception as exc: failure_kind = type(broadcast_error or exc).__name__ - self.repository.record_error( + intent = self.repository.record_error( intent.id, intent.txid, f"confirmation unresolved after {failure_kind}", ) + if intent.status == TransferStatus.CONFIRMED: + return self._confirmed_receipt(intent, already_confirmed=True) raise AssetTransferPendingError(intent.id, intent.txid) from exc intent = self.repository.mark_confirmed( @@ -289,7 +347,18 @@ def reconcile(self, operation_id: str) -> TransferReconciliation: confirmed_round=confirmed.confirmed_round, ) - status = "expired_unconfirmed" if self.gateway.current_round() > intent.last_valid_round else intent.status + current_round = self.gateway.current_round() + intent = self._reload_reconciliation_intent(intent) + if intent.status == TransferStatus.CONFIRMED: + receipt = self._confirmed_receipt(intent, already_confirmed=True) + return TransferReconciliation( + operation_id=receipt.operation_id, + txid=receipt.txid, + status=TransferStatus.CONFIRMED, + confirmed_round=receipt.confirmed_round, + ) + + status = "expired_unconfirmed" if current_round > intent.last_valid_round else intent.status return TransferReconciliation( operation_id=intent.id, txid=intent.txid, @@ -297,6 +366,20 @@ def reconcile(self, operation_id: str) -> TransferReconciliation: confirmed_round=None, ) + def _reload_reconciliation_intent( + self, + expected: AssetTransferIntent, + ) -> AssetTransferIntent: + current = self.repository.get(expected.id) + if current is None: + raise AssetTransferError(f"transfer {expected.id!r} disappeared during reconciliation") + if current.txid != expected.txid: + raise AssetTransferConflictError( + f"transfer {expected.id!r} changed transaction from " + f"{expected.txid!r} to {current.txid!r} during reconciliation" + ) + return current + @staticmethod def _assert_same_transfer( intent: AssetTransferIntent, diff --git a/flex/application/transfer_runtime.py b/flex/application/transfer_runtime.py index d5e46b0d..184b9a55 100644 --- a/flex/application/transfer_runtime.py +++ b/flex/application/transfer_runtime.py @@ -2,6 +2,7 @@ from functools import lru_cache +from env import settings from flex import db from flex.application.asset_transfers import AssetTransferService from flex.blockchain.asset_transfers import AlgorandAssetTransferGateway @@ -20,5 +21,7 @@ def get_asset_transfer_service() -> AssetTransferService: indexer=indexer_client, sender=cometa_public_key, private_key=cometa_private_key, + network=settings.algo_network, + max_fee_microalgos=settings.outbound_asset_transfer_max_fee_microalgos, ) return AssetTransferService(repository=repository, gateway=gateway) diff --git a/flex/blockchain/asset_transfers.py b/flex/blockchain/asset_transfers.py index 3f7d328f..8e3f914f 100644 --- a/flex/blockchain/asset_transfers.py +++ b/flex/blockchain/asset_transfers.py @@ -6,7 +6,7 @@ from hashlib import sha256 from typing import Any -from algosdk import account, encoding, transaction +from algosdk import account, constants, encoding, transaction from algosdk.error import AlgodHTTPError from algosdk.v2client.algod import AlgodClient from algosdk.v2client.indexer import IndexerClient @@ -17,9 +17,24 @@ InvalidAssetTransferError, PreparedAssetTransfer, ) +from flex.domain.algorand import require_algorand_uint64 TX_WAIT_ROUNDS = 4 LEASE_DOMAIN = b"cometa-asset-transfer:v1:" +NETWORK_GENESIS = { + "mainnet": ( + "mainnet-v1.0", + "wGHE2Pwdvd7S12BL5FaOP20EGYesN73ktiC1qzkkit8=", + ), + "testnet": ( + "testnet-v1.0", + "SGO1GKSzyE7IEPItTxCByw9x8FmnrCDexi9/cOUJOiI=", + ), + "betanet": ( + "betanet-v1.0", + "mFgazF+2uRS1tMiL9dsj01hJGySEmPN28B/TjjvpVW0=", + ), +} @dataclass(slots=True) @@ -28,6 +43,8 @@ class AlgorandAssetTransferGateway: indexer: IndexerClient sender: str private_key: str + network: str + max_fee_microalgos: int wait_rounds: int = TX_WAIT_ROUNDS def prepare(self, request: AssetTransferRequest) -> PreparedAssetTransfer: @@ -35,6 +52,7 @@ def prepare(self, request: AssetTransferRequest) -> PreparedAssetTransfer: raise InvalidAssetTransferError("receiver is not a valid Algorand address") params = self.algod.suggested_params() + self._apply_fee_and_network_policy(params) unsigned_transaction = transaction.AssetTransferTxn( sender=self.sender, sp=params, @@ -74,6 +92,7 @@ def _validate_persisted_transaction( expected_lease = sha256(LEASE_DOMAIN + request.operation_id.encode()).digest() expected_authorizer = account.address_from_private_key(self.private_key) expected_signer = None if expected_authorizer == self.sender else expected_authorizer + expected_genesis_id, expected_genesis_hash = self._network_genesis() expected_fields = ( self.sender, @@ -88,6 +107,8 @@ def _validate_persisted_transaction( None, None, None, + expected_genesis_id, + expected_genesis_hash, ) observed_fields = ( txn.sender, @@ -102,9 +123,12 @@ def _validate_persisted_transaction( getattr(txn, "revocation_target", None), txn.group, txn.rekey_to, + txn.genesis_id, + txn.genesis_hash, ) if not isinstance(txn, transaction.AssetTransferTxn) or observed_fields != expected_fields: raise InvalidAssetTransferError("persisted transaction fields do not match the transfer intent") + self._validate_fee(txn.fee) if txn.get_txid() != prepared.txid: raise InvalidAssetTransferError("persisted transaction ID does not match the transfer intent") if signed.authorizing_address != expected_signer: @@ -117,6 +141,45 @@ def _validate_persisted_transaction( ): raise InvalidAssetTransferError("persisted transaction signature is invalid") + def _network_genesis(self) -> tuple[str, str]: + try: + return NETWORK_GENESIS[self.network] + except KeyError as exc: + raise InvalidAssetTransferError( + f"unsupported outbound transfer network {self.network!r}", + ) from exc + + def _apply_fee_and_network_policy( + self, + params: transaction.SuggestedParams, + ) -> None: + expected_genesis_id, expected_genesis_hash = self._network_genesis() + if params.gen != expected_genesis_id or params.gh != expected_genesis_hash: + raise InvalidAssetTransferError( + "Algod suggested parameters do not match the configured network", + ) + min_fee = params.min_fee + if isinstance(min_fee, bool) or not isinstance(min_fee, int) or min_fee <= 0: + raise InvalidAssetTransferError("Algod returned an invalid minimum fee") + self._validate_fee(min_fee) + # Payouts are single outer transfers. Paying exactly the protocol + # minimum is deterministic and prevents a node-provided per-byte fee + # from silently draining the signer. Congestion therefore fails closed. + params.flat_fee = True + params.fee = min_fee + + def _validate_fee(self, fee: object) -> None: + if isinstance(fee, bool) or not isinstance(fee, int): + raise InvalidAssetTransferError("outbound transaction fee is not an integer") + if fee < constants.min_txn_fee: + raise InvalidAssetTransferError( + "outbound transaction fee is below the Algorand protocol minimum", + ) + if fee > self.max_fee_microalgos: + raise InvalidAssetTransferError( + "outbound transaction fee exceeds the configured payout policy", + ) + def wait_for_confirmation(self, txid: str) -> int: response = transaction.wait_for_confirmation( self.algod, @@ -161,10 +224,19 @@ def lookup_confirmed_transfer(self, txid: str) -> ConfirmedAssetTransfer | None: amount_micros = asset_transfer.get("amount") if not isinstance(receiver, str) or not encoding.is_valid_address(receiver): raise RuntimeError("Algorand indexer response has an invalid receiver") - if isinstance(asset_id, bool) or not isinstance(asset_id, int) or asset_id <= 0: - raise RuntimeError("Algorand indexer response has an invalid asset ID") - if isinstance(amount_micros, bool) or not isinstance(amount_micros, int) or amount_micros <= 0: - raise RuntimeError("Algorand indexer response has an invalid transfer amount") + try: + asset_id = require_algorand_uint64( + asset_id, + "Algorand indexer asset ID", + positive=True, + ) + amount_micros = require_algorand_uint64( + amount_micros, + "Algorand indexer transfer amount", + positive=True, + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc return ConfirmedAssetTransfer( txid=txid, @@ -178,9 +250,13 @@ def lookup_confirmed_transfer(self, txid: str) -> ConfirmedAssetTransfer | None: def current_round(self) -> int: status = self.algod.status() current_round = status.get("last-round") - if isinstance(current_round, bool) or not isinstance(current_round, int) or current_round < 0: - raise RuntimeError("Algorand node returned an invalid current round") - return current_round + try: + return require_algorand_uint64( + current_round, + "Algorand node current round", + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc @staticmethod def _confirmed_round(response: dict[str, Any]) -> int: @@ -194,6 +270,10 @@ def _optional_confirmed_round(response: dict[str, Any]) -> int | None: confirmed_round = response.get("confirmed-round") if confirmed_round is None or confirmed_round == 0: return None - if isinstance(confirmed_round, bool) or not isinstance(confirmed_round, int) or confirmed_round < 0: - raise RuntimeError("Algorand response contains an invalid confirmed round") - return confirmed_round + try: + return require_algorand_uint64( + confirmed_round, + "Algorand confirmed round", + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc diff --git a/flex/db/asset_transfer_intents.py b/flex/db/asset_transfer_intents.py index de63bdf2..dbd3ebc4 100644 --- a/flex/db/asset_transfer_intents.py +++ b/flex/db/asset_transfer_intents.py @@ -8,7 +8,9 @@ from pymongo.collection import Collection from pymongo.errors import DuplicateKeyError +from flex.db.bson import encode_bson_integer from flex.db.model.transfers import AssetTransferIntent +from flex.domain.algorand import require_algorand_uint64 class TransferIntentPersistenceError(RuntimeError): @@ -46,7 +48,7 @@ def reserve(self, intent: AssetTransferIntent) -> AssetTransferIntent: return reserved def record_attempt(self, operation_id: str, txid: str) -> AssetTransferIntent: - return self._update( + return self._update_non_terminal( operation_id, txid, { @@ -57,7 +59,7 @@ def record_attempt(self, operation_id: str, txid: str) -> AssetTransferIntent: def mark_submitted(self, operation_id: str, txid: str) -> AssetTransferIntent: now = datetime.now(UTC) - return self._update( + return self._update_non_terminal( operation_id, txid, { @@ -76,19 +78,54 @@ def mark_confirmed( txid: str, confirmed_round: int, ) -> AssetTransferIntent: + try: + confirmed_round = require_algorand_uint64( + confirmed_round, + "confirmed_round", + ) + except ValueError as exc: + raise TransferIntentPersistenceError(str(exc)) from exc now = datetime.now(UTC) - return self._update( - operation_id, - txid, + document = self.collection.find_one_and_update( + { + "id": operation_id, + "txid": txid, + "status": {"$ne": "confirmed"}, + }, { "$set": { "status": "confirmed", - "confirmed_round": confirmed_round, + "confirmed_round": encode_bson_integer(confirmed_round), "confirmed_at": now, "last_error": None, "updated": now, } }, + return_document=ReturnDocument.AFTER, + ) + intent = self._from_document(document) + if intent is not None: + return intent + + current = self.get(operation_id) + if ( + current is not None + and current.txid == txid + and current.status == "confirmed" + and current.confirmed_round == confirmed_round + ): + return current + if current is None: + reason = "no longer exists" + elif current.txid != txid: + reason = f"now belongs to transaction {current.txid!r}" + elif current.status == "confirmed": + reason = f"is already confirmed in round {current.confirmed_round!r}" + else: + reason = f"remains in status {current.status!r}" + raise TransferIntentPersistenceError( + f"transfer {operation_id!r} could not confirm transaction {txid!r} " + f"in round {confirmed_round}: intent {reason}" ) def record_error( @@ -97,7 +134,7 @@ def record_error( txid: str, error: str, ) -> AssetTransferIntent: - return self._update( + return self._update_non_terminal( operation_id, txid, { @@ -108,21 +145,29 @@ def record_error( }, ) - def _update( + def _update_non_terminal( self, operation_id: str, txid: str, update: dict[str, Any], ) -> AssetTransferIntent: document = self.collection.find_one_and_update( - {"id": operation_id, "txid": txid}, + { + "id": operation_id, + "txid": txid, + "status": {"$ne": "confirmed"}, + }, update, return_document=ReturnDocument.AFTER, ) intent = self._from_document(document) - if intent is None: - raise TransferIntentPersistenceError(f"transfer {operation_id!r} changed while updating transaction {txid}") - return intent + if intent is not None: + return intent + + current = self.get(operation_id) + if current is not None and current.txid == txid and current.status == "confirmed": + return current + raise TransferIntentPersistenceError(f"transfer {operation_id!r} changed while updating transaction {txid}") @staticmethod def _from_document( diff --git a/flex/db/indexes.py b/flex/db/indexes.py index 93c8873a..7286e690 100644 --- a/flex/db/indexes.py +++ b/flex/db/indexes.py @@ -5,15 +5,17 @@ from pymongo.collection import Collection from flex.db.cometa_database import CometaDatabase +from flex.domain.pricing import PriceSource logger = logging.getLogger(__name__) _UNIQUE_ID_POLICIES = ( - ("airdrop_manifests", False), + ("assets", False), ("asset_prices", True), ("asset_transfer_intents", False), ("pool_transactions", False), ("lp_transactions", False), + ("lp_tokens", False), ) _HOT_INDEXES = ( @@ -21,10 +23,33 @@ ("lp_states", "address", "address_unique"), ("pool_states", "pool_id", "pool_id_idx"), ("user_states", "address", "address_idx"), - ("lp_tokens", "id", "lp_token_id_idx"), ) +def delete_unverified_legacy_lp_prices(database: CometaDatabase) -> int: + """Remove reconstructable prices produced from unauthenticated pool balances.""" + + result = database.asset_prices.mongodb_collection.delete_many( + { + "$or": [ + { + "tinyman_algo_pool_id": { + "$exists": True, + "$ne": None, + } + }, + {"source": PriceSource.DERIVED_LP.value}, + ] + } + ) + if result.deleted_count: + logger.warning( + "Removed %d retired raw-reserve LP price projection(s)", + result.deleted_count, + ) + return result.deleted_count + + def _duplicate_field_pipeline(field_name: str) -> list[dict[str, Any]]: """Group duplicate business keys after sorting the newest record first.""" @@ -126,6 +151,37 @@ def create_unique_field_index_fail_closed( ) +def ensure_airdrop_indexes(database: CometaDatabase) -> None: + """Install the standalone airdrop invariants before any payout work.""" + + create_unique_id_index_fail_closed( + database.airdrop_manifests.mongodb_collection, + collection_name="airdrop_manifests", + ) + reward_collection = database.airdrop_rewards.mongodb_collection + duplicate_operation_ids: Sequence[dict[str, Any]] = list( + reward_collection.aggregate( + [ + {"$match": {"operation_id": {"$type": "string"}}}, + *_duplicate_field_pipeline("operation_id"), + ], + allowDiskUse=True, + ) + ) + if duplicate_operation_ids: + raise RuntimeError( + "airdrop_rewards contains " + f"{len(duplicate_operation_ids)} duplicate 'operation_id' group(s); " + "reconcile them before running an airdrop" + ) + reward_collection.create_index( + "operation_id", + unique=True, + name="operation_id_unique", + partialFilterExpression={"operation_id": {"$type": "string"}}, + ) + + def ensure_sync_state_singleton(database: CometaDatabase) -> None: """Migrate one legacy random-ID cursor and reject competing checkpoints.""" @@ -150,7 +206,11 @@ def ensure_sync_state_singleton(database: CometaDatabase) -> None: def ensure_database_indexes(database: CometaDatabase) -> dict[str, int]: """Install correctness-critical unique indexes and hot query indexes.""" - removed_by_collection: dict[str, int] = {} + # Delete the retired derived cache before deduplication so a newer unsafe + # row cannot displace a safe provider-backed observation with the same ID. + delete_unverified_legacy_lp_prices(database) + ensure_airdrop_indexes(database) + removed_by_collection: dict[str, int] = {"airdrop_manifests": 0} for collection_name, can_deduplicate in _UNIQUE_ID_POLICIES: manager = getattr(database, collection_name) @@ -180,12 +240,5 @@ def ensure_database_indexes(database: CometaDatabase) -> dict[str, int]: else: manager.mongodb_collection.create_index(field_name, name=index_name) - database.airdrop_rewards.mongodb_collection.create_index( - "operation_id", - unique=True, - name="operation_id_unique", - partialFilterExpression={"operation_id": {"$type": "string"}}, - ) - logger.info("Ensured hot query indexes for Flex collections") return removed_by_collection diff --git a/flex/db/model/transfers.py b/flex/db/model/transfers.py index 7f0e9989..2e67c2da 100644 --- a/flex/db/model/transfers.py +++ b/flex/db/model/transfers.py @@ -2,10 +2,19 @@ from dataclasses import dataclass, field from datetime import UTC, datetime +from typing import ClassVar -from dataclasses_json import dataclass_json +from dataclasses_json import config, dataclass_json +from flex.db.bson import ( + decode_bson_uint64, + decode_optional_bson_uint64, + encode_bson_integer, + encode_optional_bson_uint64, +) from flex.db.classes.base_entity import BaseEntity +from flex.db.classes.bson_uint64 import BsonUint64StorageMixin +from flex.domain.algorand import require_algorand_uint64 def _utc_now() -> datetime: @@ -14,9 +23,20 @@ def _utc_now() -> datetime: @dataclass_json @dataclass -class AssetTransferIntent(BaseEntity["AssetTransferIntent"]): +class AssetTransferIntent( + BsonUint64StorageMixin, + BaseEntity["AssetTransferIntent"], +): """A signed transaction persisted before its first broadcast attempt.""" + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "first_valid_round", + "last_valid_round", + "confirmed_round", + } + ) + id: str receiver: str asset_id: str @@ -24,12 +44,28 @@ class AssetTransferIntent(BaseEntity["AssetTransferIntent"]): note: str | None signed_transaction: str txid: str - first_valid_round: int - last_valid_round: int + first_valid_round: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + last_valid_round: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) status: str attempt_count: int = 0 - confirmed_round: int | None = None + confirmed_round: int | None = field( + default=None, + metadata=config( + encoder=encode_optional_bson_uint64, + decoder=decode_optional_bson_uint64, + ), + ) last_error: str | None = None submitted_at: datetime | None = None confirmed_at: datetime | None = None @@ -41,6 +77,21 @@ def __post_init__(self) -> None: # string-backed so the full Algorand uint64 domain is lossless. self.asset_id = str(self.asset_id) self.amount_micros = str(self.amount_micros) + self.first_valid_round = require_algorand_uint64( + self.first_valid_round, + "first_valid_round", + ) + self.last_valid_round = require_algorand_uint64( + self.last_valid_round, + "last_valid_round", + ) + if self.first_valid_round > self.last_valid_round: + raise ValueError("first_valid_round cannot exceed last_valid_round") + if self.confirmed_round is not None: + self.confirmed_round = require_algorand_uint64( + self.confirmed_round, + "confirmed_round", + ) @property def amount_micros_int(self) -> int: diff --git a/flex/tools/airdrop.py b/flex/tools/airdrop.py index 1d6534ea..18e9fa5b 100644 --- a/flex/tools/airdrop.py +++ b/flex/tools/airdrop.py @@ -21,6 +21,7 @@ from flex.application.transfer_runtime import get_asset_transfer_service from flex.blockchain.base import cometa_public_key from flex.blockchain.info import get_current_round +from flex.db.indexes import ensure_airdrop_indexes from flex.db.model.airdrop import AirdropManifest from flex.db.model.blockchain import AssetInfo from flex.db.model.priced import AirdropReward @@ -152,7 +153,6 @@ def _reserve_manifest( manifest: AirdropManifest, ) -> AirdropManifest: collection = db.airdrop_manifests.mongodb_collection - collection.create_index("id", unique=True, name="id_unique") document = collection.find_one_and_update( {"id": manifest.id}, {"$setOnInsert": manifest.to_dict()}, @@ -191,9 +191,13 @@ def _has_manifest(airdrop_id: str) -> bool: ) -def _mark_manifest_status(airdrop_id: str, status: str) -> None: - db.airdrop_manifests.mongodb_collection.update_one( - {"id": airdrop_id}, +def _mark_manifest_status(airdrop_id: str, status: str) -> str: + collection = db.airdrop_manifests.mongodb_collection + query: dict[str, object] = {"id": airdrop_id} + if status != "complete": + query["status"] = {"$ne": "complete"} + result = collection.update_one( + query, { "$set": { "status": status, @@ -201,6 +205,16 @@ def _mark_manifest_status(airdrop_id: str, status: str) -> None: } }, ) + if result.matched_count: + return status + + persisted = collection.find_one( + {"id": airdrop_id}, + projection={"status": 1}, + ) + if persisted is not None and persisted.get("status") == "complete": + return "complete" + raise AirdropError(f"airdrop manifest {airdrop_id!r} changed while marking status {status!r}") def _reconcile_existing_reward( @@ -351,6 +365,9 @@ async def send_airdrop( notes=notes, amounts=amounts, ) + # Operator scripts invoke this function without FastAPI startup. Install + # both uniqueness invariants before any reward read or transaction signing. + ensure_airdrop_indexes(db) service = transfer_service or get_asset_transfer_service() requests = { address: AssetTransferRequest( @@ -492,8 +509,15 @@ async def send_airdrop( ) if failures: - _mark_manifest_status(airdrop_id, "partial") - raise AirdropIncompleteError(failures, confirmed_transactions) + effective_status = _mark_manifest_status(airdrop_id, "partial") + if effective_status != "complete": + raise AirdropIncompleteError(failures, confirmed_transactions) + logger.info( + "Airdrop %s was completed by another worker while this worker had %s stale failure(s)", + airdrop_id, + len(failures), + ) + return confirmed_transactions _mark_manifest_status(airdrop_id, "complete") logger.info( diff --git a/tests/integration/test_mongo_asset_transfer_races.py b/tests/integration/test_mongo_asset_transfer_races.py new file mode 100644 index 00000000..5c9a05c6 --- /dev/null +++ b/tests/integration/test_mongo_asset_transfer_races.py @@ -0,0 +1,245 @@ +import os +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime +from threading import Barrier +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from pymongo import MongoClient +from pymongo.database import Database + +from flex.application.asset_transfers import ( + AssetTransferService, + TransferStatus, +) +from flex.db.asset_transfer_intents import ( + MongoAssetTransferIntentRepository, + TransferIntentPersistenceError, +) +from flex.db.model.transfers import AssetTransferIntent +from flex.tools import airdrop + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def mongo_database() -> Database: + uri = os.getenv("MONGODB_TEST_URI") + if not uri: + pytest.skip("MONGODB_TEST_URI is not configured") + + client = MongoClient( + uri, + serverSelectionTimeoutMS=2_000, + tz_aware=True, + ) + client.admin.command("ping") + database_name = f"cometa_transfer_races_{uuid4().hex}" + database = client[database_name] + try: + yield database + finally: + client.drop_database(database_name) + client.close() + + +def _intent() -> AssetTransferIntent: + timestamp = datetime(2026, 1, 1, tzinfo=UTC) + return AssetTransferIntent( + id="airdrop:summer:address", + receiver="address", + asset_id="42", + amount_micros="1000", + note="hello", + signed_transaction="signed", + txid="txid", + first_valid_round=100, + last_valid_round=1_100, + status=TransferStatus.PREPARED, + created=timestamp, + updated=timestamp, + ) + + +def _repository( + database: Database, +) -> MongoAssetTransferIntentRepository: + repository = MongoAssetTransferIntentRepository( + database["asset_transfer_intents"], + ) + repository.ensure_indexes() + repository.reserve(_intent()) + return repository + + +@pytest.mark.parametrize( + "nonterminal_update", + ["record_attempt", "mark_submitted", "record_error"], +) +def test_confirmation_wins_real_mongo_race_against_nonterminal_update( + mongo_database: Database, + nonterminal_update: str, +) -> None: + repository = _repository(mongo_database) + intent = _intent() + barrier = Barrier(2) + + def confirm() -> AssetTransferIntent: + barrier.wait(timeout=5) + return repository.mark_confirmed(intent.id, intent.txid, 777) + + def update_nonterminal() -> AssetTransferIntent: + barrier.wait(timeout=5) + if nonterminal_update == "record_attempt": + return repository.record_attempt(intent.id, intent.txid) + if nonterminal_update == "mark_submitted": + return repository.mark_submitted(intent.id, intent.txid) + return repository.record_error(intent.id, intent.txid, "late failure") + + with ThreadPoolExecutor(max_workers=2) as executor: + confirmation = executor.submit(confirm) + nonterminal = executor.submit(update_nonterminal) + confirmation.result(timeout=5) + nonterminal.result(timeout=5) + + confirmed = repository.get(intent.id) + assert confirmed is not None + assert confirmed.status == TransferStatus.CONFIRMED + assert confirmed.confirmed_round == 777 + assert confirmed.confirmed_at is not None + assert confirmed.last_error is None + + original_confirmed_at = confirmed.confirmed_at + repository.record_error(intent.id, intent.txid, "must not overwrite terminal evidence") + persisted = repository.get(intent.id) + assert persisted is not None + assert persisted.status == TransferStatus.CONFIRMED + assert persisted.confirmed_round == 777 + assert persisted.confirmed_at == original_confirmed_at + assert persisted.last_error is None + + +def test_conflicting_real_mongo_confirmations_preserve_one_terminal_result( + mongo_database: Database, +) -> None: + repository = _repository(mongo_database) + intent = _intent() + barrier = Barrier(2) + + def confirm(round_number: int) -> tuple[str, int]: + barrier.wait(timeout=5) + try: + result = repository.mark_confirmed( + intent.id, + intent.txid, + round_number, + ) + except TransferIntentPersistenceError: + return "conflict", round_number + return "confirmed", result.confirmed_round or 0 + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(confirm, round_number) for round_number in (777, 778)] + results = [future.result(timeout=5) for future in futures] + + assert sorted(status for status, _ in results) == ["confirmed", "conflict"] + persisted = repository.get(intent.id) + assert persisted is not None + assert persisted.status == TransferStatus.CONFIRMED + assert persisted.confirmed_round in {777, 778} + assert persisted.confirmed_at is not None + + confirmed_at = persisted.confirmed_at + repeated = repository.mark_confirmed( + intent.id, + intent.txid, + persisted.confirmed_round, + ) + assert repeated.confirmed_at == confirmed_at + + conflicting_round = 778 if persisted.confirmed_round == 777 else 777 + with pytest.raises(TransferIntentPersistenceError, match="already confirmed"): + repository.mark_confirmed( + intent.id, + intent.txid, + conflicting_round, + ) + unchanged = repository.get(intent.id) + assert unchanged is not None + assert unchanged.confirmed_round == persisted.confirmed_round + assert unchanged.confirmed_at == confirmed_at + + +class _ConfirmDuringLookupGateway: + def __init__( + self, + repository: MongoAssetTransferIntentRepository, + intent: AssetTransferIntent, + ) -> None: + self.repository = repository + self.intent = intent + + def lookup_confirmed_round(self, txid: str) -> int | None: + assert txid == self.intent.txid + with ThreadPoolExecutor(max_workers=1) as executor: + executor.submit( + self.repository.mark_confirmed, + self.intent.id, + txid, + 779, + ).result(timeout=5) + return None + + @staticmethod + def current_round() -> int: + return 200 + + +def test_reconcile_reloads_real_mongo_terminal_state_before_nonterminal_result( + mongo_database: Database, +) -> None: + repository = _repository(mongo_database) + intent = _intent() + service = AssetTransferService( + repository=repository, + gateway=_ConfirmDuringLookupGateway(repository, intent), # type: ignore[arg-type] + ) + + result = service.reconcile(intent.id) + + assert result.status == TransferStatus.CONFIRMED + assert result.txid == intent.txid + assert result.confirmed_round == 779 + + +def test_complete_airdrop_manifest_wins_real_mongo_partial_status_race( + mongo_database: Database, + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = mongo_database["airdrop_manifests"] + collection.insert_one({"id": "summer-2026", "status": "prepared"}) + monkeypatch.setattr( + airdrop, + "db", + SimpleNamespace( + airdrop_manifests=SimpleNamespace( + mongodb_collection=collection, + ), + ), + ) + barrier = Barrier(2) + + def mark(status: str) -> str: + barrier.wait(timeout=5) + return airdrop._mark_manifest_status("summer-2026", status) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(mark, status) for status in ("complete", "partial")] + results = [future.result(timeout=5) for future in futures] + + persisted = collection.find_one({"id": "summer-2026"}) + assert persisted is not None + assert persisted["status"] == "complete" + assert results[0] == "complete" + assert results[1] in {"partial", "complete"} diff --git a/tests/integration/test_mongo_lottery_index_upgrade.py b/tests/integration/test_mongo_lottery_index_upgrade.py new file mode 100644 index 00000000..fe4cbf37 --- /dev/null +++ b/tests/integration/test_mongo_lottery_index_upgrade.py @@ -0,0 +1,311 @@ +import os +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from pymongo import MongoClient +from pymongo.database import Database +from pymongo.errors import DuplicateKeyError + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def mongo_database() -> Database: + uri = os.getenv("MONGODB_TEST_URI") + if not uri: + pytest.skip("MONGODB_TEST_URI is not configured") + + client = MongoClient( + uri, + serverSelectionTimeoutMS=2_000, + tz_aware=True, + ) + client.admin.command("ping") + database_name = f"cometa_lottery_index_{uuid4().hex}" + database = client[database_name] + try: + yield database + finally: + client.drop_database(database_name) + client.close() + + +def test_old_lottery_id_index_is_replaced_without_a_uniqueness_gap( + mongo_database: Database, + monkeypatch, +) -> None: + from api import nft_lottery + + collection = mongo_database["lottery_draws"] + collection.insert_many( + [ + {"marker": "missing-id"}, + {"marker": "empty-id", "id": ""}, + {"marker": "current", "id": "draw-current"}, + ] + ) + collection.create_index( + "id", + unique=True, + name="id_unique", + partialFilterExpression={"id": {"$type": "string"}}, + ) + original_ids = set(collection.distinct("_id")) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + + nft_lottery.ensure_lottery_indexes() + + assert set(collection.distinct("_id")) == original_ids + indexes = collection.index_information() + assert "id_unique" not in indexes + replacement = indexes[nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME] + assert replacement["unique"] is True + assert replacement["partialFilterExpression"] == nft_lottery.LOTTERY_DRAW_ID_FILTER + + # The upgraded filter intentionally leaves legacy placeholders outside the + # index while continuing to reject every duplicate durable draw identity. + collection.insert_many([{"id": ""}, {"id": ""}, {"marker": "another-missing-id"}]) + with pytest.raises(DuplicateKeyError): + collection.insert_one({"id": "draw-current"}) + + +def test_lottery_id_index_upgrade_preserves_conflicting_records( + mongo_database: Database, + monkeypatch, +) -> None: + from api import nft_lottery + + collection = mongo_database["lottery_draws"] + collection.insert_many( + [ + {"marker": "first", "id": "duplicate"}, + {"marker": "second", "id": "duplicate"}, + ] + ) + original_ids = set(collection.distinct("_id")) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + + with pytest.raises(RuntimeError, match="preserved 1 duplicate group"): + nft_lottery.ensure_lottery_indexes() + + assert set(collection.distinct("_id")) == original_ids + assert nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME not in collection.index_information() + + +def test_staking_entitlement_allows_only_one_concurrent_draw( + mongo_database: Database, + monkeypatch, +) -> None: + from api import nft_lottery + + draws = mongo_database["lottery_draws"] + entitlements = mongo_database["lottery_entitlements"] + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=draws), + ) + monkeypatch.setattr( + nft_lottery, + "lottery_entitlements", + entitlements, + ) + monkeypatch.setattr( + nft_lottery, + "draw_id", + lambda lottery: 777, + ) + lottery = nft_lottery.NftLottery( + name="staking-summer", + asset_id=7, + min_amount=1, + probability=1.0, + available_nfts=[777], + type=nft_lottery.LotteryType.STAKING, + ) + now = datetime(2026, 7, 19, 12, tzinfo=UTC) + + def create_draw(_: int): + return nft_lottery._get_or_create_staking_draw( + lottery, + "WALLET", + now=now, + ) + + with ThreadPoolExecutor(max_workers=16) as executor: + results = list(executor.map(create_draw, range(32))) + + materialized = [draw for draw in results if draw is not None] + assert materialized + assert len({draw.id for draw in materialized}) == 1 + assert draws.count_documents({}) == 1 + entitlement = entitlements.find_one({}) + assert entitlement is not None + assert entitlement["generation"] == 1 + assert entitlement["active"]["draw_id"] == materialized[0].id + assert entitlement["active"]["status"] == nft_lottery.StakingEntitlementStatus.MATERIALIZED + + +def test_staking_entitlement_recovers_a_crash_before_draw_insert( + mongo_database: Database, + monkeypatch, +) -> None: + from api import nft_lottery + + draws = mongo_database["lottery_draws"] + entitlements = mongo_database["lottery_entitlements"] + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=draws), + ) + monkeypatch.setattr( + nft_lottery, + "lottery_entitlements", + entitlements, + ) + monkeypatch.setattr( + nft_lottery, + "draw_id", + lambda lottery: 888, + ) + lottery = nft_lottery.NftLottery( + name="staking-recovery", + asset_id=7, + min_amount=1, + probability=1.0, + available_nfts=[888], + type=nft_lottery.LotteryType.STAKING, + ) + now = datetime(2026, 7, 19, 12, tzinfo=UTC) + claimed = nft_lottery._claim_staking_entitlement( + lottery_name=lottery.name, + wallet="WALLET", + now=now, + ) + + assert claimed is not None + assert draws.count_documents({}) == 0 + + recovered = nft_lottery._get_or_create_staking_draw( + lottery, + "WALLET", + now=now, + ) + + assert recovered is not None + assert recovered.id == claimed["active"]["draw_id"] + assert recovered.prize == 888 + assert draws.count_documents({}) == 1 + + +def test_staking_entitlement_repairs_confirmed_draw_before_next_generation( + mongo_database: Database, + monkeypatch, +) -> None: + from api import nft_lottery + + draws = mongo_database["lottery_draws"] + entitlements = mongo_database["lottery_entitlements"] + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=draws), + ) + monkeypatch.setattr( + nft_lottery, + "lottery_entitlements", + entitlements, + ) + monkeypatch.setattr( + nft_lottery, + "draw_id", + lambda lottery: 999, + ) + lottery = nft_lottery.NftLottery( + name="staking-confirmed-repair", + asset_id=7, + min_amount=1, + probability=1.0, + available_nfts=[999], + type=nft_lottery.LotteryType.STAKING, + ) + first_time = datetime(2026, 7, 19, 12, tzinfo=UTC) + first = nft_lottery._get_or_create_staking_draw( + lottery, + "WALLET", + now=first_time, + ) + assert first is not None + draws.update_one( + {"id": first.id}, + { + "$set": { + "claimed": True, + "payout_status": nft_lottery.LotteryPayoutStatus.CONFIRMED, + } + }, + ) + + second = nft_lottery._get_or_create_staking_draw( + lottery, + "WALLET", + now=first_time + timedelta(seconds=nft_lottery.MIN_DRAW_INTERVAL), + ) + + assert second is not None + assert second.id != first.id + entitlement = entitlements.find_one({}) + assert entitlement is not None + assert entitlement["generation"] == 2 + + +def test_confirmed_staking_entitlement_is_terminal_against_stale_error( + mongo_database: Database, + monkeypatch, +) -> None: + from api import nft_lottery + + entitlements = mongo_database["lottery_entitlements"] + monkeypatch.setattr( + nft_lottery, + "lottery_entitlements", + entitlements, + ) + entitlements.insert_one( + { + "_id": "entitlement", + "generation": 1, + "active": { + "draw_id": "draw", + "status": nft_lottery.StakingEntitlementStatus.CONFIRMED, + }, + } + ) + draw = nft_lottery.LotteryDraw( + id="draw", + wallet="WALLET", + prize=999, + entitlement_id="entitlement", + entitlement_generation=1, + ) + + nft_lottery._update_staking_entitlement_status( + draw, + nft_lottery.StakingEntitlementStatus.UNRESOLVED, + ) + + persisted = entitlements.find_one({"_id": "entitlement"}) + assert persisted is not None + assert persisted["active"]["status"] == nft_lottery.StakingEntitlementStatus.CONFIRMED diff --git a/tests/unit/test_airdrop.py b/tests/unit/test_airdrop.py index 32cfa659..b98a9e1e 100644 --- a/tests/unit/test_airdrop.py +++ b/tests/unit/test_airdrop.py @@ -19,6 +19,13 @@ class FakeRewardCollection: def __init__(self, rewards: list[AirdropReward] | None = None) -> None: self.documents = [reward.to_dict() for reward in rewards or []] + self.indexes: list[tuple[tuple, dict]] = [] + + def aggregate(self, *args, **kwargs): + return [] + + def create_index(self, *args, **kwargs) -> None: + self.indexes.append((args, kwargs)) def find_one_and_update(self, query, update, **kwargs): document = next( @@ -53,10 +60,14 @@ def get_many(self, **query): class FakeManifestCollection: def __init__(self) -> None: self.documents: dict[str, dict] = {} + self.complete_before_partial = False def create_index(self, *args, **kwargs) -> None: return None + def aggregate(self, *args, **kwargs): + return [] + def find_one(self, query, projection=None): document = self.documents.get(query["id"]) return dict(document) if document is not None else None @@ -66,8 +77,24 @@ def find_one_and_update(self, query, update, **kwargs): self.documents.setdefault(manifest_id, dict(update["$setOnInsert"])) return dict(self.documents[manifest_id]) - def update_one(self, query, update) -> None: - self.documents[query["id"]].update(update["$set"]) + def update_one(self, query, update): + document = self.documents.get(query["id"]) + expected_status = query.get("status") + if ( + document is not None + and self.complete_before_partial + and isinstance(expected_status, dict) + and expected_status.get("$ne") == "complete" + ): + document["status"] = "complete" + if document is None or ( + isinstance(expected_status, dict) + and "$ne" in expected_status + and document.get("status") == expected_status["$ne"] + ): + return SimpleNamespace(matched_count=0) + document.update(update["$set"]) + return SimpleNamespace(matched_count=1) class FakeTransferService: @@ -194,6 +221,16 @@ def test_airdrop_allocates_the_exact_budget_and_persists_every_receipt(monkeypat assert manifests.documents["summer-2026"]["status"] == "complete" +@pytest.mark.parametrize("stale_status", ["prepared", "partial"]) +def test_complete_airdrop_manifest_cannot_regress(monkeypatch, stale_status: str) -> None: + _, manifests = _install_fake_db(monkeypatch) + manifests.documents["summer-2026"] = {"id": "summer-2026", "status": "complete"} + + airdrop._mark_manifest_status("summer-2026", stale_status) + + assert manifests.documents["summer-2026"]["status"] == "complete" + + def test_airdrop_configuration_conflict_aborts_before_any_broadcast(monkeypatch) -> None: address, other_address = _addresses(2) existing = AirdropReward( @@ -257,6 +294,28 @@ def test_airdrop_reports_partial_failure_and_continues_safe_recipients(monkeypat assert manifests.documents["summer-2026"]["status"] == "partial" +def test_stale_airdrop_failure_observes_concurrent_terminal_completion(monkeypatch) -> None: + rewards, manifests = _install_fake_db(monkeypatch) + addresses = _addresses(2) + service = FakeTransferService(fail_address=addresses[1]) + manifests.complete_before_partial = True + + transactions = asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares=dict.fromkeys(addresses, 1), + notes=["hello"], + airdrop_id="summer-2026", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert len(transactions) == 1 + assert len(rewards.mongodb_collection.documents) == 1 + assert manifests.documents["summer-2026"]["status"] == "complete" + + def test_airdrop_rejects_zero_allocations_before_execution(monkeypatch) -> None: _install_fake_db(monkeypatch) addresses = _addresses(2) diff --git a/tests/unit/test_algorand_asset_transfer_gateway.py b/tests/unit/test_algorand_asset_transfer_gateway.py index a05451de..6f608fd2 100644 --- a/tests/unit/test_algorand_asset_transfer_gateway.py +++ b/tests/unit/test_algorand_asset_transfer_gateway.py @@ -1,3 +1,5 @@ +from dataclasses import replace + import pytest from algosdk import account, encoding, transaction @@ -9,16 +11,27 @@ class FakeAlgod: - def __init__(self) -> None: + def __init__( + self, + *, + min_fee: object = 1_000, + genesis_id: str = "mainnet-v1.0", + genesis_hash: str = "wGHE2Pwdvd7S12BL5FaOP20EGYesN73ktiC1qzkkit8=", + ) -> None: self.sent: list[transaction.SignedTransaction] = [] + self.min_fee = min_fee + self.genesis_id = genesis_id + self.genesis_hash = genesis_hash def suggested_params(self) -> transaction.SuggestedParams: return transaction.SuggestedParams( - fee=1_000, + fee=99_999, first=100, last=1_100, - gh=b"0" * 32, - flat_fee=True, + gh=self.genesis_hash, + gen=self.genesis_id, + flat_fee=False, + min_fee=self.min_fee, ) def send_transaction(self, signed: transaction.SignedTransaction) -> str: @@ -57,6 +70,8 @@ def test_gateway_prepares_a_replay_safe_signed_transaction() -> None: indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] sender=sender, private_key=private_key, + network="mainnet", + max_fee_microalgos=1_000, ) request = AssetTransferRequest( operation_id="airdrop:summer:recipient", @@ -76,6 +91,7 @@ def test_gateway_prepares_a_replay_safe_signed_transaction() -> None: assert decoded.transaction.amount == 1_000 assert decoded.transaction.note == b"hello" assert len(decoded.transaction.lease) == 32 + assert decoded.transaction.fee == 1_000 assert prepared.first_valid_round == 100 assert prepared.last_valid_round == 1_100 @@ -102,6 +118,8 @@ def test_operation_id_deterministically_selects_the_lease() -> None: indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] sender=sender, private_key=private_key, + network="mainnet", + max_fee_microalgos=1_000, ) request = AssetTransferRequest( operation_id="lottery:draw:42", @@ -127,6 +145,8 @@ def test_gateway_rejects_a_swapped_signed_payload_before_network_io() -> None: indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] sender=sender, private_key=private_key, + network="mainnet", + max_fee_microalgos=1_000, ) expected = AssetTransferRequest( operation_id="airdrop:summer:recipient", @@ -149,3 +169,93 @@ def test_gateway_rejects_a_swapped_signed_payload_before_network_io() -> None: gateway.broadcast(swapped, expected) assert algod.sent == [] + + +@pytest.mark.parametrize("min_fee", [None, True, -1, 0, 1, 999, 1_001]) +def test_gateway_rejects_an_invalid_or_excessive_network_fee( + min_fee: object, +) -> None: + private_key, sender = account.generate_account() + _, receiver = account.generate_account() + algod = FakeAlgod(min_fee=min_fee) + gateway = AlgorandAssetTransferGateway( + algod=algod, # type: ignore[arg-type] + indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] + sender=sender, + private_key=private_key, + network="mainnet", + max_fee_microalgos=1_000, + ) + + with pytest.raises(InvalidAssetTransferError, match="fee"): + gateway.prepare( + AssetTransferRequest( + operation_id="lottery:fee-policy", + receiver=receiver, + asset_id=7, + amount_micros=1, + ) + ) + + assert algod.sent == [] + + +def test_gateway_rejects_wrong_network_parameters_before_signing() -> None: + private_key, sender = account.generate_account() + _, receiver = account.generate_account() + algod = FakeAlgod( + genesis_id="testnet-v1.0", + genesis_hash="SGO1GKSzyE7IEPItTxCByw9x8FmnrCDexi9/cOUJOiI=", + ) + gateway = AlgorandAssetTransferGateway( + algod=algod, # type: ignore[arg-type] + indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] + sender=sender, + private_key=private_key, + network="mainnet", + max_fee_microalgos=1_000, + ) + + with pytest.raises(InvalidAssetTransferError, match="configured network"): + gateway.prepare( + AssetTransferRequest( + operation_id="lottery:wrong-network", + receiver=receiver, + asset_id=7, + amount_micros=1, + ) + ) + + +def test_gateway_rejects_a_persisted_fee_above_policy_before_network_io() -> None: + private_key, sender = account.generate_account() + _, receiver = account.generate_account() + algod = FakeAlgod() + gateway = AlgorandAssetTransferGateway( + algod=algod, # type: ignore[arg-type] + indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] + sender=sender, + private_key=private_key, + network="mainnet", + max_fee_microalgos=1_000, + ) + request = AssetTransferRequest( + operation_id="lottery:persisted-fee", + receiver=receiver, + asset_id=7, + amount_micros=1, + ) + prepared = gateway.prepare(request) + decoded = encoding.msgpack_decode(prepared.signed_transaction) + decoded.transaction.fee = 2_000 + resigned = decoded.transaction.sign(private_key) + tampered = replace( + prepared, + signed_transaction=encoding.msgpack_encode(resigned), + txid=decoded.transaction.get_txid(), + ) + + with pytest.raises(InvalidAssetTransferError, match="fee"): + gateway.broadcast(tampered, request) + + assert algod.sent == [] diff --git a/tests/unit/test_asset_transfer_repository.py b/tests/unit/test_asset_transfer_repository.py index 51fbdcf9..26084fc8 100644 --- a/tests/unit/test_asset_transfer_repository.py +++ b/tests/unit/test_asset_transfer_repository.py @@ -1,9 +1,14 @@ from datetime import UTC, datetime from unittest.mock import Mock +import pytest +from bson import Decimal128 from pymongo import ReturnDocument -from flex.db.asset_transfer_intents import MongoAssetTransferIntentRepository +from flex.db.asset_transfer_intents import ( + MongoAssetTransferIntentRepository, + TransferIntentPersistenceError, +) from flex.db.model.transfers import AssetTransferIntent @@ -72,7 +77,108 @@ def test_confirmation_is_guarded_by_operation_and_transaction_id() -> None: assert result.status == "confirmed" assert result.confirmed_round == 777 args, kwargs = collection.find_one_and_update.call_args - assert args[0] == {"id": confirmed.id, "txid": confirmed.txid} + assert args[0] == { + "id": confirmed.id, + "txid": confirmed.txid, + "status": {"$ne": "confirmed"}, + } assert args[1]["$set"]["status"] == "confirmed" - assert args[1]["$set"]["confirmed_round"] == 777 + assert args[1]["$set"]["confirmed_round"] == Decimal128("777") assert kwargs == {"return_document": ReturnDocument.AFTER} + + +def test_repeated_identical_confirmation_preserves_terminal_evidence() -> None: + collection = Mock() + confirmed = _intent() + confirmed.status = "confirmed" + confirmed.confirmed_round = 777 + confirmed.confirmed_at = datetime(2026, 1, 2, tzinfo=UTC) + collection.find_one_and_update.return_value = None + collection.find_one.return_value = confirmed.to_dict() + repository = MongoAssetTransferIntentRepository(collection) + + result = repository.mark_confirmed(confirmed.id, confirmed.txid, 777) + + assert result == confirmed + assert result.confirmed_at == datetime(2026, 1, 2, tzinfo=UTC) + query = collection.find_one_and_update.call_args.args[0] + assert query["status"] == {"$ne": "confirmed"} + collection.find_one.assert_called_once_with({"id": confirmed.id}) + + +@pytest.mark.parametrize( + ("current", "txid", "round_number", "message"), + [ + (None, "txid", 777, "no longer exists"), + (_intent(), "other-txid", 777, "now belongs to transaction"), + (_intent(), "txid", 778, "already confirmed in round"), + (_intent(), "txid", 777, "remains in status"), + ], +) +def test_confirmation_cas_miss_fails_closed( + current: AssetTransferIntent | None, + txid: str, + round_number: int, + message: str, +) -> None: + if current is not None and message == "already confirmed in round": + current.status = "confirmed" + current.confirmed_round = 777 + collection = Mock() + collection.find_one_and_update.return_value = None + collection.find_one.return_value = None if current is None else current.to_dict() + repository = MongoAssetTransferIntentRepository(collection) + + with pytest.raises(TransferIntentPersistenceError, match=message): + repository.mark_confirmed("airdrop:summer:address", txid, round_number) + + +@pytest.mark.parametrize( + "update", + [ + lambda repository, intent: repository.record_attempt(intent.id, intent.txid), + lambda repository, intent: repository.mark_submitted(intent.id, intent.txid), + lambda repository, intent: repository.record_error(intent.id, intent.txid, "late error"), + ], +) +def test_non_confirmation_updates_cannot_mutate_confirmed_intent(update) -> None: + collection = Mock() + confirmed = _intent() + confirmed.status = "confirmed" + confirmed.attempt_count = 4 + confirmed.confirmed_round = 777 + collection.find_one_and_update.return_value = None + collection.find_one.return_value = confirmed.to_dict() + repository = MongoAssetTransferIntentRepository(collection) + + result = update(repository, confirmed) + + assert result == confirmed + query = collection.find_one_and_update.call_args.args[0] + assert query == { + "id": confirmed.id, + "txid": confirmed.txid, + "status": {"$ne": "confirmed"}, + } + collection.find_one.assert_called_once_with({"id": confirmed.id}) + + +@pytest.mark.parametrize( + "update", + [ + lambda repository, intent: repository.record_attempt(intent.id, "other-txid"), + lambda repository, intent: repository.mark_submitted(intent.id, "other-txid"), + lambda repository, intent: repository.record_error(intent.id, "other-txid", "late error"), + ], +) +def test_non_confirmation_update_rejects_transaction_mismatch(update) -> None: + collection = Mock() + confirmed = _intent() + confirmed.status = "confirmed" + confirmed.confirmed_round = 777 + collection.find_one_and_update.return_value = None + collection.find_one.return_value = confirmed.to_dict() + repository = MongoAssetTransferIntentRepository(collection) + + with pytest.raises(TransferIntentPersistenceError, match="changed while updating"): + update(repository, confirmed) diff --git a/tests/unit/test_asset_transfers.py b/tests/unit/test_asset_transfers.py index 748f8fbf..a4381569 100644 --- a/tests/unit/test_asset_transfers.py +++ b/tests/unit/test_asset_transfers.py @@ -5,6 +5,7 @@ from flex.application.asset_transfers import ( AssetTransferConflictError, + AssetTransferError, AssetTransferExpiredError, AssetTransferPendingError, AssetTransferRequest, @@ -64,6 +65,54 @@ def _intent(self, operation_id: str, txid: str) -> AssetTransferIntent: return intent +class ConfirmOnUpdateRepository(InMemoryIntentRepository): + def __init__(self, update_name: str) -> None: + super().__init__() + self.update_name = update_name + + def _confirm(self, intent: AssetTransferIntent, update_name: str) -> AssetTransferIntent: + if self.update_name == update_name: + intent.status = TransferStatus.CONFIRMED + intent.confirmed_round = 777 + return intent + + def record_attempt(self, operation_id: str, txid: str) -> AssetTransferIntent: + return self._confirm( + super().record_attempt(operation_id, txid), + "record_attempt", + ) + + def mark_submitted(self, operation_id: str, txid: str) -> AssetTransferIntent: + return self._confirm( + super().mark_submitted(operation_id, txid), + "mark_submitted", + ) + + def record_error( + self, + operation_id: str, + txid: str, + error: str, + ) -> AssetTransferIntent: + return self._confirm( + super().record_error(operation_id, txid, error), + "record_error", + ) + + +class ReconcileReloadRepository(InMemoryIntentRepository): + def __init__(self, reloaded: AssetTransferIntent | None) -> None: + super().__init__() + self.reloaded = reloaded + self.get_count = 0 + + def get(self, operation_id: str) -> AssetTransferIntent | None: + self.get_count += 1 + if self.get_count == 1: + return super().get(operation_id) + return self.reloaded + + class FakeTransferGateway: def __init__(self) -> None: self.prepare_count = 0 @@ -73,6 +122,7 @@ def __init__(self) -> None: self.lookup_round: int | None = None self.broadcast_error: Exception | None = None self.wait_error: Exception | None = None + self.returned_txid: str | None = None def prepare(self, request: AssetTransferRequest) -> PreparedAssetTransfer: self.prepare_count += 1 @@ -91,6 +141,8 @@ def broadcast( self.broadcasted.append(prepared.signed_transaction) if self.broadcast_error is not None: raise self.broadcast_error + if self.returned_txid is not None: + return self.returned_txid return prepared.signed_transaction.replace("signed-", "txid-") def wait_for_confirmation(self, txid: str) -> int: @@ -204,6 +256,41 @@ def test_confirmed_retry_does_not_contact_the_gateway() -> None: assert gateway.broadcasted == [] +def test_concurrent_confirmation_before_broadcast_returns_terminal_receipt() -> None: + repository = ConfirmOnUpdateRepository("record_attempt") + existing = _intent() + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + service = AssetTransferService(repository, gateway) + + receipt = service.execute(_request()) + + assert receipt.already_confirmed is True + assert receipt.confirmed_round == 777 + assert gateway.broadcasted == [] + + +@pytest.mark.parametrize("update_name", ["mark_submitted", "record_error"]) +def test_concurrent_confirmation_after_broadcast_is_not_reported_pending( + update_name: str, +) -> None: + repository = ConfirmOnUpdateRepository(update_name) + existing = _intent() + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + if update_name == "record_error": + gateway.wait_error = TimeoutError("another worker confirmed") + else: + gateway.returned_txid = existing.txid + service = AssetTransferService(repository, gateway) + + receipt = service.execute(_request()) + + assert receipt.already_confirmed is True + assert receipt.confirmed_round == 777 + assert gateway.broadcasted == ["persisted-signed-transaction"] + + def test_idempotency_key_cannot_be_reused_for_another_amount() -> None: repository = InMemoryIntentRepository() existing = _intent() @@ -263,6 +350,64 @@ def test_reconcile_marks_an_observed_transaction_confirmed() -> None: assert result.confirmed_round == 777 +def test_reconcile_returns_confirmation_committed_during_chain_lookup() -> None: + confirmed = _intent( + status=TransferStatus.CONFIRMED, + confirmed_round=778, + ) + repository = ReconcileReloadRepository(confirmed) + existing = _intent() + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + service = AssetTransferService(repository, gateway) + + result = service.reconcile(existing.id) + + assert repository.get_count == 2 + assert result.status == TransferStatus.CONFIRMED + assert result.txid == existing.txid + assert result.confirmed_round == 778 + + +def test_reconcile_fails_if_the_intent_disappears_during_chain_lookup() -> None: + repository = ReconcileReloadRepository(None) + existing = _intent() + repository.intents[existing.id] = existing + service = AssetTransferService(repository, FakeTransferGateway()) + + with pytest.raises(AssetTransferError, match="disappeared during reconciliation"): + service.reconcile(existing.id) + + +def test_reconcile_fails_if_the_persisted_transaction_changes() -> None: + replacement = _intent(txid="replacement-txid") + repository = ReconcileReloadRepository(replacement) + existing = _intent() + repository.intents[existing.id] = existing + service = AssetTransferService(repository, FakeTransferGateway()) + + with pytest.raises(AssetTransferConflictError, match="changed transaction"): + service.reconcile(existing.id) + + +def test_reconcile_builds_nonterminal_result_from_reloaded_intent() -> None: + reloaded = _intent( + status=TransferStatus.PREPARED, + last_valid_round=99, + ) + repository = ReconcileReloadRepository(reloaded) + existing = _intent(last_valid_round=1_000) + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + gateway.current = 100 + service = AssetTransferService(repository, gateway) + + result = service.reconcile(existing.id) + + assert result.status == "expired_unconfirmed" + assert result.confirmed_round is None + + def test_full_algorand_uint64_values_are_persisted_without_bson_integers() -> None: repository = InMemoryIntentRepository() gateway = FakeTransferGateway() diff --git a/tests/unit/test_database_indexes.py b/tests/unit/test_database_indexes.py index 90b39af6..e3685670 100644 --- a/tests/unit/test_database_indexes.py +++ b/tests/unit/test_database_indexes.py @@ -6,6 +6,8 @@ from flex.db.indexes import ( create_unique_id_index_fail_closed, deduplicate_and_create_unique_id_index, + delete_unverified_legacy_lp_prices, + ensure_airdrop_indexes, ensure_database_indexes, ensure_sync_state_singleton, ) @@ -18,6 +20,7 @@ def _manager(collection: Mock) -> SimpleNamespace: def _database(**collections: Mock) -> SimpleNamespace: names = ( "airdrop_manifests", + "assets", "asset_prices", "asset_transfer_intents", "pool_transactions", @@ -94,10 +97,12 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: name: Mock() for name in ( "airdrop_manifests", + "assets", "asset_prices", "asset_transfer_intents", "pool_transactions", "lp_transactions", + "lp_tokens", ) } for collection in unique_collections.values(): @@ -106,24 +111,83 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: lp_states.aggregate.return_value = [] sync_states = Mock() sync_states.find.return_value = [] + airdrop_rewards = Mock() + airdrop_rewards.aggregate.return_value = [] database = _database( **unique_collections, lp_states=lp_states, sync_states=sync_states, + airdrop_rewards=airdrop_rewards, + ) + database.asset_prices.mongodb_collection.delete_many.return_value = SimpleNamespace( + deleted_count=1, ) removed = ensure_database_indexes(database) assert removed == { "airdrop_manifests": 0, + "assets": 0, "asset_prices": 0, "asset_transfer_intents": 0, "pool_transactions": 0, "lp_transactions": 0, + "lp_tokens": 0, } for collection in unique_collections.values(): collection.create_index.assert_called_once_with("id", unique=True, name="id_unique") + database.asset_prices.mongodb_collection.delete_many.assert_called_once_with( + { + "$or": [ + { + "tinyman_algo_pool_id": { + "$exists": True, + "$ne": None, + } + }, + {"source": "derived_lp"}, + ] + } + ) + assert database.asset_prices.mongodb_collection.mock_calls.index( + call.delete_many( + { + "$or": [ + { + "tinyman_algo_pool_id": { + "$exists": True, + "$ne": None, + } + }, + {"source": "derived_lp"}, + ] + } + ) + ) < database.asset_prices.mongodb_collection.mock_calls.index( + call.aggregate( + [ + { + "$sort": { + "id": 1, + "observed_at": -1, + "updated": -1, + "_id": -1, + } + }, + { + "$group": { + "_id": "$id", + "count": {"$sum": 1}, + "keep_id": {"$first": "$_id"}, + "all_ids": {"$push": "$_id"}, + } + }, + {"$match": {"count": {"$gt": 1}}}, + ], + allowDiskUse=True, + ) + ) assert database.lp_states.mongodb_collection.create_index.call_args_list == [ call("token_id", unique=True, name="token_id_unique"), @@ -131,7 +195,6 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: ] database.pool_states.mongodb_collection.create_index.assert_called_once_with("pool_id", name="pool_id_idx") database.user_states.mongodb_collection.create_index.assert_called_once_with("address", name="address_idx") - database.lp_tokens.mongodb_collection.create_index.assert_called_once_with("id", name="lp_token_id_idx") database.airdrop_rewards.mongodb_collection.create_index.assert_called_once_with( "operation_id", unique=True, @@ -145,6 +208,60 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: ) +def test_standalone_airdrop_indexes_fail_closed_on_duplicate_operations() -> None: + manifests = Mock() + manifests.aggregate.return_value = [] + rewards = Mock() + rewards.aggregate.return_value = [ + { + "_id": "airdrop:summer:wallet", + "count": 2, + "keep_id": "first", + "all_ids": ["first", "second"], + } + ] + + with pytest.raises(RuntimeError, match="duplicate 'operation_id'"): + ensure_airdrop_indexes( + _database( + airdrop_manifests=manifests, + airdrop_rewards=rewards, + ) + ) + + manifests.create_index.assert_called_once_with( + "id", + unique=True, + name="id_unique", + ) + rewards.create_index.assert_not_called() + rewards.delete_many.assert_not_called() + + +def test_legacy_lp_price_cleanup_reports_deleted_rows() -> None: + collection = Mock() + collection.delete_many.return_value = SimpleNamespace(deleted_count=3) + + removed = delete_unverified_legacy_lp_prices( + _database(asset_prices=collection), + ) + + assert removed == 3 + collection.delete_many.assert_called_once_with( + { + "$or": [ + { + "tinyman_algo_pool_id": { + "$exists": True, + "$ne": None, + } + }, + {"source": "derived_lp"}, + ] + } + ) + + def test_single_legacy_sync_cursor_is_migrated_without_guessing_between_competitors() -> None: collection = Mock() collection.find.return_value = [ @@ -188,7 +305,12 @@ def test_competing_sync_cursors_fail_closed() -> None: def test_correctness_critical_index_failure_is_not_swallowed() -> None: database = _database() database.airdrop_manifests.mongodb_collection.aggregate.return_value = [] + database.airdrop_rewards.mongodb_collection.aggregate.return_value = [] + database.assets.mongodb_collection.aggregate.return_value = [] database.asset_prices.mongodb_collection.aggregate.return_value = [] + database.asset_prices.mongodb_collection.delete_many.return_value = SimpleNamespace( + deleted_count=0, + ) database.asset_prices.mongodb_collection.create_index.side_effect = RuntimeError("index build failed") with pytest.raises(RuntimeError, match="index build failed"): diff --git a/tests/unit/test_nft_lottery_payouts.py b/tests/unit/test_nft_lottery_payouts.py index 4fac286d..923a2f0b 100644 --- a/tests/unit/test_nft_lottery_payouts.py +++ b/tests/unit/test_nft_lottery_payouts.py @@ -1,6 +1,10 @@ from copy import deepcopy +from hashlib import sha256 from types import SimpleNamespace +import pytest +from pymongo.errors import OperationFailure + from flex.application.asset_transfers import AssetTransferReceipt @@ -19,18 +23,75 @@ def _matches(document: dict, query: dict) -> bool: if document.get(field) == expected["$ne"]: return False continue + if "$in" in expected: + if document.get(field) not in expected["$in"]: + return False + continue + if "$eq" in expected: + if document.get(field) != expected["$eq"]: + return False + continue if document.get(field) != expected: return False return True class FakeLotteryCollection: - def __init__(self, documents: list[dict]) -> None: + def __init__( + self, + documents: list[dict], + *, + index_information: dict[str, dict] | None = None, + ) -> None: self.documents = documents self.indexes: list[tuple[tuple, dict]] = [] + self.dropped_indexes: list[str] = [] + self.index_events: list[tuple[str, str]] = [] + self.find_one_and_update_calls: list[tuple[dict, dict]] = [] + self._index_information = deepcopy( + index_information + or { + "_id_": { + "key": [("_id", 1)], + } + } + ) - def create_index(self, *args, **kwargs) -> None: + def aggregate(self, pipeline, **kwargs): + del kwargs + matching_ids: dict[str, list[object]] = {} + for document in self.documents: + draw_id = document.get("id") + if isinstance(draw_id, str) and draw_id: + matching_ids.setdefault(draw_id, []).append(document.get("_id")) + return [ + { + "_id": draw_id, + "count": len(document_ids), + "document_ids": document_ids, + } + for draw_id, document_ids in matching_ids.items() + if len(document_ids) > 1 + ][: pipeline[-1].get("$limit", 10)] + + def index_information(self): + return deepcopy(self._index_information) + + def create_index(self, *args, **kwargs) -> str: self.indexes.append((args, kwargs)) + index_name = kwargs["name"] + self._index_information[index_name] = { + "key": [(args[0], 1)], + "unique": kwargs.get("unique", False), + "partialFilterExpression": deepcopy(kwargs.get("partialFilterExpression")), + } + self.index_events.append(("create", index_name)) + return index_name + + def drop_index(self, name: str) -> None: + self.dropped_indexes.append(name) + self._index_information.pop(name) + self.index_events.append(("drop", name)) def find(self, query): return [deepcopy(document) for document in self.documents if _matches(document, query)] @@ -40,6 +101,7 @@ def find_one(self, query): return deepcopy(document) if document is not None else None def find_one_and_update(self, query, update, **kwargs): + self.find_one_and_update_calls.append((deepcopy(query), deepcopy(update))) document = next((item for item in self.documents if _matches(item, query)), None) if document is None: return None @@ -50,10 +112,178 @@ def update_one(self, query, update): document = next((item for item in self.documents if _matches(item, query)), None) if document is not None: document.update(update.get("$set", {})) - return SimpleNamespace(modified_count=int(document is not None)) + matched_count = int(document is not None) + return SimpleNamespace( + matched_count=matched_count, + modified_count=matched_count, + ) -def test_lottery_payouts_backfill_unique_draw_ids_and_claim_exact_documents(monkeypatch) -> None: +class ClaimBeforePrepareCollection(FakeLotteryCollection): + def update_one(self, query, update): + if "payout_operation_id" in query: + self.documents[0].update( + { + "claimed": True, + "payout_operation_id": "nft:manual-reconciliation", + "payout_txid": "manual-chain-tx", + "payout_status": "confirmed", + } + ) + return super().update_one(query, update) + + +class ConcurrentLegacyDropCollection(FakeLotteryCollection): + def drop_index(self, name: str) -> None: + self._index_information.pop(name, None) + self.index_events.append(("concurrent_drop", name)) + raise OperationFailure("index not found", code=27) + + +def _legacy_draw_id(document_id: str) -> str: + return f"legacy-{sha256(f'lottery-draw:{document_id}'.encode()).hexdigest()}" + + +def _receipt(asset_id: int, idempotency_key: str) -> AssetTransferReceipt: + return AssetTransferReceipt( + operation_id=f"nft:{idempotency_key}", + txid=f"tx-{asset_id}", + confirmed_round=777, + already_confirmed=False, + ) + + +def test_new_lottery_draw_starts_with_pending_payout(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection([]) + manager = SimpleNamespace( + collection=collection, + create=lambda draw: draw, + ) + monkeypatch.setattr(nft_lottery, "lottery_draws", manager) + + draw = nft_lottery._create_draw( + lottery_name="summer", + prize=101, + wallet="WALLET-A", + timestamp=1.0, + ) + + assert draw.id + assert draw.payout_status == nft_lottery.LotteryPayoutStatus.PENDING + assert collection.indexes == [ + ( + ("id",), + { + "unique": True, + "name": nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME, + "partialFilterExpression": { + "id": { + "$type": "string", + "$gt": "", + } + }, + }, + ) + ] + + +def test_lottery_index_upgrade_builds_replacement_before_dropping_legacy( + monkeypatch, +) -> None: + from api import nft_lottery + + documents = [ + {"_id": "legacy-a"}, + {"_id": "legacy-b", "id": ""}, + {"_id": "current", "id": "draw-current"}, + ] + collection = FakeLotteryCollection( + deepcopy(documents), + index_information={ + "_id_": {"key": [("_id", 1)]}, + "id_unique": { + "key": [("id", 1)], + "unique": True, + "partialFilterExpression": {"id": {"$type": "string"}}, + }, + }, + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + + nft_lottery.ensure_lottery_indexes() + + assert collection.documents == documents + assert collection.index_events == [ + ("create", nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME), + ("drop", "id_unique"), + ] + replacement = collection.index_information()[nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME] + assert replacement["partialFilterExpression"] == nft_lottery.LOTTERY_DRAW_ID_FILTER + + +def test_lottery_index_upgrade_preserves_duplicate_evidence_and_fails_closed( + monkeypatch, +) -> None: + from api import nft_lottery + + documents = [ + {"_id": "mongo-a", "id": "duplicate"}, + {"_id": "mongo-b", "id": "duplicate"}, + ] + collection = FakeLotteryCollection(deepcopy(documents)) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + + with pytest.raises(RuntimeError, match="preserved 1 duplicate group"): + nft_lottery.ensure_lottery_indexes() + + assert collection.documents == documents + assert collection.indexes == [] + assert collection.dropped_indexes == [] + + +def test_lottery_index_upgrade_converges_after_concurrent_legacy_drop( + monkeypatch, +) -> None: + from api import nft_lottery + + collection = ConcurrentLegacyDropCollection( + [{"_id": "current", "id": "draw-current"}], + index_information={ + "_id_": {"key": [("_id", 1)]}, + "id_unique": { + "key": [("id", 1)], + "unique": True, + "partialFilterExpression": {"id": {"$type": "string"}}, + }, + }, + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + + nft_lottery.ensure_lottery_indexes() + + replacement = collection.index_information()[nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME] + assert replacement["partialFilterExpression"] == nft_lottery.LOTTERY_DRAW_ID_FILTER + assert collection.index_events == [ + ("create", nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME), + ("concurrent_drop", "id_unique"), + ] + + +def test_legacy_lottery_payouts_require_reconciliation_and_never_send(monkeypatch) -> None: # Import after test collection helpers so module-level infrastructure stays # outside the behavioral assertion. from api import nft_lottery @@ -75,6 +305,7 @@ def test_lottery_payouts_backfill_unique_draw_ids_and_claim_exact_documents(monk "timestamp": 2.0, "lottery_name": "summer", "claimed": False, + "payout_txid": "legacy-chain-tx", }, ] ) @@ -87,13 +318,389 @@ def test_lottery_payouts_backfill_unique_draw_ids_and_claim_exact_documents(monk def send_nft(address: str, asset_id: int, *, idempotency_key: str): calls.append((address, asset_id, idempotency_key)) - operation_id = f"nft:{idempotency_key}" - return AssetTransferReceipt( - operation_id=operation_id, - txid=f"tx-{asset_id}", - confirmed_round=777, - already_confirmed=False, - ) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 0 + assert result["error_count"] == 2 + assert calls == [] + assert not any(document["claimed"] for document in collection.documents) + assert all(document["id"].startswith("legacy-") for document in collection.documents) + assert all( + document["payout_status"] == nft_lottery.LotteryPayoutStatus.RECONCILIATION_REQUIRED + for document in collection.documents + ) + assert collection.documents[1]["payout_txid"] == "legacy-chain-tx" + + retry = nft_lottery.send_all_prizes() + + assert retry["sent_count"] == 0 + assert retry["error_count"] == 2 + assert calls == [] + + +def test_existing_draw_without_payout_state_is_legacy_and_fails_closed(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "mongo-a", + "id": "pre-status-draw", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 0 + assert result["error_count"] == 1 + assert calls == [] + assert collection.documents[0]["payout_status"] == nft_lottery.LotteryPayoutStatus.RECONCILIATION_REQUIRED + + +def test_empty_legacy_identity_and_status_are_migrated_fail_closed(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "mongo-a", + "id": "", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + "payout_status": "", + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + monkeypatch.setattr( + nft_lottery, + "send_nft", + lambda *args, **kwargs: pytest.fail("legacy draw must not broadcast"), + ) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 0 + assert result["error_count"] == 1 + assert collection.documents[0]["id"].startswith("legacy-") + assert collection.documents[0]["payout_status"] == nft_lottery.LotteryPayoutStatus.RECONCILIATION_REQUIRED + + +def test_missing_claimed_is_atomically_normalized_and_fails_closed(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "mongo-a", + "id": "draw-a", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + monkeypatch.setattr( + nft_lottery, + "send_nft", + lambda *args, **kwargs: pytest.fail("uncertain legacy draw must not broadcast"), + ) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 0 + assert result["error_count"] == 1 + assert collection.documents[0]["claimed"] is False + assert collection.documents[0]["payout_status"] == nft_lottery.LotteryPayoutStatus.RECONCILIATION_REQUIRED + migration_query, migration_update = collection.find_one_and_update_calls[0] + assert migration_query["claimed"] == {"$exists": False} + assert migration_update["$set"] == { + "id": "draw-a", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.RECONCILIATION_REQUIRED, + } + + +def test_null_claimed_with_exact_durable_operation_is_safe_to_resume(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "mongo-a", + "id": "draw-a", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": None, + "payout_operation_id": "nft:lottery:draw-a", + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 1 + assert result["error_count"] == 0 + assert calls == [("WALLET-A", 101, "lottery:draw-a")] + assert collection.documents[0]["claimed"] is True + assert collection.documents[0]["payout_status"] == nft_lottery.LotteryPayoutStatus.CONFIRMED + migration_query, migration_update = collection.find_one_and_update_calls[0] + assert migration_query["claimed"] == {"$eq": None} + assert migration_update["$set"] == { + "id": "draw-a", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.PREPARED, + } + + +def test_malformed_legacy_draw_does_not_block_safe_payouts(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "malformed", + "prize": 101, + "claimed": False, + }, + { + "_id": "mongo-b", + "id": "draw-b", + "wallet": "WALLET-B", + "prize": 202, + "timestamp": 2.0, + "lottery_name": "summer", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + }, + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 1 + assert result["error_count"] == 1 + assert calls == [("WALLET-B", 202, "lottery:draw-b")] + assert "migration failed" in result["results"][0]["error"] + assert collection.documents[1]["claimed"] is True + + +def test_legacy_lottery_payout_with_matching_operation_is_safe_to_resume(monkeypatch) -> None: + from api import nft_lottery + + document_id = "draw-a" + legacy_id = _legacy_draw_id(document_id) + collection = FakeLotteryCollection( + [ + { + "_id": document_id, + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + "payout_operation_id": f"nft:lottery:{legacy_id}", + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 1 + assert result["error_count"] == 0 + assert calls == [("WALLET-A", 101, f"lottery:{legacy_id}")] + assert collection.documents[0]["claimed"] is True + assert collection.documents[0]["payout_status"] == nft_lottery.LotteryPayoutStatus.CONFIRMED + + +def test_failed_new_lottery_payout_becomes_unresolved(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "mongo-a", + "id": "draw-a", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + + def fail_send_nft(address: str, asset_id: int, *, idempotency_key: str): + raise RuntimeError("indexer unavailable") + + monkeypatch.setattr(nft_lottery, "send_nft", fail_send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 0 + assert result["error_count"] == 1 + assert collection.documents[0]["payout_operation_id"] == "nft:lottery:draw-a" + assert collection.documents[0]["payout_status"] == nft_lottery.LotteryPayoutStatus.UNRESOLVED + assert collection.documents[0]["send_error"] == "indexer unavailable" + + +def test_claim_won_before_reservation_never_broadcasts_again(monkeypatch) -> None: + from api import nft_lottery + + collection = ClaimBeforePrepareCollection( + [ + { + "_id": "mongo-a", + "id": "draw-a", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 0 + assert result["error_count"] == 0 + assert result["results"][0]["already_claimed"] is True + assert result["results"][0]["txid"] == "manual-chain-tx" + assert calls == [] + + +def test_new_lottery_payouts_use_stable_ids_and_claim_exact_documents(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "mongo-a", + "id": "draw-a", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + }, + { + "_id": "mongo-b", + "id": "draw-b", + "wallet": "WALLET-B", + "prize": 202, + "timestamp": 2.0, + "lottery_name": "summer", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + }, + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) monkeypatch.setattr(nft_lottery, "send_nft", send_nft) @@ -101,14 +708,18 @@ def send_nft(address: str, asset_id: int, *, idempotency_key: str): assert result["sent_count"] == 2 assert result["error_count"] == 0 - assert len(calls) == 2 - assert len({call[2] for call in calls}) == 2 + assert calls == [ + ("WALLET-A", 101, "lottery:draw-a"), + ("WALLET-B", 202, "lottery:draw-b"), + ] assert [document["payout_txid"] for document in collection.documents] == [ "tx-101", "tx-202", ] assert all(document["claimed"] for document in collection.documents) - assert all(document["id"].startswith("legacy-") for document in collection.documents) + assert all( + document["payout_status"] == nft_lottery.LotteryPayoutStatus.CONFIRMED for document in collection.documents + ) first_calls = list(calls) retry = nft_lottery.send_all_prizes() From 44698ee7e59943a365b6e8ebe07147fd9aa3187f Mon Sep 17 00:00:00 2001 From: wackloner Date: Sun, 19 Jul 2026 22:26:39 +0700 Subject: [PATCH 09/10] separate lp ledger from trusted pricing --- api/background.py | 16 +- docs/architecture/lp-projection.md | 60 ++- flex/api.py | 5 +- flex/application/price_refresh.py | 9 +- flex/blockchain/info.py | 51 ++- flex/data/asset_prices.py | 61 ++- flex/data/assets.py | 104 ++++- flex/data/lp_prices.py | 345 ---------------- flex/data/lp_registry.py | 131 ++++++ flex/data/lp_states.py | 205 ++++------ flex/data/lp_tokens.py | 50 ++- flex/data/stats.py | 11 +- flex/data/tinyman_lps.py | 101 ----- flex/db/lp_projection.py | 79 ++-- flex/db/model/blockchain.py | 29 +- flex/db/model/liquidity_pools.py | 50 ++- flex/domain/algorand.py | 17 + flex/domain/lp_projection.py | 32 +- flex/domain/pricing.py | 22 +- flex/providers/price_router.py | 8 +- flex/sync_pools.py | 124 +++--- .../test_legacy_price_cutover_mongo.py | 72 ++++ .../test_mongo_financial_projection.py | 248 ++++++++++- .../unit/test_algorand_snapshot_boundaries.py | 101 +++++ tests/unit/test_asset_price_fallback.py | 93 ++++- tests/unit/test_asset_supply.py | 251 +++++++++++- tests/unit/test_bson_uint64_models.py | 57 ++- tests/unit/test_lp_api_contract.py | 40 +- tests/unit/test_lp_projection_domain.py | 27 ++ tests/unit/test_lp_projection_repository.py | 147 ++++++- tests/unit/test_lp_token_identity.py | 51 +++ tests/unit/test_lp_transaction_projection.py | 384 +++++++++++++++++- tests/unit/test_price_background.py | 106 ++--- tests/unit/test_price_router.py | 102 ++--- tests/unit/test_pricing_domain.py | 195 +++++---- tests/unit/test_stats_pricing.py | 49 +++ tests/unit/test_tinyman_price_projection.py | 99 ----- 37 files changed, 2421 insertions(+), 1111 deletions(-) delete mode 100644 flex/data/lp_prices.py create mode 100644 flex/data/lp_registry.py delete mode 100644 flex/data/tinyman_lps.py create mode 100644 flex/domain/algorand.py create mode 100644 tests/integration/test_legacy_price_cutover_mongo.py create mode 100644 tests/unit/test_algorand_snapshot_boundaries.py create mode 100644 tests/unit/test_lp_projection_domain.py create mode 100644 tests/unit/test_lp_token_identity.py create mode 100644 tests/unit/test_stats_pricing.py delete mode 100644 tests/unit/test_tinyman_price_projection.py diff --git a/api/background.py b/api/background.py index 2c195352..41672287 100644 --- a/api/background.py +++ b/api/background.py @@ -25,7 +25,7 @@ is_asset_price_stale, update_asset_price, ) -from flex.data.lp_prices import get_lp_token_definitions, update_lp_token_prices +from flex.data.lp_registry import get_lp_token_definitions from flex.migrations import migrate_background from flex.sync_pools import sync_pools_loop @@ -161,7 +161,8 @@ async def update_asset_prices_background(): all_assets = db.assets.get_all() - # Exclude LP tokens — they are priced separately by update_lp_token_prices() + # LP tokens have no verified economic-reserve adapter. Keep them out of the + # generic provider refresh instead of deriving prices from account balances. try: lp_defs = await get_lp_token_definitions() lp_token_ids = {d["lp_token_id"] for d in lp_defs} @@ -241,14 +242,9 @@ async def update_asset_prices_background(): ) if settings.background_lp_prices_update: - # This legacy worker prices raw account balances. It is opt-in until - # every supported DEX has a verified economic-reserve adapter. - try: - await update_lp_token_prices(current_round) - except Exception: - logger.exception("LP token price update failed") - else: - logger.info("Background LP price updates are disabled") + logger.error( + "BACKGROUND_LP_PRICES_UPDATE is retired and ignored; raw pool-account balances cannot publish prices", + ) def run_background(): diff --git a/docs/architecture/lp-projection.md b/docs/architecture/lp-projection.md index 487907c9..42ea0e05 100644 --- a/docs/architecture/lp-projection.md +++ b/docs/architecture/lp-projection.md @@ -19,8 +19,13 @@ repairs the missing marker, and does not apply the delta again. Before the first write, the complete block batch is checked for its expected round, deterministic order, duplicate IDs, and conflicting payloads. LP -self-transfers are discarded as net zero. Clawback and close semantics fail -closed only when they affect an LP account. +self-transfer amount legs are discarded as net zero, but their network fee is +still projected. Every pool-paid fee is a separate, stable +`#fee@` ALGO event, so retry cannot combine or repeat it. +Clawback and close semantics fail closed when they affect a tracked LP account. +Indexer rounds, asset IDs, amounts, balances, and fees reject booleans, +coercible strings, negatives, and overflow before any delta is negated or any +repository write begins. The repository fails closed when: @@ -29,24 +34,42 @@ The repository fails closed when: - a later cursor has passed an unrecorded event; - a balance would leave the Algorand uint64 range; - legacy root aliases make an inner event ambiguous; -- duplicate LP token IDs or addresses exist. +- duplicate LP token IDs or addresses exist; +- one LP-token ASA is discovered with conflicting immutable pool metadata. -Reserves, issued LP supply, IDs, rounds, positions, and deltas are Python +Balances, issued LP supply, IDs, rounds, positions, fees, and deltas are Python integers encoded as BSON Decimal128 where MongoDB int64 is insufficient. -Pricing stays in `Decimal`; floats exist only in legacy API fields. ASA total -supply is read directly from Indexer base units and persisted as a decimal -string. - -## Snapshots and derived prices +Token-token pools keep fee-funding ALGO in +`operational_algo_balance_micros`, separate from the two economic asset +balances. ASA total supply is read directly from Indexer base units and +persisted as a decimal string with `total_supply_source=indexer`. A legacy or +generic rewrite without that provenance is refetched and migrated atomically +before a financial read may use it. A unique asset ID index makes concurrent +creation fail closed. LP-token registration likewise uses a unique Decimal128 +ASA ID plus atomic `get-or-create`, then compares pool ID, assets, address, and +DEX provider against the persisted winner. + +## Snapshots and pricing isolation Account balances and `current-round` come from the same Indexer response. An authoritative snapshot advances the cursor to that round’s end sentinel. A stale snapshot cannot overwrite a newer event, and conflicting balances for the -same snapshot round require reconciliation. Price writers update only derived -fields when the state cursor still matches; a monotonic observation timestamp -prevents an older calculation from overwriting a newer quote. Tinyman reserve -projection writes only the underlying asset quote; the LP token price has one -canonical writer. +same snapshot round require reconciliation. Snapshots update only integer ledger +fields and cursors: they never calculate or publish a price. +Duplicate holdings and out-of-range values are rejected at the Indexer adapter; +issued supply is range-checked before subtraction. Refresh builds an immutable +candidate state, and the repository repeats validation before issuing its +single-document CAS, so a failed snapshot cannot partially mutate in-memory or +persisted balances. + +Indexer account balances are useful for reconciliation but are not proof of a +DEX’s economic reserves. Donations, minimum-balance funding, and protocol excess +can all be present. The ledger projector therefore has no price-publishing +dependency. Public LP prices come from the separately validated `asset_prices` +read model with source and freshness checks. The raw-balance publisher has been +removed; startup deletes its identifiable legacy rows and read paths +independently reject them. A future DEX adapter must derive economic reserves +from verified protocol state before it can become a price source. ## Worker ordering @@ -69,9 +92,6 @@ documents simultaneously. Replay guarantees convergence; strict cross-pool visibility would require a replica-set transaction or a one-document ledger aggregate. -Indexer account balances are authoritative for reconciliation, but not -necessarily for a DEX’s economic reserve accounting: donations and protocol -excess balances may be included. The legacy account-balance price worker is -therefore independently disabled by default with -`BACKGROUND_LP_PRICES_UPDATE=false` until each DEX has a verified app-state -adapter. +`BACKGROUND_LP_PRICES_UPDATE` remains accepted only for deployment +compatibility and cannot restore the removed publisher. Enabling LP ledger sync +does not bypass this boundary. diff --git a/flex/api.py b/flex/api.py index 31f12009..b2b85e4e 100644 --- a/flex/api.py +++ b/flex/api.py @@ -287,8 +287,9 @@ async def _fetch_asset_safe(aid): except Exception as e: logger.error(f"Batch price creation failed: {e}") - # LP prices are now in asset_prices collection (populated by background worker). - # Keep lp_states key as empty dict for frontend compatibility. + # This route only consumes independently validated stored observations. + # No active publisher derives LP prices from pool-account balances. + # Keep lp_states empty for frontend compatibility. lp_states_dict = {} assets_dict = {a.id: a.to_details().to_dict() for a in assets_list} diff --git a/flex/application/price_refresh.py b/flex/application/price_refresh.py index 67ac245a..190b5b2c 100644 --- a/flex/application/price_refresh.py +++ b/flex/application/price_refresh.py @@ -6,7 +6,13 @@ import httpx -from flex.domain.pricing import DecimalInput, PriceQuote, PriceSource, PricingError +from flex.domain.pricing import ( + DecimalInput, + PriceQuote, + PriceSource, + PricingError, + validate_observation_timestamp, +) from flex.meta_error import MetaError from flex.providers import vestige @@ -76,6 +82,7 @@ def validate_provider_quote( stale_after=fresh_for, observed_round=observed_round, ) + validate_observation_timestamp(quote.observed_at) quote.to_legacy_floats() return quote except PriceRefreshError: diff --git a/flex/blockchain/info.py b/flex/blockchain/info.py index 7c494a48..6917b76e 100644 --- a/flex/blockchain/info.py +++ b/flex/blockchain/info.py @@ -6,7 +6,8 @@ from blockchain.node import get_current_round as _sync_get_current_round from flex.blockchain.base import algod_client, indexer_client -from flex.db.model.blockchain import Asset +from flex.db.model.blockchain import TOTAL_SUPPLY_SOURCE_INDEXER, Asset +from flex.domain.algorand import require_algorand_uint64 logger = logging.getLogger(__name__) @@ -26,6 +27,7 @@ class AssetBalanceSnapshot: reserve="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ", total_supply=10_000_000_000, total_supply_micros=10_000_000_000_000_000, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, ) @@ -50,6 +52,7 @@ async def fetch_asset(asset_id: int) -> Asset: reserve=params.get("reserve", ""), total_supply=0, total_supply_micros=params["total"], + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, ) @@ -69,12 +72,48 @@ async def get_address_asset_snapshot( """Read balances and their authoritative Indexer round in one response.""" data = await _run_sync(indexer_client.account_info, address) - asset_balances = {asset["asset-id"]: asset["amount"] for asset in data["account"]["assets"]} + account_data = data.get("account") + if not isinstance(account_data, dict): + raise RuntimeError("Indexer account snapshot has no account object") + raw_assets = account_data.get("assets") + if not isinstance(raw_assets, list): + raise RuntimeError("Indexer account snapshot has no asset holdings") + + asset_balances: dict[int, int] = {} + for raw_holding in raw_assets: + if not isinstance(raw_holding, dict): + raise RuntimeError("Indexer account snapshot contains a malformed asset holding") + try: + asset_id = require_algorand_uint64( + raw_holding.get("asset-id"), + "Indexer asset-id", + positive=True, + ) + amount = require_algorand_uint64( + raw_holding.get("amount"), + "Indexer asset amount", + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + if asset_id in asset_balances: + raise RuntimeError(f"Indexer account snapshot repeats asset {asset_id}") + asset_balances[asset_id] = amount + if include_algo: - asset_balances[0] = data["account"]["amount"] - observed_round = data.get("current-round") - if isinstance(observed_round, bool) or not isinstance(observed_round, int) or observed_round < 0: - raise RuntimeError("Indexer account snapshot has no valid current-round") + try: + asset_balances[0] = require_algorand_uint64( + account_data.get("amount"), + "Indexer ALGO amount", + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + try: + observed_round = require_algorand_uint64( + data.get("current-round"), + "Indexer current-round", + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc return AssetBalanceSnapshot( balances=asset_balances, observed_round=observed_round, diff --git a/flex/data/asset_prices.py b/flex/data/asset_prices.py index 667edc4e..497975a0 100644 --- a/flex/data/asset_prices.py +++ b/flex/data/asset_prices.py @@ -17,11 +17,13 @@ from flex.data.assets import get_asset_details from flex.db.model.priced import AssetPrice, AssetPriceInfo from flex.domain.pricing import ( + MAX_OBSERVATION_CLOCK_SKEW, PriceQuote, PriceSource, PriceUnavailableError, PricingError, is_observation_stale, + validate_observation_timestamp, ) from flex.util import build_key_str @@ -70,6 +72,15 @@ def is_asset_price_stale( ) +def is_unverified_legacy_lp_price(asset_price: object) -> bool: + """Return whether a price came from the retired raw-reserve projection.""" + + return ( + getattr(asset_price, "tinyman_algo_pool_id", None) is not None + or getattr(asset_price, "source", None) == PriceSource.DERIVED_LP.value + ) + + def validated_stored_asset_price( asset_price: AssetPrice, *, @@ -78,6 +89,13 @@ def validated_stored_asset_price( ) -> AssetPrice | None: """Return a stored price only when its values and age satisfy policy.""" + if is_unverified_legacy_lp_price(asset_price): + logger.warning( + "Ignoring retired raw-reserve LP projection for asset %s", + getattr(asset_price, "id", "unknown"), + ) + return None + current_time = now or datetime.now(UTC) if is_asset_price_stale( asset_price, @@ -126,10 +144,20 @@ def _skip_asset_price_cache(asset_price: AssetPrice) -> bool: def _upsert_asset_price(asset_price: AssetPrice) -> bool: """Persist a quote only if it is not older than the stored observation.""" + if is_unverified_legacy_lp_price(asset_price): + raise PricingError( + "raw reserve LP projections are retired and cannot be persisted", + ) if not isinstance(asset_price.observed_at, datetime): raise PricingError("persisted asset prices require an observation timestamp") - asset_price.updated = datetime.now(UTC) + current_time = datetime.now(UTC) + observed_at = validate_observation_timestamp( + asset_price.observed_at, + now=current_time, + ) + asset_price.observed_at = observed_at + asset_price.updated = current_time doc = asset_price.to_dict() doc.pop("_id", None) created = doc.pop("created", asset_price.created) @@ -142,7 +170,12 @@ def _upsert_asset_price(asset_price: AssetPrice) -> bool: {"observed_at": None}, { "observed_at": { - "$lte": asset_price.observed_at, + "$lte": observed_at, + } + }, + { + "observed_at": { + "$gt": current_time + MAX_OBSERVATION_CLOCK_SKEW, } }, ], @@ -173,8 +206,21 @@ def _upsert_asset_price(asset_price: AssetPrice) -> bool: try: collection.insert_one({**doc, "created": created}) except DuplicateKeyError: + # The concurrent insert may be older than this observation. Re-run the + # same conditional write so arrival order cannot defeat timestamp order. + retry = collection.update_one( + selector, + {"$set": doc}, + upsert=False, + ) + if retry.matched_count: + logger.info( + "Replaced a concurrently inserted older price for asset %s", + asset_price.id, + ) + return True logger.info( - "A concurrent price writer won the insert for asset %s", + "A concurrent price writer kept a newer observation for asset %s", asset_price.id, ) return False @@ -183,9 +229,9 @@ def _upsert_asset_price(asset_price: AssetPrice) -> bool: def _current_price_after_rejected_write(asset_id: int) -> AssetPrice: current = db.asset_prices.get_one(id=asset_id) - if current is None: + if current is None or is_unverified_legacy_lp_price(current): raise PyMongoError( - f"price write for asset {asset_id} lost a race but no winner exists", + f"price write for asset {asset_id} lost a race but no safe winner exists", ) return current @@ -197,8 +243,8 @@ async def create_asset_prices_batch( ) -> list[AssetPrice]: """Create prices for multiple assets using Vestige batch API. - LP token prices are handled by the background worker (lp_prices.py), - not by this function. + LP tokens are excluded by the background registry until a verified + economic-reserve adapter exists. """ from flex.providers.vestige import vestige_batch_prices @@ -294,6 +340,7 @@ async def update_asset_price( price_algo=price_algo, price_usd=price_usd, last_update_round=quote.observed_round if quote.observed_round is not None else current_round, + tinyman_algo_pool_id=None, source=quote.source.value, observed_at=quote.observed_at, ) diff --git a/flex/data/assets.py b/flex/data/assets.py index a9d0e4c5..026bc2c5 100644 --- a/flex/data/assets.py +++ b/flex/data/assets.py @@ -1,10 +1,20 @@ +import asyncio import logging +from datetime import UTC, datetime +from typing import Any from aiocache import cached +from pymongo import ReturnDocument from flex import db -from flex.blockchain.info import fetch_asset, ALGO_ASSET -from flex.db.model.blockchain import Asset, AssetInfo, AssetDetails +from flex.blockchain.info import ALGO_ASSET, fetch_asset +from flex.db.bson import decode_bson_uint64 +from flex.db.model.blockchain import ( + TOTAL_SUPPLY_SOURCE_INDEXER, + Asset, + AssetDetails, + AssetInfo, +) from flex.providers.tinyman import get_asset_logo_url from flex.util import build_key_str @@ -15,29 +25,103 @@ async def create_asset(asset_id: int) -> Asset: asset = await fetch_asset(asset_id) asset.logo_url = get_asset_logo_url(asset_id) db.assets.create(asset) - logger.info(f'New Asset: id={asset.id}, name={asset.name}') + logger.info(f"New Asset: id={asset.id}, name={asset.name}") return asset -@cached(ttl=300, namespace='full_asset', key_builder=build_key_str) +@cached(ttl=300, namespace="full_asset", key_builder=build_key_str) async def get_full_asset(asset_id: int) -> Asset: + document = await asyncio.to_thread( + db.assets.mongodb_collection.find_one, + Asset.encode_query({"id": asset_id}), + projection={"total_supply_micros": 1, "total_supply_source": 1}, + ) + if document is not None and _canonical_supply(document) is None: + await _backfill_canonical_supply(asset_id, document) + asset = db.assets.get_by_primary_key(asset_id, throw_ex=False) if asset is None: asset = await create_asset(asset_id) + if not asset.total_supply_is_authoritative: + raise RuntimeError(f"asset {asset_id} canonical supply migration could not be verified") return asset -@cached(namespace='asset_total_supply', key_builder=build_key_str) +def _canonical_supply(document: dict[str, Any] | None) -> int | None: + if document is None or document.get("total_supply_source") != TOTAL_SUPPLY_SOURCE_INDEXER: + return None + value = document.get("total_supply_micros") + return None if value is None else decode_bson_uint64(value) + + +async def _backfill_canonical_supply( + asset_id: int, + document: dict[str, Any], +) -> int: + """Replace a lossy legacy display supply with authoritative Indexer units.""" + + document_id = document.get("_id") + if document_id is None: + raise RuntimeError(f"asset {asset_id} is missing its MongoDB identity") + + authoritative = await fetch_asset(asset_id) + total_supply_micros = authoritative.total_supply_micros + updated = await asyncio.to_thread( + db.assets.mongodb_collection.find_one_and_update, + { + "_id": document_id, + "$or": [ + {"total_supply_micros": {"$exists": False}}, + {"total_supply_micros": None}, + {"total_supply_source": {"$ne": TOTAL_SUPPLY_SOURCE_INDEXER}}, + ], + }, + { + "$set": { + "total_supply_micros": str(total_supply_micros), + "total_supply_source": TOTAL_SUPPLY_SOURCE_INDEXER, + "total_supply": authoritative.total_supply, + "updated": datetime.now(UTC), + } + }, + return_document=ReturnDocument.AFTER, + ) + if updated is None: + updated = await asyncio.to_thread( + db.assets.mongodb_collection.find_one, + {"_id": document_id}, + projection={"total_supply_micros": 1, "total_supply_source": 1}, + ) + persisted_supply = _canonical_supply(updated) + if persisted_supply is None: + raise RuntimeError(f"asset {asset_id} canonical supply migration was not persisted") + if persisted_supply != total_supply_micros: + raise RuntimeError(f"asset {asset_id} canonical supply changed during migration") + return persisted_supply + + +@cached(namespace="asset_total_supply", key_builder=build_key_str) async def get_asset_total_supply(asset_id: int) -> int: + query = Asset.encode_query({"id": asset_id}) + document = await asyncio.to_thread( + db.assets.mongodb_collection.find_one, + query, + projection={"total_supply_micros": 1, "total_supply_source": 1}, + ) + canonical_supply = _canonical_supply(document) + if canonical_supply is not None: + return canonical_supply + if document is not None: + return await _backfill_canonical_supply(asset_id, document) return (await get_full_asset(asset_id)).total_supply_micros -@cached(namespace='asset_info', key_builder=build_key_str) +@cached(namespace="asset_info", key_builder=build_key_str) async def get_asset_info(asset_id: int) -> AssetInfo: return (await get_full_asset(asset_id)).to_info() -@cached(namespace='asset_details', key_builder=build_key_str) +@cached(namespace="asset_details", key_builder=build_key_str) async def get_asset_details(asset_id: int) -> AssetDetails: return (await get_full_asset(asset_id)).to_details() @@ -46,7 +130,7 @@ async def get_asset_details_by_query(query_dict: dict) -> list[AssetDetails]: return [asset.to_details() for asset in db.assets.get_many_by_query(query_dict)] -@cached(ttl=20, namespace='all_asset_details', key='420') +@cached(ttl=20, namespace="all_asset_details", key="420") async def get_all_asset_details() -> list[AssetDetails]: all_assets = db.assets.get_all() return [asset.to_details() for asset in all_assets] @@ -61,7 +145,7 @@ async def amount_to_micros(asset_id: int, amount: float) -> int: async def load_all_assets_data() -> list[Asset]: - logger.info('Loading all assets data.') + logger.info("Loading all assets data.") asset_ids = {ALGO_ASSET.id} @@ -80,5 +164,5 @@ async def load_all_assets_data() -> list[Asset]: asset = await get_full_asset(asset_id) assets.append(asset) - logger.info(f'{len(asset_ids)} assets data loaded.') + logger.info(f"{len(asset_ids)} assets data loaded.") return assets diff --git a/flex/data/lp_prices.py b/flex/data/lp_prices.py deleted file mode 100644 index 86daf88c..00000000 --- a/flex/data/lp_prices.py +++ /dev/null @@ -1,345 +0,0 @@ -import asyncio -import logging -from datetime import timedelta -from decimal import Decimal - -from aiocache import cached -from algosdk.error import AlgodHTTPError - -from blockchain.node import get_current_round -from core.db.contracts import get_contracts_by_type -from core.util import parse_bignum -from env import settings -from flex import db -from flex.blockchain.base import algod_client -from flex.blockchain.info import _run_sync -from flex.data.asset_prices import _upsert_asset_price -from flex.data.assets import get_asset_details -from flex.db.model.priced import AssetPrice -from flex.domain.pricing import ( - PriceQuote, - PriceSource, - PriceUnavailableError, - PricingError, -) -from flex.domain.pricing import ( - calculate_lp_token_price_algo as calculate_lp_token_price_algo_exact, -) -from flex.providers import price_router - -logger = logging.getLogger(__name__) - -LP_CONCURRENCY = 2 -LP_REQUEST_DELAY = 1.0 # seconds between algod calls per slot - - -class LpTokenRegistryError(RuntimeError): - """The LP registry cannot safely distinguish derived from external prices.""" - - -def _extract_stake_token_id(contract) -> int | None: - """Extract stake token ID from contract metadata or cache.""" - meta = contract.metadata or {} - stid = meta.get("stake_token_id") - if stid is not None: - return int(stid) - - cache = meta.get("cache", {}) - initial = cache.get("initial", {}) - raw = initial.get("stakeToken") or initial.get("token") - if raw is None: - return None - if isinstance(raw, dict) and raw.get("type") == "BigNumber" and "hex" in raw: - return int(raw["hex"], 16) - try: - return int(raw) - except (ValueError, TypeError): - return None - - -def _get_active_stake_token_ids() -> set[int]: - """Return stake token IDs for farm contracts where rewards are still active. - - Only includes contracts where endBlock >= current_round (reward period not expired). - Ended farms have static LP reserves — their prices don't need frequent updates. - """ - contracts = get_contracts_by_type("farm") - current_round = get_current_round() - active_ids = set() - total = 0 - - for c in contracts: - stid = _extract_stake_token_id(c) - if not stid: - continue - total += 1 - - meta = c.metadata or {} - cache = meta.get("cache") - if not cache: - continue # no cache = can't determine endBlock, skip - - try: - end_block = parse_bignum(cache.get("initial", {}).get("endBlock")) - if end_block >= current_round: - active_ids.add(stid) - except (ValueError, TypeError): - continue - - logger.info(f"Active LP tokens: {len(active_ids)}/{total} farm contracts") - return active_ids - - -@cached(ttl=300, namespace="lp_token_defs") -async def get_lp_token_definitions() -> list[dict]: - """Build LP token registry from DB sources (no algod calls). - - Sources (in priority order): - 1. lp_tokens collection (has verified asset1_id, asset2_id, dex_provider) - 2. farming_pools collection (has first_token, second_token, dex_name) - 3. Contract metadata (asset1_id, dex fields) - - Returns list of dicts with lp_token_id, asset1_id, asset2_id, dex. - """ - contracts = get_contracts_by_type("farm") - - # Collect all stake token IDs from farm contracts - stake_token_ids = set() - expected_lp_token_ids = set() - for c in contracts: - stid = _extract_stake_token_id(c) - if stid: - stake_token_ids.add(stid) - meta = c.metadata or {} - if "asset1_id" in meta or "asset_1_id" in meta: - expected_lp_token_ids.add(stid) - - if not stake_token_ids: - logger.warning("No stake tokens found in farm contracts") - return [] - - lp_defs_by_id: dict[int, dict] = {} - from_lp_tokens = 0 - from_farming_pools = 0 - from_metadata = 0 - - # Source 1: lp_tokens collection (best quality — has verified pool data) - try: - lp_tokens_list = db.lp_tokens.get_many_by_query({"id": {"$in": list(stake_token_ids)}}) - for lt in lp_tokens_list: - lp_defs_by_id[lt.id] = { - "lp_token_id": lt.id, - "asset1_id": lt.asset1_id, - "asset2_id": lt.asset2_id, - "dex": lt.dex_provider, - } - from_lp_tokens += 1 - except Exception as exc: - raise LpTokenRegistryError("failed to query lp_tokens") from exc - - # Source 2: farming_pools collection - try: - farming_pools_list = db.farming_pools.get_all() - for fp in farming_pools_list: - stid = fp.stake_token.id - if stid not in stake_token_ids or stid in lp_defs_by_id: - continue - asset1_id = fp.first_token.id - asset2_id = fp.second_token.id - # Normalize: non-ALGO token as asset1 (matches lp_tokens convention) - if asset1_id == 0: - asset1_id, asset2_id = asset2_id, asset1_id - lp_defs_by_id[stid] = { - "lp_token_id": stid, - "asset1_id": asset1_id, - "asset2_id": asset2_id, - "dex": fp.dex_name, - } - from_farming_pools += 1 - except Exception as exc: - raise LpTokenRegistryError("failed to query farming_pools") from exc - - # Source 3: contract metadata - for c in contracts: - meta = c.metadata or {} - stid = _extract_stake_token_id(c) - if not stid or stid in lp_defs_by_id: - continue - asset1_id = meta.get("asset1_id", meta.get("asset_1_id")) - asset2_id = meta.get("asset2_id", meta.get("asset_2_id", 0)) - dex = meta.get("dex") or meta.get("dex_provider") - if asset1_id is not None and dex: - lp_defs_by_id[stid] = { - "lp_token_id": stid, - "asset1_id": int(asset1_id), - "asset2_id": int(asset2_id), - "dex": dex, - } - from_metadata += 1 - - unresolved_lp_ids = expected_lp_token_ids - lp_defs_by_id.keys() - if unresolved_lp_ids: - raise LpTokenRegistryError( - f"LP registry is incomplete for token ids {sorted(unresolved_lp_ids)}", - ) - - result = list(lp_defs_by_id.values()) - logger.info( - f"LP token definitions: {len(result)}/{len(stake_token_ids)} " - f"(lp_tokens={from_lp_tokens}, farming_pools={from_farming_pools}, " - f"metadata={from_metadata}, " - f"unresolved={len(stake_token_ids) - len(result)})" - ) - return result - - -def _get_pool_address(asset_info: dict) -> str: - """Get the pool address for an LP token. - - Tinyman V2 stores the pool address in 'reserve' (creator is the factory). - Other DEXes use 'creator' as the pool address directly. - """ - params = asset_info["params"] - reserve = params.get("reserve") - creator = params["creator"] - if reserve and reserve != creator: - return reserve - return creator - - -async def calculate_lp_token_price_algo(lp_def: dict) -> Decimal: - """Calculate LP token price from on-chain reserves and asset1 price. - - Formula: asset1_price_algo * asset1_reserve * 2 / circulating_lp_supply - The entire calculation stays in Decimal until the persistence boundary. - """ - price_algo, _ = await _calculate_lp_token_price_algo_with_quote(lp_def) - return price_algo - - -async def _calculate_lp_token_price_algo_with_quote( - lp_def: dict, -) -> tuple[Decimal, PriceQuote]: - """Calculate an LP price and retain the source asset observation.""" - - lp_token_id = lp_def["lp_token_id"] - asset1_id = lp_def["asset1_id"] - - # 1. Get LP token info (pool address + total supply) - asset_info = await _run_sync(algod_client.asset_info, lp_token_id) - pool_address = _get_pool_address(asset_info) - total_supply_micros = asset_info["params"]["total"] - - # 2. Get pool account balances - account_info = await _run_sync(algod_client.account_info, pool_address) - - asset_balances = {} - for asset in account_info.get("assets", []): - asset_balances[asset["asset-id"]] = asset["amount"] - # ALGO balance (needed when asset1 or asset2 is ALGO) - asset_balances[0] = account_info.get("amount", 0) - - asset1_reserve_micros = asset_balances.get(asset1_id, 0) - pool_lp_balance_micros = asset_balances.get(lp_token_id, 0) - - asset1_details = await get_asset_details(asset1_id) - asset1_quote = await price_router.get_asset_price_quote(asset1_id) - - return ( - calculate_lp_token_price_algo_exact( - asset1_price_algo=asset1_quote.algo, - asset1_reserve_micros=asset1_reserve_micros, - asset1_decimals=asset1_details.decimals, - total_lp_supply_micros=total_supply_micros, - pool_lp_balance_micros=pool_lp_balance_micros, - lp_token_decimals=asset_info["params"]["decimals"], - ), - asset1_quote, - ) - - -async def _update_single_lp( - lp_def: dict, - algo_quote: PriceQuote, - current_round: int, -) -> bool: - """Calculate and persist a single LP token price. Returns True on success.""" - try: - price_algo, asset1_quote = await _calculate_lp_token_price_algo_with_quote(lp_def) - quote = PriceQuote.from_raw( - asset_id=lp_def["lp_token_id"], - algo=price_algo, - usd=price_algo * algo_quote.usd, - source=PriceSource.DERIVED_LP, - stale_after=timedelta(seconds=settings.lp_prices_update_interval), - observed_round=current_round, - observed_at=min(asset1_quote.observed_at, algo_quote.observed_at), - ) - legacy_algo, legacy_usd = quote.to_legacy_floats() - asset_details = await get_asset_details(quote.asset_id) - asset_price = AssetPrice( - id=quote.asset_id, - name=asset_details.name, - price_algo=legacy_algo, - price_usd=legacy_usd, - last_update_round=current_round, - source=quote.source.value, - observed_at=quote.observed_at, - ) - _upsert_asset_price(asset_price) - return True - except (PriceUnavailableError, PricingError) as exc: - logger.warning("Failed LP price for token %s: %s", lp_def["lp_token_id"], exc) - return False - except (AlgodHTTPError, KeyError, TypeError, ValueError) as exc: - logger.warning("Failed LP price for token %s: %s", lp_def["lp_token_id"], exc) - return False - - -_last_lp_update_round: int = 0 - - -async def update_lp_token_prices(current_round: int) -> None: - """Calculate and persist LP token prices for active farm contracts only. - - Skips if fewer than lp_prices_update_interval seconds have elapsed. - Only prices LP tokens from farms that are still active or have stake, - dramatically reducing algod calls (from ~160 to ~10-20 tokens). - """ - global _last_lp_update_round - min_round_gap = int(settings.lp_prices_update_interval / settings.block_time) - if _last_lp_update_round and (current_round - _last_lp_update_round) < min_round_gap: - return - - lp_defs = await get_lp_token_definitions() - if not lp_defs: - return - - # Only price LP tokens from active farms (endBlock not passed or still has stake) - active_ids = _get_active_stake_token_ids() - active_defs = [d for d in lp_defs if d["lp_token_id"] in active_ids] - logger.info(f"LP pricing: {len(active_defs)} active out of {len(lp_defs)} total definitions") - - if not active_defs: - _last_lp_update_round = current_round - return - - try: - algo_quote = await price_router.get_algo_price_quote() - except (PriceUnavailableError, PricingError) as exc: - logger.error("All ALGO price providers failed, skipping LP update: %s", exc) - return - - semaphore = asyncio.Semaphore(LP_CONCURRENCY) - - async def _bounded(lp_def: dict) -> bool: - async with semaphore: - result = await _update_single_lp(lp_def, algo_quote, current_round) - await asyncio.sleep(LP_REQUEST_DELAY) - return result - - results = await asyncio.gather(*[_bounded(d) for d in active_defs]) - lp_updated = sum(1 for r in results if r) - _last_lp_update_round = current_round - - logger.info(f"LP token prices: {lp_updated}/{len(active_defs)} updated") diff --git a/flex/data/lp_registry.py b/flex/data/lp_registry.py new file mode 100644 index 00000000..8097a111 --- /dev/null +++ b/flex/data/lp_registry.py @@ -0,0 +1,131 @@ +"""Read-only LP token classification used to keep price routing fail closed.""" + +import logging +from typing import Any, TypedDict + +from aiocache import cached + +from core.db.contracts import get_contracts_by_type +from flex import db + +logger = logging.getLogger(__name__) + + +class LpTokenRegistryError(RuntimeError): + """The LP registry cannot safely distinguish LP tokens from regular assets.""" + + +class LpTokenDefinition(TypedDict): + lp_token_id: int + asset1_id: int + asset2_id: int + dex: str + + +def _extract_stake_token_id(contract: Any) -> int | None: + """Extract a stake token ID from contract metadata or its cached state.""" + + metadata = contract.metadata or {} + stake_token_id = metadata.get("stake_token_id") + if stake_token_id is not None: + return int(stake_token_id) + + initial = metadata.get("cache", {}).get("initial", {}) + raw = initial.get("stakeToken") or initial.get("token") + if raw is None: + return None + if isinstance(raw, dict) and raw.get("type") == "BigNumber" and "hex" in raw: + return int(raw["hex"], 16) + try: + return int(raw) + except (TypeError, ValueError): + return None + + +@cached(ttl=300, namespace="lp_token_defs") +async def get_lp_token_definitions() -> list[LpTokenDefinition]: + """Build a read-only LP registry without deriving prices from balances.""" + + contracts = get_contracts_by_type("farm") + stake_token_ids: set[int] = set() + for contract in contracts: + stake_token_id = _extract_stake_token_id(contract) + if stake_token_id: + stake_token_ids.add(stake_token_id) + + if not stake_token_ids: + logger.warning("No stake tokens found in farm contracts") + return [] + + definitions: dict[int, LpTokenDefinition] = {} + from_lp_tokens = 0 + from_farming_pools = 0 + from_metadata = 0 + + try: + for token in db.lp_tokens.get_many_by_query( + {"id": {"$in": list(stake_token_ids)}}, + ): + definitions[token.id] = LpTokenDefinition( + lp_token_id=token.id, + asset1_id=token.asset1_id, + asset2_id=token.asset2_id, + dex=token.dex_provider, + ) + from_lp_tokens += 1 + except Exception as exc: + raise LpTokenRegistryError("failed to query lp_tokens") from exc + + try: + for pool in db.farming_pools.get_all(): + stake_token_id = pool.stake_token.id + if stake_token_id not in stake_token_ids or stake_token_id in definitions: + continue + asset1_id = pool.first_token.id + asset2_id = pool.second_token.id + if asset1_id == 0: + asset1_id, asset2_id = asset2_id, asset1_id + definitions[stake_token_id] = LpTokenDefinition( + lp_token_id=stake_token_id, + asset1_id=asset1_id, + asset2_id=asset2_id, + dex=pool.dex_name, + ) + from_farming_pools += 1 + except Exception as exc: + raise LpTokenRegistryError("failed to query farming_pools") from exc + + for contract in contracts: + metadata = contract.metadata or {} + stake_token_id = _extract_stake_token_id(contract) + if not stake_token_id or stake_token_id in definitions: + continue + asset1_id = metadata.get("asset1_id", metadata.get("asset_1_id")) + asset2_id = metadata.get("asset2_id", metadata.get("asset_2_id", 0)) + dex = metadata.get("dex") or metadata.get("dex_provider") + if asset1_id is not None and dex: + definitions[stake_token_id] = LpTokenDefinition( + lp_token_id=stake_token_id, + asset1_id=int(asset1_id), + asset2_id=int(asset2_id), + dex=str(dex), + ) + from_metadata += 1 + + unresolved_stake_token_ids = stake_token_ids - definitions.keys() + if unresolved_stake_token_ids: + raise LpTokenRegistryError( + f"LP classification is incomplete for farm stake token ids {sorted(unresolved_stake_token_ids)}", + ) + + result = list(definitions.values()) + logger.info( + "LP token definitions: %s/%s (lp_tokens=%s, farming_pools=%s, metadata=%s, unresolved=%s)", + len(result), + len(stake_token_ids), + from_lp_tokens, + from_farming_pools, + from_metadata, + len(unresolved_stake_token_ids), + ) + return result diff --git a/flex/data/lp_states.py b/flex/data/lp_states.py index 88cd179a..908825a2 100644 --- a/flex/data/lp_states.py +++ b/flex/data/lp_states.py @@ -1,101 +1,33 @@ import asyncio import logging -from datetime import UTC, datetime +from dataclasses import replace from aiocache import cached from env import settings from flex import db from flex.blockchain.info import get_address_asset_snapshot -from flex.data.assets import get_asset_total_supply, get_full_asset -from flex.data.lp_tokens import get_lp_token_by_id, lp_token_from_tinyman_pool +from flex.data.assets import get_asset_total_supply +from flex.data.lp_tokens import ( + get_lp_token_by_id, + lp_token_from_tinyman_pool, + persist_lp_token, +) from flex.db.lp_projection import ( LpProjectionPersistenceError, MongoLpProjectionRepository, ) from flex.db.model.blockchain import LpToken from flex.db.model.liquidity_pools import LpState, LpTransaction +from flex.domain.algorand import require_algorand_uint64 from flex.domain.lp_projection import lp_round_end_order -from flex.domain.pricing import ( - base_units_to_decimal, - calculate_lp_token_price_from_issued_supply, - decimal_to_legacy_float, -) from flex.meta_error import MetaError from flex.providers.tinyman import fetch_algo_tinyman_pool_by_asset_id -from flex.providers.vestige import vestige_full_asset_price from flex.util import build_key_str logger = logging.getLogger(__name__) -async def get_price_algo(asset_id) -> float: - # TODO: fix DB caching - # asset_db_price = db.asset_prices.get_one(id=asset_id) - # if asset_db_price is not None: - # price_algo = asset_db_price.price_algo - # else: - # price_algo = (await vestige_full_asset_price(asset_id)).algo - asset_price = await vestige_full_asset_price(asset_id) - return asset_price.algo - - -async def recalculate_lp_state_price_algo_with_micros(lp_state: LpState) -> LpState: - # Capture this before external reads. A slower, older calculation must not - # overwrite a newer quote for the same balance cursor. - lp_state.derived_observed_at = datetime.now(UTC) - asset1, asset2, lp_token = await asyncio.gather( - get_full_asset(lp_state.asset1_id), - get_full_asset(lp_state.asset2_id), - get_full_asset(lp_state.token_id), - ) - asset1_reserve = base_units_to_decimal( - lp_state.asset1_reserve_micros, - decimals=asset1.decimals, - field="asset1_reserve_micros", - ) - asset2_reserve = base_units_to_decimal( - lp_state.asset2_reserve_micros, - decimals=asset2.decimals, - field="asset2_reserve_micros", - ) - issued_tokens = base_units_to_decimal( - lp_state.total_tokens_micros, - decimals=lp_token.decimals, - field="issued_lp_supply_micros", - ) - lp_state.asset1_reserve = decimal_to_legacy_float( - asset1_reserve, - field="asset1_reserve", - ) - lp_state.asset2_reserve = decimal_to_legacy_float( - asset2_reserve, - field="asset2_reserve", - ) - lp_state.total_tokens = decimal_to_legacy_float( - issued_tokens, - field="issued_tokens", - ) - - if lp_state.total_tokens_micros == 0: - lp_state.token_price_algo = 0 - return lp_state - - asset1_price_algo = await get_price_algo(lp_state.asset1_id) - exact_price = calculate_lp_token_price_from_issued_supply( - asset1_price_algo=asset1_price_algo, - asset1_reserve_micros=lp_state.asset1_reserve_micros, - asset1_decimals=asset1.decimals, - issued_lp_supply_micros=lp_state.total_tokens_micros, - lp_token_decimals=lp_token.decimals, - ) - lp_state.token_price_algo = decimal_to_legacy_float( - exact_price, - field="token_price_algo", - ) - return lp_state - - async def create_lp_state_by_lp_token_id(lp_token_id: int) -> LpState: lp_token = await get_lp_token_by_id(lp_token_id) if lp_token is None: @@ -109,16 +41,10 @@ async def create_lp_state_by_lp_token(lp_token: LpToken) -> LpState: if lp_token.asset1_id == 0: raise MetaError(f"ALGO LP token asset1 ID = 0, not id2: {lp_token}") - if lp_token.asset2_id == 0: - snapshot = await get_address_asset_snapshot( - lp_token.address, - include_algo=True, - ) - else: - snapshot = await get_address_asset_snapshot( - lp_token.address, - include_algo=False, - ) + snapshot = await get_address_asset_snapshot( + lp_token.address, + include_algo=True, + ) balances = snapshot.balances asset1_reserve_micros = balances[lp_token.asset1_id] @@ -127,7 +53,21 @@ async def create_lp_state_by_lp_token(lp_token: LpToken) -> LpState: # TODO: add field total_supply_micros to LP token lp_token_total_supply_micros = await get_asset_total_supply(lp_token.id) - issued_lp_tokens_micros = lp_token_total_supply_micros - lp_token_reserve_micros + try: + total_supply_micros = require_algorand_uint64( + lp_token_total_supply_micros, + "LP token total supply", + ) + reserve_micros = require_algorand_uint64( + lp_token_reserve_micros, + "pool-owned LP token reserve", + ) + issued_lp_tokens_micros = require_algorand_uint64( + total_supply_micros - reserve_micros, + "issued LP token supply", + ) + except ValueError as exc: + raise LpProjectionPersistenceError(str(exc)) from exc current_round = snapshot.observed_round lp_state = LpState( @@ -144,11 +84,11 @@ async def create_lp_state_by_lp_token(lp_token: LpToken) -> LpState: asset2_reserve=0, total_tokens=0, token_price_algo=0, + operational_algo_balance_micros=(0 if 0 in {lp_token.asset1_id, lp_token.asset2_id} else balances[0]), last_updated_round=current_round, last_event_order=lp_round_end_order(current_round), is_algo_pool=lp_token.asset2_id == 0, ) - lp_state = await recalculate_lp_state_price_algo_with_micros(lp_state) if lp_state.is_algo_pool: logger.info(f"Created new LP state for ALGO pool, asa_id={lp_token.asset1_id}:\n{lp_state.pretty_str()}") @@ -175,35 +115,62 @@ async def create_lp_state_by_lp_token(lp_token: LpToken) -> LpState: async def update_lp_state(lp_state: LpState) -> LpState: - if lp_state.asset1_id == 0 or lp_state.asset2_id == 0: - # TODO: take values from DB sync - snapshot = await get_address_asset_snapshot( - lp_state.address, - include_algo=True, - ) - else: - snapshot = await get_address_asset_snapshot( - lp_state.address, - include_algo=False, - ) + snapshot = await get_address_asset_snapshot( + lp_state.address, + include_algo=True, + ) balances = snapshot.balances - lp_state.asset1_reserve_micros = balances[lp_state.asset1_id] - lp_state.asset2_reserve_micros = balances[lp_state.asset2_id] - lp_token_reserve_micros = balances[lp_state.token_id] lp_token_total_supply_micros = await get_asset_total_supply(lp_state.token_id) - lp_state.total_tokens_micros = lp_token_total_supply_micros - lp_token_reserve_micros - - lp_state = await recalculate_lp_state_price_algo_with_micros(lp_state) current_round = snapshot.observed_round + try: + total_supply_micros = require_algorand_uint64( + lp_token_total_supply_micros, + "LP token total supply", + ) + reserve_micros = require_algorand_uint64( + lp_token_reserve_micros, + "pool-owned LP token reserve", + ) + candidate = replace( + lp_state, + asset1_reserve_micros=require_algorand_uint64( + balances[lp_state.asset1_id], + "asset1 reserve", + ), + asset2_reserve_micros=require_algorand_uint64( + balances[lp_state.asset2_id], + "asset2 reserve", + ), + total_tokens_micros=require_algorand_uint64( + total_supply_micros - reserve_micros, + "issued LP token supply", + ), + operational_algo_balance_micros=( + 0 + if 0 in {lp_state.asset1_id, lp_state.asset2_id} + else require_algorand_uint64( + balances[0], + "operational ALGO balance", + ) + ), + last_updated_round=require_algorand_uint64( + current_round, + "LP snapshot round", + ), + ) + except (KeyError, ValueError) as exc: + raise LpProjectionPersistenceError( + f"invalid authoritative LP snapshot for token {lp_state.token_id}: {exc}", + ) from exc repository = MongoLpProjectionRepository( states=db.lp_states.mongodb_collection, events=db.lp_transactions.mongodb_collection, ) return await asyncio.to_thread( repository.replace_snapshot, - lp_state, + candidate, observed_round=current_round, ) @@ -298,29 +265,13 @@ async def update_lp_states_with_transactions( repository.project, transaction, ) - if outcome.requires_derived_refresh: - changed_addresses.add(transaction.pool_address) if outcome.changed_balances: + changed_addresses.add(transaction.pool_address) applied_event_count += 1 - updated_states: list[LpState] = [] - for pool_address in sorted(changed_addresses): - for _ in range(2): - state = await asyncio.to_thread(repository.get_state, pool_address) - expected_cursor = state.last_event_order - if expected_cursor is None: - raise LpProjectionPersistenceError(f"LP state {state.token_id} has no event cursor") - state = await recalculate_lp_state_price_algo_with_micros(state) - updated = await asyncio.to_thread( - repository.update_derived_fields, - state, - expected_cursor=expected_cursor, - ) - if updated is not None: - updated_states.append(updated) - break - else: - raise LpProjectionPersistenceError(f"LP state {pool_address} kept changing during price persistence") + updated_states = [ + await asyncio.to_thread(repository.get_state, pool_address) for pool_address in sorted(changed_addresses) + ] logger.info( "Updated %s LP states with %s new transaction(s)", @@ -382,6 +333,6 @@ async def get_tinyman_pool_lp_state_by_asset_id(asset_id: int) -> LpState | None lp_token = db.lp_tokens.get_by_primary_key(tinyman_pool.lp_token_id, throw_ex=False) if lp_token is None: lp_token = await lp_token_from_tinyman_pool(tinyman_pool) - db.lp_tokens.create(lp_token) + lp_token = persist_lp_token(lp_token) return await create_lp_state_by_lp_token(lp_token) diff --git a/flex/data/lp_tokens.py b/flex/data/lp_tokens.py index 3953ebe2..a042fa77 100644 --- a/flex/data/lp_tokens.py +++ b/flex/data/lp_tokens.py @@ -4,16 +4,41 @@ from flex import db from flex.blockchain.info import get_address_app_ids +from flex.db.model.blockchain import LpToken from flex.providers import vestige from flex.providers.pact import get_pact_pool_info -from flex.providers.tinyman import get_tinyman_pool_info, TinymanPoolInfo +from flex.providers.tinyman import TinymanPoolInfo, get_tinyman_pool_info from flex.providers.vestige import DexProvider, fetch_lp_token -from flex.db.model.blockchain import LpToken from flex.util import build_key_str logger = logging.getLogger(__name__) +class LpTokenIdentityConflictError(RuntimeError): + """Raised when an ASA ID is already bound to different pool metadata.""" + + +def persist_lp_token(candidate: LpToken) -> LpToken: + """Atomically register an LP token and verify its immutable identity.""" + + persisted = db.lp_tokens.get_or_create(candidate) + identity_fields = ( + "id", + "pool_id", + "asset1_id", + "asset2_id", + "address", + "dex_provider", + ) + if tuple(getattr(persisted, field) for field in identity_fields) != tuple( + getattr(candidate, field) for field in identity_fields + ): + raise LpTokenIdentityConflictError( + f"LP token {candidate.id} is already registered with different pool metadata" + ) + return persisted + + async def lp_token_from_tinyman_pool(tinyman_pool: TinymanPoolInfo) -> LpToken: app_ids = await get_address_app_ids(tinyman_pool.address) return LpToken( @@ -22,7 +47,7 @@ async def lp_token_from_tinyman_pool(tinyman_pool: TinymanPoolInfo) -> LpToken: asset1_id=tinyman_pool.asset1_id, asset2_id=tinyman_pool.asset2_id, address=tinyman_pool.address, - dex_provider=DexProvider.TINYMAN_V2 + dex_provider=DexProvider.TINYMAN_V2, ) @@ -37,7 +62,7 @@ async def fetch_lp_token_strong(lp_token_id: int, asset1_id: int, asset2_id: int asset1_id=asset1_id, asset2_id=asset2_id, address=pact_pool.address, - dex_provider=dex_provider + dex_provider=dex_provider, ) else: return await vestige.fetch_lp_token(lp_token_id, asset1_id, asset2_id, dex_provider) @@ -47,37 +72,40 @@ async def fetch_lp_token_strong(lp_token_id: int, asset1_id: int, asset2_id: int tinyman_pool = await get_tinyman_pool_info(asset1_id, asset2_id) return await lp_token_from_tinyman_pool(tinyman_pool) except Exception as e: - logger.error(f'Tinyman pool for assets {asset1_id} and {asset2_id} not found: {e}') + logger.error(f"Tinyman pool for assets {asset1_id} and {asset2_id} not found: {e}") return await fetch_lp_token(lp_token_id, asset1_id, asset2_id, dex_provider) async def fetch_lp_token_by_id(lp_token_id: int) -> LpToken | None: - farming_pool = db.farming_pools.get_one(**{'stake_token.id': lp_token_id}) + farming_pool = db.farming_pools.get_one(**{"stake_token.id": lp_token_id}) if farming_pool is None: return None if farming_pool.first_token.id < farming_pool.second_token.id: - farming_pool.first_token.id, farming_pool.second_token.id = farming_pool.second_token.id, farming_pool.first_token.id + farming_pool.first_token.id, farming_pool.second_token.id = ( + farming_pool.second_token.id, + farming_pool.first_token.id, + ) return await fetch_lp_token_strong( lp_token_id=lp_token_id, asset1_id=farming_pool.first_token.id, asset2_id=farming_pool.second_token.id, - dex_provider=farming_pool.dex_name + dex_provider=farming_pool.dex_name, ) -@cached(namespace='lp_token_get_all', key_builder=build_key_str) +@cached(namespace="lp_token_get_all", key_builder=build_key_str) async def get_all_lp_tokens() -> list[LpToken]: return db.lp_tokens.get_all() -@cached(namespace='lp_token_by_id', key_builder=build_key_str) +@cached(namespace="lp_token_by_id", key_builder=build_key_str) async def get_lp_token_by_id(lp_token_id: int) -> LpToken | None: lp_token = db.lp_tokens.get_by_primary_key(lp_token_id, throw_ex=False) if lp_token is None: lp_token = await fetch_lp_token_by_id(lp_token_id) if lp_token is not None: - db.lp_tokens.create(lp_token) + lp_token = persist_lp_token(lp_token) return lp_token diff --git a/flex/data/stats.py b/flex/data/stats.py index e3df8524..eca934bc 100644 --- a/flex/data/stats.py +++ b/flex/data/stats.py @@ -1,7 +1,9 @@ import logging +from datetime import timedelta +from env import settings from flex import db -from flex.data.asset_prices import get_asset_price +from flex.data.asset_prices import get_asset_price, validated_stored_asset_price from flex.db.model.pools import PoolType logger = logging.getLogger(__name__) @@ -10,7 +12,12 @@ async def calculate_total_tvl_usd_for_type(type: PoolType) -> float: pools = db.pool_states.get_many(type=type) all_asset_prices = db.asset_prices.get_all() - price_usd_by_id = {price.id: price.price_usd for price in all_asset_prices} + max_age = timedelta(seconds=settings.asset_prices_max_stale) + price_usd_by_id = { + price.id: price.price_usd + for price in all_asset_prices + if validated_stored_asset_price(price, max_age=max_age) is not None + } total_usd = 0 for pool in pools: pool_token_price_usd = price_usd_by_id.get(pool.stake_token.id) diff --git a/flex/data/tinyman_lps.py b/flex/data/tinyman_lps.py deleted file mode 100644 index eb3d0afd..00000000 --- a/flex/data/tinyman_lps.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Validated Tinyman ALGO-pool price projection.""" - -import logging -from datetime import UTC, datetime, timedelta -from decimal import Decimal, localcontext - -from env import settings -from flex.blockchain.info import ALGO_ASSET -from flex.data.asset_prices import _upsert_asset_price -from flex.data.assets import get_asset_details -from flex.db.model.liquidity_pools import LpState -from flex.db.model.priced import AssetPrice -from flex.domain.pricing import ( - PERSISTED_PRICE_PRECISION, - InvalidLiquidityPoolError, - InvalidPriceError, - PriceQuote, - PriceSource, -) - -logger = logging.getLogger(__name__) - - -def _pool_observed_at(lp_state: LpState) -> datetime: - observed_at = getattr(lp_state, "updated", None) - if not isinstance(observed_at, datetime): - raise InvalidLiquidityPoolError("Tinyman pool state has no observation timestamp") - if observed_at.tzinfo is None: - return observed_at.replace(tzinfo=UTC) - return observed_at.astimezone(UTC) - - -async def _tinyman_asset_quote( - lp_state: LpState, - *, - algo_quote: PriceQuote, -) -> tuple[PriceQuote, str]: - if not lp_state.is_algo_pool or lp_state.asset2_id != ALGO_ASSET.id: - raise InvalidLiquidityPoolError("Tinyman price source must be an asset/ALGO pool") - if lp_state.asset1_reserve_micros <= 0 or lp_state.asset2_reserve_micros <= 0: - raise InvalidLiquidityPoolError("Tinyman reserves must be positive") - if algo_quote.asset_id != ALGO_ASSET.id: - raise InvalidPriceError("Tinyman projection requires an ALGO/USD quote") - - asset_details = await get_asset_details(lp_state.asset1_id) - with localcontext() as context: - context.prec = PERSISTED_PRICE_PRECISION - asset_reserve = Decimal(lp_state.asset1_reserve_micros) / Decimal( - 10**asset_details.decimals, - ) - algo_reserve = Decimal(lp_state.asset2_reserve_micros) / Decimal( - 10**ALGO_ASSET.decimals, - ) - asset_price_algo = +(algo_reserve / asset_reserve) - algo_usd = algo_quote.usd - - stale_after = timedelta(seconds=settings.asset_prices_ttl) - observed_at = min(_pool_observed_at(lp_state), algo_quote.observed_at) - asset_quote = PriceQuote.from_raw( - asset_id=lp_state.asset1_id, - algo=asset_price_algo, - usd=asset_price_algo * algo_usd, - source=PriceSource.TINYMAN, - stale_after=stale_after, - observed_round=lp_state.last_updated_round, - observed_at=observed_at, - ) - return asset_quote, asset_details.name - - -async def update_tinyman_algo_asset_price( - lp_state: LpState, - algo_quote: PriceQuote, -) -> AssetPrice: - """Project one validated Tinyman observation into the asset read model.""" - - asset_quote, asset_name = await _tinyman_asset_quote( - lp_state, - algo_quote=algo_quote, - ) - - price_algo, price_usd = asset_quote.to_legacy_floats() - asset_price = AssetPrice( - id=asset_quote.asset_id, - price_algo=price_algo, - price_usd=price_usd, - last_update_round=asset_quote.observed_round or 0, - tinyman_algo_pool_id=lp_state.id, - name=asset_name, - source=asset_quote.source.value, - observed_at=asset_quote.observed_at, - ) - _upsert_asset_price(asset_price) - logger.debug( - "Updated asset %s from Tinyman ALGO pool %s: algo=%s, usd=%s", - asset_price.id, - lp_state.id, - asset_price.price_algo, - asset_price.price_usd, - ) - return asset_price diff --git a/flex/db/lp_projection.py b/flex/db/lp_projection.py index 5aa597e7..dca47378 100644 --- a/flex/db/lp_projection.py +++ b/flex/db/lp_projection.py @@ -11,8 +11,11 @@ from flex.db.bson import encode_bson_integer from flex.db.model.liquidity_pools import LpState, LpTransaction -from flex.domain.lp_projection import ( +from flex.domain.algorand import ( MAX_ALGORAND_UINT, + require_algorand_uint64, +) +from flex.domain.lp_projection import ( LpBalanceDelta, lp_balance_delta, lp_round_end_order, @@ -40,10 +43,6 @@ class LpProjectionOutcome: def changed_balances(self) -> bool: return self.result is LpProjectionResult.APPLIED - @property - def requires_derived_refresh(self) -> bool: - return self.result is not LpProjectionResult.SNAPSHOT_COVERED - @dataclass(slots=True) class MongoLpProjectionRepository: @@ -148,42 +147,6 @@ def project(self, transaction: LpTransaction) -> LpProjectionOutcome: result=LpProjectionResult.APPLIED, ) - def update_derived_fields( - self, - state: LpState, - *, - expected_cursor: str, - ) -> LpState | None: - if state.derived_observed_at is None: - raise LpProjectionPersistenceError("derived LP fields have no observation timestamp") - document = self.states.find_one_and_update( - { - "token_id": encode_bson_integer(state.token_id), - "last_event_order": expected_cursor, - "$or": [ - {"derived_observed_at": {"$exists": False}}, - {"derived_observed_at": None}, - { - "derived_observed_at": { - "$lte": state.derived_observed_at, - } - }, - ], - }, - { - "$set": { - "asset1_reserve": state.asset1_reserve, - "asset2_reserve": state.asset2_reserve, - "total_tokens": state.total_tokens, - "token_price_algo": state.token_price_algo, - "derived_observed_at": state.derived_observed_at, - "updated": datetime.now(UTC), - } - }, - return_document=ReturnDocument.AFTER, - ) - return self._from_state_document(document) - def replace_snapshot( self, state: LpState, @@ -192,6 +155,27 @@ def replace_snapshot( ) -> LpState: """Replace balances only when the authoritative snapshot is newer.""" + try: + for field_name in ( + "id", + "token_id", + "asset1_id", + "asset2_id", + "asset1_reserve_micros", + "asset2_reserve_micros", + "total_tokens_micros", + "operational_algo_balance_micros", + ): + require_algorand_uint64( + getattr(state, field_name), + f"LP snapshot {field_name}", + ) + require_algorand_uint64( + observed_round, + "LP snapshot observed_round", + ) + except ValueError as exc: + raise LpProjectionPersistenceError(str(exc)) from exc snapshot_order = lp_round_end_order(observed_round) document = self.states.find_one_and_update( { @@ -213,11 +197,9 @@ def replace_snapshot( "total_tokens_micros": encode_bson_integer( state.total_tokens_micros, ), - "asset1_reserve": state.asset1_reserve, - "asset2_reserve": state.asset2_reserve, - "total_tokens": state.total_tokens, - "token_price_algo": state.token_price_algo, - "derived_observed_at": state.derived_observed_at, + "operational_algo_balance_micros": encode_bson_integer( + state.operational_algo_balance_micros, + ), "last_updated_round": encode_bson_integer( observed_round, ), @@ -314,6 +296,7 @@ def _assert_event_document( persisted_event = LpTransaction.from_dict(persisted) if ( persisted_event.pool_address, + persisted_event.user_address, persisted_event.asa_id, persisted_event.delta_amount_micros, persisted_event.confirmed_round, @@ -321,6 +304,7 @@ def _assert_event_document( persisted_event.event_order, ) != ( expected.pool_address, + expected.user_address, expected.asa_id, expected.delta_amount_micros, expected.confirmed_round, @@ -339,11 +323,12 @@ def _legacy_alias(self, transaction: LpTransaction) -> str | None: return None @staticmethod - def _balances(state: LpState) -> tuple[int, int, int]: + def _balances(state: LpState) -> tuple[int, int, int, int]: return ( state.asset1_reserve_micros, state.asset2_reserve_micros, state.total_tokens_micros, + state.operational_algo_balance_micros, ) @staticmethod diff --git a/flex/db/model/blockchain.py b/flex/db/model/blockchain.py index ceea33bb..671309d6 100644 --- a/flex/db/model/blockchain.py +++ b/flex/db/model/blockchain.py @@ -17,6 +17,7 @@ from flex.db.classes.bson_uint64 import BsonUint64StorageMixin UINT64_MAX = (1 << 64) - 1 +TOTAL_SUPPLY_SOURCE_INDEXER = "indexer" class _MissingTotalSupply(int): @@ -26,6 +27,10 @@ class _MissingTotalSupply(int): _MISSING_TOTAL_SUPPLY = _MissingTotalSupply(-1) +class _UnverifiedTotalSupply(int): + """Compatibility value that must never be serialized as canonical.""" + + def _decode_total_supply(value: object) -> int: if isinstance(value, _MissingTotalSupply): return value @@ -204,7 +209,20 @@ class Asset( # Keep an int in memory and serialize it as a decimal string in MongoDB. total_supply_micros: int = field( default=_MISSING_TOTAL_SUPPLY, - metadata=config(encoder=str, decoder=_decode_total_supply), + metadata=config( + encoder=str, + decoder=_decode_total_supply, + exclude=lambda value: isinstance(value, _UnverifiedTotalSupply), + ), + ) + total_supply_source: str | None = None + # Kept out of storage: old documents may be decoded for display, but code + # that makes financial decisions must first migrate from the Indexer. + total_supply_is_authoritative: bool = field( + init=False, + default=False, + repr=False, + metadata=config(exclude=lambda _: True), ) created: datetime = field(default_factory=datetime.now) @@ -212,15 +230,18 @@ class Asset( def __post_init__(self) -> None: if isinstance(self.total_supply_micros, _MissingTotalSupply): - # Documents written before the canonical field was introduced only - # contain display units. This is necessarily best-effort because a - # historical float may already have lost precision. + # Retain a display-compatible value for old read models, but mark + # it as non-authoritative so financial paths cannot trust it. + self.total_supply_source = None self.total_supply_micros = self.amount_to_micros(self.total_supply) self.total_supply_micros = _validate_uint64( self.total_supply_micros, field_name="total_supply_micros", ) + self.total_supply_is_authoritative = self.total_supply_source == TOTAL_SUPPLY_SOURCE_INDEXER + if not self.total_supply_is_authoritative: + self.total_supply_micros = _UnverifiedTotalSupply(self.total_supply_micros) # The persisted base-unit amount is authoritative. Keep the old float # field as a presentation-only compatibility value. self.total_supply = self.micros_to_amount(self.total_supply_micros) diff --git a/flex/db/model/liquidity_pools.py b/flex/db/model/liquidity_pools.py index d72b418c..29ebde95 100644 --- a/flex/db/model/liquidity_pools.py +++ b/flex/db/model/liquidity_pools.py @@ -11,7 +11,8 @@ ) from flex.db.classes.base_entity import BaseEntity from flex.db.classes.bson_uint64 import BsonUint64StorageMixin -from flex.domain.lp_projection import lp_event_order +from flex.domain.algorand import MAX_ALGORAND_UINT, require_algorand_uint64 +from flex.domain.lp_projection import InvalidLpProjectionError, lp_event_order @dataclass_json @@ -56,6 +57,7 @@ class LpState( "asset1_reserve_micros", "asset2_reserve_micros", "total_tokens_micros", + "operational_algo_balance_micros", } ) @@ -117,6 +119,15 @@ class LpState( total_tokens: float token_price_algo: float + # Token-token pools still hold ALGO to fund network fees. Keep this + # operational balance separate from economic reserves and public pricing. + operational_algo_balance_micros: int = field( + default=0, + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ), + ) is_algo_pool: bool = False last_event_order: str | None = None derived_observed_at: datetime | None = None @@ -125,6 +136,16 @@ class LpState( updated: datetime = field(default_factory=datetime.now) created: datetime = field(default_factory=datetime.now) + def __post_init__(self) -> None: + try: + for field_name in self.BSON_UINT64_FIELDS: + require_algorand_uint64( + getattr(self, field_name), + field_name, + ) + except ValueError as exc: + raise InvalidLpProjectionError(str(exc)) from exc + @classmethod def primary_key_name(cls) -> str: return "token_id" @@ -189,9 +210,26 @@ class LpTransaction(BaseEntity["LpTransaction"]): updated: datetime = field(default_factory=datetime.now) def __post_init__(self) -> None: - if self.event_order is None: - self.event_order = lp_event_order( - self.confirmed_round, - self.id, - self.event_position, + try: + require_algorand_uint64(self.asa_id, "asa_id") + require_algorand_uint64(self.confirmed_round, "confirmed_round") + require_algorand_uint64(self.event_position, "event_position") + except ValueError as exc: + raise InvalidLpProjectionError(str(exc)) from exc + if ( + isinstance(self.delta_amount_micros, bool) + or not isinstance(self.delta_amount_micros, int) + or not -MAX_ALGORAND_UINT <= self.delta_amount_micros <= MAX_ALGORAND_UINT + ): + raise InvalidLpProjectionError( + "delta_amount_micros must fit the signed Algorand uint64 domain", ) + expected_order = lp_event_order( + self.confirmed_round, + self.id, + self.event_position, + ) + if self.event_order is None: + self.event_order = expected_order + elif self.event_order != expected_order: + raise InvalidLpProjectionError("event_order does not match the immutable event fields") diff --git a/flex/domain/algorand.py b/flex/domain/algorand.py new file mode 100644 index 00000000..70a0f6f7 --- /dev/null +++ b/flex/domain/algorand.py @@ -0,0 +1,17 @@ +"""Algorand protocol value boundaries shared by external adapters.""" + +MAX_ALGORAND_UINT = 2**64 - 1 + + +def require_algorand_uint64( + value: object, + field: str, + *, + positive: bool = False, +) -> int: + """Return an exact protocol integer or reject coercible lookalikes.""" + + if isinstance(value, bool) or not isinstance(value, int) or value < int(positive) or value > MAX_ALGORAND_UINT: + requirement = "a positive Algorand uint64" if positive else "an Algorand uint64" + raise ValueError(f"{field} must be {requirement}") + return value diff --git a/flex/domain/lp_projection.py b/flex/domain/lp_projection.py index 7f1be615..db1034c9 100644 --- a/flex/domain/lp_projection.py +++ b/flex/domain/lp_projection.py @@ -3,7 +3,11 @@ from dataclasses import dataclass from typing import Literal -MAX_ALGORAND_UINT = 2**64 - 1 +from flex.domain.algorand import ( + MAX_ALGORAND_UINT, + require_algorand_uint64, +) + EVENT_ROUND_WIDTH = 20 ROUND_END_SUFFIX = "~" @@ -11,6 +15,7 @@ "asset1_reserve_micros", "asset2_reserve_micros", "total_tokens_micros", + "operational_algo_balance_micros", ] @@ -31,20 +36,16 @@ def lp_event_order( ) -> str: """Build a lexicographically sortable, deterministic event cursor.""" - if ( - isinstance(confirmed_round, bool) - or not isinstance(confirmed_round, int) - or not 0 <= confirmed_round <= MAX_ALGORAND_UINT - ): - raise InvalidLpProjectionError("confirmed_round must be an Algorand uint64") + try: + require_algorand_uint64(confirmed_round, "confirmed_round") + except ValueError as exc: + raise InvalidLpProjectionError(str(exc)) from exc if not isinstance(event_id, str) or not event_id or ":" in event_id: raise InvalidLpProjectionError("event_id must be non-empty and cannot contain ':'") - if ( - isinstance(event_position, bool) - or not isinstance(event_position, int) - or not 0 <= event_position <= MAX_ALGORAND_UINT - ): - raise InvalidLpProjectionError("event_position must be an Algorand uint64") + try: + require_algorand_uint64(event_position, "event_position") + except ValueError as exc: + raise InvalidLpProjectionError(str(exc)) from exc return f"{confirmed_round:0{EVENT_ROUND_WIDTH}d}:{event_position:0{EVENT_ROUND_WIDTH}d}:{event_id}" @@ -114,6 +115,11 @@ def lp_balance_delta( field="asset2_reserve_micros", amount=event_pool_delta_micros, ) + if event_asset_id == 0: + return LpBalanceDelta( + field="operational_algo_balance_micros", + amount=event_pool_delta_micros, + ) raise InvalidLpProjectionError( f"asset {event_asset_id} does not belong to LP token {token_id}", ) diff --git a/flex/domain/pricing.py b/flex/domain/pricing.py index 02c87e56..b5cb9975 100644 --- a/flex/domain/pricing.py +++ b/flex/domain/pricing.py @@ -13,6 +13,7 @@ MAX_ASSET_DECIMALS = 19 CALCULATION_PRECISION = 80 PERSISTED_PRICE_PRECISION = 34 +MAX_OBSERVATION_CLOCK_SKEW = timedelta(minutes=1) class PricingError(ValueError): @@ -66,6 +67,20 @@ def _utc(value: datetime) -> datetime: return value.astimezone(UTC) +def validate_observation_timestamp( + observed_at: datetime, + *, + now: datetime | None = None, +) -> datetime: + """Reject timestamps that could pin a quote ahead of wall-clock time.""" + + observed = _utc(observed_at) + current_time = _utc(now or datetime.now(UTC)) + if observed > current_time + MAX_OBSERVATION_CLOCK_SKEW: + raise InvalidPriceError("observed_at is too far in the future") + return observed + + def _legacy_float(value: Decimal, *, field: str) -> float: converted = float(value) if not isfinite(converted) or (value != 0 and converted == 0): @@ -156,6 +171,8 @@ def from_raw( def is_stale(self, *, now: datetime | None = None) -> bool: current_time = _utc(now or datetime.now(UTC)) + if self.observed_at > current_time + MAX_OBSERVATION_CLOCK_SKEW: + return True return current_time - self.observed_at >= self.stale_after def to_legacy_floats(self) -> tuple[float, float]: @@ -174,7 +191,10 @@ def is_observation_stale( if fresh_for <= timedelta(0): raise ValueError("fresh_for must be positive") current_time = _utc(now or datetime.now(UTC)) - return current_time - _utc(observed_at) >= fresh_for + observed = _utc(observed_at) + if observed > current_time + MAX_OBSERVATION_CLOCK_SKEW: + return True + return current_time - observed >= fresh_for def _non_negative_int(value: int, *, field: str) -> int: diff --git a/flex/providers/price_router.py b/flex/providers/price_router.py index 9a453bcf..ae12cb23 100644 --- a/flex/providers/price_router.py +++ b/flex/providers/price_router.py @@ -8,6 +8,7 @@ from env import settings from flex import db +from flex.data.asset_prices import is_unverified_legacy_lp_price from flex.domain.pricing import ( DecimalInput, PriceQuote, @@ -163,7 +164,12 @@ def _stored_quote( now: datetime, max_age: timedelta, ) -> PriceQuote | None: - if record is None: + if record is None or is_unverified_legacy_lp_price(record): + if record is not None: + logger.warning( + "Ignoring retired raw-reserve LP projection for asset %s", + asset_id, + ) return None try: observed_at = _record_observed_at(record) diff --git a/flex/sync_pools.py b/flex/sync_pools.py index 6ee41c95..1d132073 100644 --- a/flex/sync_pools.py +++ b/flex/sync_pools.py @@ -9,7 +9,6 @@ from flex import db from flex.blockchain.base import indexer_client from flex.blockchain.info import get_current_round -from flex.data.asset_prices import create_and_update_asset_prices from flex.data.assets import load_all_assets_data from flex.data.lp_states import ( create_lp_states_from_all_pools, @@ -21,17 +20,17 @@ update_pool_state, update_pool_states_with_transactions, ) -from flex.data.tinyman_lps import update_tinyman_algo_asset_price from flex.db.model.blockchain import PoolTransaction, SyncBlock, SyncState from flex.db.model.liquidity_pools import LpState, LpTransaction from flex.db.model.pool_states import PoolState, UserState -from flex.db.model.priced import AssetPrice from flex.db.sync_coordinator import ( MongoSyncCoordinator, SyncCoordinatorError, ) -from flex.domain.lp_projection import lp_cursor_complete_through -from flex.domain.pricing import PricingError +from flex.domain.algorand import require_algorand_uint64 +from flex.domain.lp_projection import ( + lp_cursor_complete_through, +) from flex.domain.transactions import ( ASSET_TRANSFER_TX, PAYMENT_TX, @@ -39,15 +38,24 @@ flatten_transfer_payments, projection_event_id, ) -from flex.providers.price_router import get_algo_price_quote from flex.sync_state import get_sync_state, is_sync_delayed from flex.util import build_key_str logger = logging.getLogger(__name__) -def get_all_lp_state_addresses() -> set[str]: - return set(lp_state.address for lp_state in db.lp_states.get_all()) +def get_lp_tracked_assets_by_address() -> dict[str, frozenset[int]]: + return { + lp_state.address: frozenset( + { + 0, + lp_state.asset1_id, + lp_state.asset2_id, + lp_state.token_id, + } + ) + for lp_state in db.lp_states.get_all() + } def find_transfer_payment_transactions(txns: list[dict]) -> list[dict]: @@ -63,18 +71,41 @@ async def get_pool_state_by_address(address: str) -> PoolState: return db.pool_states.get_one(address=address) +def _transaction_fee(tx: dict, *, txid: str) -> int: + try: + return require_algorand_uint64( + tx.get("fee"), + "LP transaction fee", + ) + except ValueError as exc: + raise ValueError(f"{exc}: {txid}") from exc + + async def process_lp_transactions(transactions: list[dict]) -> list[LpTransaction]: - all_lp_addresses = get_all_lp_state_addresses() + tracked_assets_by_address = get_lp_tracked_assets_by_address() + all_lp_addresses = tracked_assets_by_address.keys() lp_transactions = [] for event_position, tx in enumerate(transactions): txid = tx["id"] sender = tx["sender"] - confirmed_round = tx["confirmed-round"] + confirmed_round = require_algorand_uint64( + tx.get("confirmed-round"), + "LP transaction confirmed-round", + ) if ASSET_TRANSFER_TX in tx: transfer = tx[ASSET_TRANSFER_TX] receiver = transfer["receiver"] + asa_id = require_algorand_uint64( + transfer.get("asset-id"), + "LP transaction asset-id", + positive=True, + ) + amount = require_algorand_uint64( + transfer.get("amount"), + "LP transaction amount", + ) affected_addresses = { sender, receiver, @@ -84,9 +115,13 @@ async def process_lp_transactions(transactions: list[dict]) -> list[LpTransactio if not (affected_addresses & all_lp_addresses): continue if any(field in transfer for field in ("sender", "close-to", "close-amount")): - raise ValueError(f"LP projection does not support clawback/close semantics: {txid}") - asa_id = transfer["asset-id"] - amount = transfer["amount"] + tracked_affected_pool = any( + address in tracked_assets_by_address and asa_id in tracked_assets_by_address[address] + for address in affected_addresses + if address is not None + ) + if tracked_affected_pool: + raise ValueError(f"LP projection does not support clawback/close semantics: {txid}") elif PAYMENT_TX in tx: payment = tx[PAYMENT_TX] receiver = payment["receiver"] @@ -100,17 +135,27 @@ async def process_lp_transactions(transactions: list[dict]) -> list[LpTransactio if any(field in payment for field in ("close-remainder-to", "close-amount")): raise ValueError(f"LP projection does not support payment close semantics: {txid}") asa_id = 0 - amount = payment["amount"] + amount = require_algorand_uint64( + payment.get("amount"), + "LP transaction amount", + ) else: raise ValueError(f"Invalid transaction type: {tx}") - if sender == receiver and sender in all_lp_addresses: + fee = _transaction_fee(tx, txid=txid) + sender_assets = tracked_assets_by_address.get(sender) + receiver_assets = tracked_assets_by_address.get(receiver) + sender_tracks_asset = sender_assets is not None and asa_id in sender_assets + receiver_tracks_asset = receiver_assets is not None and asa_id in receiver_assets + is_pool_self_transfer = sender == receiver and sender_tracks_asset + if is_pool_self_transfer: # A self-transfer has zero net effect on the pool. Emitting both # legs would create the same scoped event ID twice and make a - # deduplicating projector persist only one side. - continue + # deduplicating projector persist only one side. Its network fee + # is still a real ALGO debit and is projected separately below. + pass - if sender in all_lp_addresses: + elif sender_tracks_asset: lp_tx = LpTransaction( id=projection_event_id(txid, sender), pool_address=sender, @@ -122,7 +167,7 @@ async def process_lp_transactions(transactions: list[dict]) -> list[LpTransactio ) lp_transactions.append(lp_tx) - if receiver in all_lp_addresses: + if not is_pool_self_transfer and receiver_tracks_asset: lp_tx = LpTransaction( id=projection_event_id(txid, receiver), pool_address=receiver, @@ -134,6 +179,19 @@ async def process_lp_transactions(transactions: list[dict]) -> list[LpTransactio ) lp_transactions.append(lp_tx) + if sender_assets is not None and fee: + lp_transactions.append( + LpTransaction( + id=projection_event_id(f"{txid}#fee", sender), + pool_address=sender, + user_address=sender, + asa_id=0, + delta_amount_micros=-fee, + confirmed_round=confirmed_round, + event_position=event_position, + ) + ) + return lp_transactions @@ -196,32 +254,6 @@ async def update_lp_states( ) -async def update_asset_prices(updated_lp_states: list[LpState]) -> list[AssetPrice]: - if len(updated_lp_states) == 0: - return [] - - algo_quote = await get_algo_price_quote() - updated_asset_prices = [] - for lp_state in updated_lp_states: - if lp_state.is_algo_pool: - try: - updated_asset_price = await update_tinyman_algo_asset_price( - lp_state, - algo_quote, - ) - updated_asset_prices.append(updated_asset_price) - except PricingError as exc: - logger.warning( - "Skipping invalid Tinyman LP state %s: %s", - lp_state.id, - exc, - ) - - if len(updated_asset_prices) > 0: - logger.debug(f"Updated {len(updated_asset_prices)} asset prices.") - return updated_asset_prices - - def _snapshot_checkpoint_round( *, previous_round: int | None, @@ -262,7 +294,6 @@ async def catch_up_the_sync_manually(sync_state: SyncState, current_round: int) logger.info("\n\nSyncing LP states from authoritative account snapshots.\n") _ = await create_lp_states_from_all_pools() snapshotted_states = await update_all_lp_states_linear() - _ = await create_and_update_asset_prices() else: snapshotted_states = [] @@ -365,7 +396,6 @@ async def sync_pools_loop(): if settings.sync_liquidity_pools else [] ) - _ = await update_asset_prices(updated_lp_states) sync_state = await asyncio.to_thread( coordinator.complete_round, diff --git a/tests/integration/test_legacy_price_cutover_mongo.py b/tests/integration/test_legacy_price_cutover_mongo.py new file mode 100644 index 00000000..721d8265 --- /dev/null +++ b/tests/integration/test_legacy_price_cutover_mongo.py @@ -0,0 +1,72 @@ +import os +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from bson import Decimal128 +from pymongo import MongoClient +from pymongo.database import Database + +from flex.db.indexes import delete_unverified_legacy_lp_prices + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def mongo_database() -> Database: + uri = os.getenv("MONGODB_TEST_URI") + if not uri: + pytest.skip("MONGODB_TEST_URI is not configured") + + client = MongoClient( + uri, + serverSelectionTimeoutMS=2_000, + tz_aware=True, + ) + client.admin.command("ping") + database = client[f"cometa_legacy_price_cutover_{uuid4().hex}"] + try: + yield database + finally: + client.drop_database(database.name) + client.close() + + +def test_cutover_deletes_both_legacy_signatures_and_preserves_safe_prices( + mongo_database: Database, +) -> None: + collection = mongo_database["asset_prices"] + collection.insert_many( + [ + {"id": Decimal128("1"), "source": "vestige"}, + { + "id": Decimal128("2"), + "source": "tinyman", + "tinyman_algo_pool_id": None, + }, + {"id": Decimal128("3")}, + { + "id": Decimal128("4"), + "source": "vestige", + "tinyman_algo_pool_id": Decimal128("99"), + }, + {"id": Decimal128("5"), "source": "derived_lp"}, + { + "id": Decimal128("6"), + "source": "derived_lp", + "tinyman_algo_pool_id": None, + }, + ] + ) + + removed = delete_unverified_legacy_lp_prices( + SimpleNamespace( + asset_prices=SimpleNamespace( + mongodb_collection=collection, + ) + ) + ) + + remaining_ids = {document["id"].to_decimal() for document in collection.find({}, projection={"id": 1})} + assert removed == 3 + assert remaining_ids == {1, 2, 3} diff --git a/tests/integration/test_mongo_financial_projection.py b/tests/integration/test_mongo_financial_projection.py index 2a6e15be..8b714ce6 100644 --- a/tests/integration/test_mongo_financial_projection.py +++ b/tests/integration/test_mongo_financial_projection.py @@ -1,7 +1,9 @@ +import asyncio import os from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta from threading import Barrier, Event +from types import SimpleNamespace from uuid import uuid4 import pytest @@ -9,18 +11,33 @@ from pymongo import MongoClient from pymongo.database import Database -from flex.db.indexes import create_unique_field_index_fail_closed +from flex import sync_pools +from flex.data import asset_prices as asset_price_data +from flex.data import assets as asset_data +from flex.db.indexes import ( + create_unique_field_index_fail_closed, + create_unique_id_index_fail_closed, + delete_unverified_legacy_lp_prices, +) from flex.db.lp_projection import ( LpProjectionResult, MongoLpProjectionRepository, ) -from flex.db.model.blockchain import SyncState +from flex.db.model.blockchain import ( + TOTAL_SUPPLY_SOURCE_INDEXER, + UINT64_MAX, + Asset, + SyncState, +) from flex.db.model.liquidity_pools import LpState, LpTransaction +from flex.db.model.priced import AssetPrice from flex.db.sync_coordinator import ( MongoSyncCoordinator, SyncCoordinatorError, ) from flex.domain.lp_projection import lp_round_end_order +from flex.domain.pricing import MAX_OBSERVATION_CLOCK_SKEW +from flex.domain.transactions import ASSET_TRANSFER_TX pytestmark = pytest.mark.integration @@ -99,12 +116,17 @@ def _state(*, reserve: int = 100) -> LpState: ) -def _transaction(*, amount: int = 10) -> LpTransaction: +def _transaction( + *, + amount: int = 10, + asa_id: int = 7, + event_id: str = "TX@POOL", +) -> LpTransaction: return LpTransaction( - id="TX@POOL", + id=event_id, pool_address="POOL", user_address="USER", - asa_id=7, + asa_id=asa_id, delta_amount_micros=amount, confirmed_round=100, event_position=1, @@ -200,6 +222,74 @@ def test_real_mongo_decimal128_increment_reaches_uint64_max( assert raw["asset1_reserve_micros"] == Decimal128(str(maximum)) +def test_real_mongo_projects_token_pool_fee_to_operational_algo( + mongo_database: Database, +) -> None: + state = _state() + state.asset2_id = 8 + state.operational_algo_balance_micros = 1_000 + repository = _repository(mongo_database, state) + + outcome = repository.project( + _transaction( + amount=-1_000, + asa_id=0, + event_id="TX#fee@POOL", + ), + ) + + raw = mongo_database["lp_states"].find_one({"address": "POOL"}) + assert outcome.state.asset1_reserve_micros == 100 + assert outcome.state.asset2_reserve_micros == 100 + assert outcome.state.operational_algo_balance_micros == 0 + assert raw is not None + assert raw["operational_algo_balance_micros"] == Decimal128("0") + + +def test_real_mongo_replays_amount_and_fee_from_one_raw_transaction( + mongo_database: Database, + monkeypatch: pytest.MonkeyPatch, +) -> None: + state = _state() + state.asset2_id = 8 + state.operational_algo_balance_micros = 2_000 + repository = _repository(mongo_database, state) + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 8, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 100, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "USER", + }, + } + + transactions = asyncio.run(sync_pools.process_lp_transactions([raw_transaction])) + ordered = sorted(transactions, key=lambda transaction: transaction.event_order) + first_outcomes = [repository.project(transaction) for transaction in ordered] + replay_outcomes = [repository.project(transaction) for transaction in ordered] + + persisted = repository.get_state("POOL") + assert [transaction.id for transaction in transactions] == [ + "TX@POOL", + "TX#fee@POOL", + ] + assert persisted.asset1_reserve_micros == 95 + assert persisted.asset2_reserve_micros == 100 + assert persisted.operational_algo_balance_micros == 1_000 + assert persisted.last_event_order == max(transaction.event_order for transaction in transactions) + assert mongo_database["lp_transactions"].count_documents({}) == 2 + assert all(outcome.result is LpProjectionResult.APPLIED for outcome in first_outcomes) + assert all(outcome.result is LpProjectionResult.ALREADY_APPLIED for outcome in replay_outcomes) + + def test_real_mongo_promotes_legacy_int64_balance_to_decimal128( mongo_database: Database, ) -> None: @@ -214,6 +304,7 @@ def test_real_mongo_promotes_legacy_int64_balance_to_decimal128( "asset1_reserve_micros", "asset2_reserve_micros", "total_tokens_micros", + "operational_algo_balance_micros", ): payload[field_name] = int(payload[field_name].to_decimal()) @@ -255,6 +346,153 @@ def test_duplicate_financial_business_key_fails_without_deletion( assert collection.count_documents({}) == 2 +def test_duplicate_asset_identity_fails_without_deletion( + mongo_database: Database, +) -> None: + collection = mongo_database["assets"] + collection.insert_many( + [ + {"id": Decimal128("42"), "name": "first"}, + {"id": Decimal128("42"), "name": "second"}, + ] + ) + + with pytest.raises(RuntimeError, match="duplicate immutable ID"): + create_unique_id_index_fail_closed( + collection, + collection_name="assets", + ) + + assert collection.count_documents({}) == 2 + + +def test_real_mongo_deletes_only_retired_lp_price_projections( + mongo_database: Database, +) -> None: + collection = mongo_database["asset_prices"] + collection.insert_many( + [ + {"id": Decimal128("1"), "source": "vestige"}, + {"id": Decimal128("2"), "source": "vestige", "tinyman_algo_pool_id": None}, + { + "id": Decimal128("3"), + "source": "tinyman", + "tinyman_algo_pool_id": Decimal128("99"), + }, + ] + ) + + removed = delete_unverified_legacy_lp_prices( + SimpleNamespace( + asset_prices=SimpleNamespace( + mongodb_collection=collection, + ) + ) + ) + + assert removed == 1 + assert {document["id"].to_decimal() for document in collection.find({}, projection={"id": 1})} == {1, 2} + + +def test_valid_price_replaces_far_future_observation( + mongo_database: Database, + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = mongo_database["asset_prices"] + collection.create_index("id", unique=True) + now = datetime.now(UTC) + collection.insert_one( + { + "id": Decimal128("42"), + "name": "poisoned", + "price_algo": 999.0, + "price_usd": 999.0, + "last_update_round": Decimal128("1"), + "source": "vestige", + "observed_at": now + MAX_OBSERVATION_CLOCK_SKEW + timedelta(days=1), + "created": now, + "updated": now, + } + ) + monkeypatch.setattr( + asset_price_data, + "db", + SimpleNamespace( + asset_prices=SimpleNamespace( + mongodb_collection=collection, + ) + ), + ) + candidate = AssetPrice( + id=42, + name="verified", + price_algo=0.25, + price_usd=1.0, + last_update_round=2, + source="vestige", + observed_at=now, + ) + + assert asset_price_data._upsert_asset_price(candidate) is True + + stored = collection.find_one({"id": Decimal128("42")}) + assert stored is not None + assert stored["name"] == "verified" + assert abs(stored["observed_at"] - now) < timedelta(milliseconds=1) + + +@pytest.mark.asyncio +async def test_concurrent_real_mongo_supply_migration_persists_provenance_once( + mongo_database: Database, + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = mongo_database["assets"] + inserted = collection.insert_one( + { + "id": Decimal128("42"), + "total_supply": float(UINT64_MAX), + "total_supply_micros": str(2**53), + } + ) + legacy_document = collection.find_one({"_id": inserted.inserted_id}) + assert legacy_document is not None + monkeypatch.setattr( + asset_data, + "db", + SimpleNamespace( + assets=SimpleNamespace( + mongodb_collection=collection, + ) + ), + ) + authoritative = Asset( + id=42, + name="Canonical Asset", + decimals=0, + unit_name="CANON", + creator="CREATOR", + reserve="RESERVE", + total_supply=0, + total_supply_micros=UINT64_MAX, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, + ) + + async def fetch_asset(asset_id: int) -> Asset: + assert asset_id == 42 + await asyncio.sleep(0) + return authoritative + + monkeypatch.setattr(asset_data, "fetch_asset", fetch_asset) + + results = await asyncio.gather(*(asset_data._backfill_canonical_supply(42, legacy_document) for _ in range(8))) + + persisted = collection.find_one({"_id": inserted.inserted_id}) + assert results == [UINT64_MAX] * 8 + assert persisted is not None + assert persisted["total_supply_micros"] == str(UINT64_MAX) + assert persisted["total_supply_source"] == TOTAL_SUPPLY_SOURCE_INDEXER + + def test_expired_real_mongo_sync_lease_fences_stale_owner( mongo_database: Database, ) -> None: diff --git a/tests/unit/test_algorand_snapshot_boundaries.py b/tests/unit/test_algorand_snapshot_boundaries.py new file mode 100644 index 00000000..5c22f76d --- /dev/null +++ b/tests/unit/test_algorand_snapshot_boundaries.py @@ -0,0 +1,101 @@ +import asyncio + +import pytest + +from flex.blockchain import info +from flex.domain.algorand import MAX_ALGORAND_UINT + + +def _response(*, assets: list[dict], amount: object = 10, round_number: object = 20) -> dict: + return { + "account": { + "assets": assets, + "amount": amount, + }, + "current-round": round_number, + } + + +def _install_response(monkeypatch, response: dict) -> None: + async def run_sync(func, *args): + del func, args + return response + + monkeypatch.setattr(info, "_run_sync", run_sync) + + +def test_account_snapshot_preserves_full_uint64_domain(monkeypatch) -> None: + _install_response( + monkeypatch, + _response( + assets=[{"asset-id": MAX_ALGORAND_UINT, "amount": MAX_ALGORAND_UINT}], + amount=MAX_ALGORAND_UINT, + round_number=MAX_ALGORAND_UINT, + ), + ) + + snapshot = asyncio.run( + info.get_address_asset_snapshot("ACCOUNT", include_algo=True), + ) + + assert snapshot.balances == { + 0: MAX_ALGORAND_UINT, + MAX_ALGORAND_UINT: MAX_ALGORAND_UINT, + } + assert snapshot.observed_round == MAX_ALGORAND_UINT + + +@pytest.mark.parametrize( + "assets", + [ + [{"asset-id": True, "amount": 1}], + [{"asset-id": 0, "amount": 1}], + [{"asset-id": 1, "amount": -1}], + [{"asset-id": 1, "amount": MAX_ALGORAND_UINT + 1}], + [ + {"asset-id": 1, "amount": 1}, + {"asset-id": 1, "amount": 2}, + ], + ], +) +def test_account_snapshot_rejects_malformed_or_duplicate_holdings( + monkeypatch, + assets: list[dict], +) -> None: + _install_response(monkeypatch, _response(assets=assets)) + + with pytest.raises(RuntimeError): + asyncio.run( + info.get_address_asset_snapshot("ACCOUNT", include_algo=True), + ) + + +@pytest.mark.parametrize( + ("amount", "round_number"), + [ + (True, 1), + (-1, 1), + (MAX_ALGORAND_UINT + 1, 1), + (1, True), + (1, -1), + (1, MAX_ALGORAND_UINT + 1), + ], +) +def test_account_snapshot_rejects_invalid_algo_or_round( + monkeypatch, + amount: object, + round_number: object, +) -> None: + _install_response( + monkeypatch, + _response( + assets=[], + amount=amount, + round_number=round_number, + ), + ) + + with pytest.raises(RuntimeError): + asyncio.run( + info.get_address_asset_snapshot("ACCOUNT", include_algo=True), + ) diff --git a/tests/unit/test_asset_price_fallback.py b/tests/unit/test_asset_price_fallback.py index c46c141d..f5e3919f 100644 --- a/tests/unit/test_asset_price_fallback.py +++ b/tests/unit/test_asset_price_fallback.py @@ -7,7 +7,12 @@ from env import settings from flex.data import asset_prices from flex.db.model.priced import AssetPrice -from flex.domain.pricing import PriceUnavailableError +from flex.domain.pricing import ( + PriceQuote, + PriceSource, + PriceUnavailableError, + PricingError, +) def _stored_price( @@ -15,6 +20,8 @@ def _stored_price( age_seconds: int, price_algo: float = 2.0, price_usd: float = 0.5, + tinyman_algo_pool_id: int | None = None, + source: str = "vestige", ) -> AssetPrice: observed_at = datetime.now(UTC) - timedelta(seconds=age_seconds) return AssetPrice( @@ -23,7 +30,8 @@ def _stored_price( price_algo=price_algo, price_usd=price_usd, last_update_round=123, - source="vestige", + tinyman_algo_pool_id=tinyman_algo_pool_id, + source=source, observed_at=observed_at, created=observed_at, updated=observed_at, @@ -78,6 +86,75 @@ def test_refresh_failure_rejects_price_older_than_max_stale(monkeypatch) -> None asyncio.run(asset_prices.get_asset_price_not_cached(42)) +def test_refresh_failure_never_serves_legacy_raw_lp_projection(monkeypatch) -> None: + stored = _stored_price(age_seconds=0, tinyman_algo_pool_id=999) + _database(monkeypatch, stored) + monkeypatch.setattr(asset_prices, "update_asset_price", _failed_refresh) + + with pytest.raises(PriceUnavailableError, match="maximum stale window"): + asyncio.run(asset_prices.get_asset_price_not_cached(42)) + + +def test_refresh_failure_never_serves_derived_lp_source_without_pool_marker( + monkeypatch, +) -> None: + stored = _stored_price(age_seconds=0, source=PriceSource.DERIVED_LP.value) + _database(monkeypatch, stored) + monkeypatch.setattr(asset_prices, "update_asset_price", _failed_refresh) + + with pytest.raises(PriceUnavailableError, match="maximum stale window"): + asyncio.run(asset_prices.get_asset_price_not_cached(42)) + + +def test_provider_refresh_removes_legacy_raw_lp_provenance(monkeypatch) -> None: + stored = _stored_price(age_seconds=0, tinyman_algo_pool_id=999) + observed_at = datetime.now(UTC) + quote = PriceQuote.from_raw( + asset_id=42, + algo="3", + usd="0.75", + source=PriceSource.VESTIGE, + observed_at=observed_at, + observed_round=456, + stale_after=timedelta(seconds=settings.asset_prices_ttl), + ) + persisted = [] + + async def fetch_quote(*args, **kwargs) -> PriceQuote: + return quote + + monkeypatch.setattr(asset_prices, "fetch_vestige_price_quote", fetch_quote) + monkeypatch.setattr( + asset_prices, + "_upsert_asset_price", + lambda price: persisted.append(price) is None, + ) + + result = asyncio.run(asset_prices.update_asset_price(stored, current_round=456)) + + assert result.tinyman_algo_pool_id is None + assert result.source == PriceSource.VESTIGE.value + assert result.observed_at == observed_at + assert persisted == [result] + + +def test_legacy_raw_lp_projection_cannot_cross_persistence_boundary() -> None: + stored = _stored_price(age_seconds=0, tinyman_algo_pool_id=999) + + with pytest.raises(PricingError, match="projections are retired"): + asset_prices._upsert_asset_price(stored) + + +def test_derived_lp_source_without_pool_marker_cannot_be_persisted() -> None: + stored = _stored_price( + age_seconds=0, + source=PriceSource.DERIVED_LP.value, + ) + + with pytest.raises(PricingError, match="projections are retired"): + asset_prices._upsert_asset_price(stored) + + def test_invalid_fresh_database_value_is_refreshed(monkeypatch) -> None: stored = _stored_price(age_seconds=0, price_algo=float("nan")) refreshed = _stored_price(age_seconds=0, price_algo=3.0, price_usd=0.75) @@ -105,12 +182,22 @@ def test_list_reads_filter_invalid_and_expired_prices(monkeypatch) -> None: age_seconds=settings.asset_prices_max_stale + 1, ) expired.id = 44 + legacy = _stored_price( + age_seconds=0, + tinyman_algo_pool_id=999, + ) + legacy.id = 45 + derived = _stored_price( + age_seconds=0, + source=PriceSource.DERIVED_LP.value, + ) + derived.id = 46 monkeypatch.setattr( asset_prices, "db", SimpleNamespace( asset_prices=SimpleNamespace( - get_all=lambda: [valid, invalid, expired], + get_all=lambda: [valid, invalid, expired, legacy, derived], ), ), ) diff --git a/tests/unit/test_asset_supply.py b/tests/unit/test_asset_supply.py index f98ebe7b..2974ade9 100644 --- a/tests/unit/test_asset_supply.py +++ b/tests/unit/test_asset_supply.py @@ -1,10 +1,17 @@ from collections.abc import Callable +from copy import deepcopy +from types import SimpleNamespace from typing import Any import pytest from flex.blockchain import info -from flex.db.model.blockchain import UINT64_MAX, Asset +from flex.data import assets as asset_data +from flex.db.model.blockchain import ( + TOTAL_SUPPLY_SOURCE_INDEXER, + UINT64_MAX, + Asset, +) def _asset_response(*, total: int, decimals: int) -> dict[str, Any]: @@ -41,10 +48,13 @@ async def fake_run_sync(func: Callable[..., Any], *args: Any) -> dict[str, Any]: stored = asset.to_dict() assert stored["total_supply_micros"] == str(total) - assert Asset.from_dict(stored).total_supply_micros == total + assert stored["total_supply_source"] == TOTAL_SUPPLY_SOURCE_INDEXER + restored = Asset.from_dict(stored) + assert restored.total_supply_micros == total + assert restored.total_supply_is_authoritative is True -def test_legacy_asset_document_backfills_canonical_supply() -> None: +def test_legacy_asset_document_marks_reconstructed_supply_non_authoritative() -> None: legacy_document = { "id": 42, "name": "Legacy Asset", @@ -58,7 +68,8 @@ def test_legacy_asset_document_backfills_canonical_supply() -> None: asset = Asset.from_dict(legacy_document) assert asset.total_supply_micros == 123_456_789 - assert asset.to_dict()["total_supply_micros"] == "123456789" + assert asset.total_supply_is_authoritative is False + assert "total_supply_micros" not in asset.to_dict() def test_canonical_supply_wins_over_lossy_legacy_float() -> None: @@ -73,10 +84,12 @@ def test_canonical_supply_wins_over_lossy_legacy_float() -> None: reserve="RESERVE", total_supply=float(canonical_supply), total_supply_micros=canonical_supply, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, ) assert int(asset.total_supply) != canonical_supply assert asset.total_supply_micros == canonical_supply + assert asset.total_supply_is_authoritative is True @pytest.mark.parametrize("invalid_supply", [-1, UINT64_MAX + 1]) @@ -92,3 +105,233 @@ def test_asset_rejects_supply_outside_algorand_uint64(invalid_supply: int) -> No total_supply=0, total_supply_micros=invalid_supply, ) + + +class _AssetCollection: + def __init__(self, document: dict[str, Any]) -> None: + self.document = document + self.update_query: dict[str, Any] | None = None + + def find_one( + self, + query: dict[str, Any], + projection: dict[str, int] | None = None, + ) -> dict[str, Any] | None: + if "_id" in query and query["_id"] != self.document["_id"]: + return None + result = deepcopy(self.document) + if projection is None: + return result + return {key: value for key, value in result.items() if key == "_id" or projection.get(key)} + + def find_one_and_update( + self, + query: dict[str, Any], + update: dict[str, Any], + **kwargs: Any, + ) -> dict[str, Any] | None: + del kwargs + self.update_query = query + may_migrate = ( + self.document.get("total_supply_micros") is None + or self.document.get("total_supply_source") != TOTAL_SUPPLY_SOURCE_INDEXER + ) + if query["_id"] != self.document["_id"] or not may_migrate: + return None + self.document.update(update["$set"]) + return deepcopy(self.document) + + +class _AssetManager: + def __init__(self, collection: _AssetCollection) -> None: + self.mongodb_collection = collection + + def get_by_primary_key( + self, + asset_id: int, + *, + throw_ex: bool, + ) -> Asset | None: + del asset_id, throw_ex + return Asset.from_dict(deepcopy(self.mongodb_collection.document)) + + +@pytest.mark.asyncio +async def test_financial_supply_backfills_from_indexer_instead_of_legacy_float( + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = _AssetCollection( + { + "_id": "asset-42", + "id": 42, + "total_supply": float(UINT64_MAX), + } + ) + monkeypatch.setattr( + asset_data, + "db", + SimpleNamespace( + assets=SimpleNamespace( + mongodb_collection=collection, + ) + ), + ) + authoritative = Asset( + id=42, + name="Canonical Asset", + decimals=0, + unit_name="CANON", + creator="CREATOR", + reserve="RESERVE", + total_supply=0, + total_supply_micros=UINT64_MAX, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, + ) + fetches: list[int] = [] + + async def fetch_asset(asset_id: int) -> Asset: + fetches.append(asset_id) + return authoritative + + monkeypatch.setattr(asset_data, "fetch_asset", fetch_asset) + + supply = await asset_data.get_asset_total_supply.__wrapped__(42) + + assert supply == UINT64_MAX + assert fetches == [42] + assert collection.document["total_supply_micros"] == str(UINT64_MAX) + assert collection.document["total_supply_source"] == TOTAL_SUPPLY_SOURCE_INDEXER + assert collection.update_query == { + "_id": "asset-42", + "$or": [ + {"total_supply_micros": {"$exists": False}}, + {"total_supply_micros": None}, + {"total_supply_source": {"$ne": TOTAL_SUPPLY_SOURCE_INDEXER}}, + ], + } + + +@pytest.mark.asyncio +async def test_financial_supply_uses_persisted_canonical_units_without_indexer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = _AssetCollection( + { + "_id": "asset-42", + "id": 42, + "total_supply": 1.0, + "total_supply_micros": str(2**53 + 1), + "total_supply_source": TOTAL_SUPPLY_SOURCE_INDEXER, + } + ) + monkeypatch.setattr( + asset_data, + "db", + SimpleNamespace( + assets=SimpleNamespace( + mongodb_collection=collection, + ) + ), + ) + + async def unexpected_fetch(asset_id: int) -> Asset: + pytest.fail(f"unexpected Indexer read for asset {asset_id}") + + monkeypatch.setattr(asset_data, "fetch_asset", unexpected_fetch) + + supply = await asset_data.get_asset_total_supply.__wrapped__(42) + + assert supply == 2**53 + 1 + assert collection.update_query is None + + +@pytest.mark.asyncio +async def test_unverified_persisted_supply_is_overwritten_from_indexer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = _AssetCollection( + { + "_id": "asset-42", + "id": 42, + "total_supply": float(2**53), + "total_supply_micros": str(2**53), + "total_supply_source": None, + } + ) + monkeypatch.setattr( + asset_data, + "db", + SimpleNamespace( + assets=SimpleNamespace( + mongodb_collection=collection, + ) + ), + ) + authoritative_supply = 2**53 + 1 + + async def fetch_asset(asset_id: int) -> Asset: + assert asset_id == 42 + return Asset( + id=42, + name="Canonical Asset", + decimals=0, + unit_name="CANON", + creator="CREATOR", + reserve="RESERVE", + total_supply=0, + total_supply_micros=authoritative_supply, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, + ) + + monkeypatch.setattr(asset_data, "fetch_asset", fetch_asset) + + supply = await asset_data.get_asset_total_supply.__wrapped__(42) + + assert supply == authoritative_supply + assert collection.document["total_supply_micros"] == str(authoritative_supply) + assert collection.document["total_supply_source"] == TOTAL_SUPPLY_SOURCE_INDEXER + + +@pytest.mark.asyncio +async def test_full_asset_read_migrates_legacy_supply_before_returning( + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = _AssetCollection( + { + "_id": "asset-42", + "id": 42, + "name": "Legacy Asset", + "decimals": 0, + "unit_name": "OLD", + "creator": "CREATOR", + "reserve": "RESERVE", + "total_supply": float(UINT64_MAX), + } + ) + manager = _AssetManager(collection) + monkeypatch.setattr( + asset_data, + "db", + SimpleNamespace(assets=manager), + ) + + async def fetch_asset(asset_id: int) -> Asset: + assert asset_id == 42 + return Asset( + id=42, + name="Canonical Asset", + decimals=0, + unit_name="CANON", + creator="CREATOR", + reserve="RESERVE", + total_supply=0, + total_supply_micros=UINT64_MAX, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, + ) + + monkeypatch.setattr(asset_data, "fetch_asset", fetch_asset) + + asset = await asset_data.get_full_asset.__wrapped__(42) + + assert asset.total_supply_micros == UINT64_MAX + assert asset.total_supply_is_authoritative is True diff --git a/tests/unit/test_bson_uint64_models.py b/tests/unit/test_bson_uint64_models.py index 8c8284f1..75a04595 100644 --- a/tests/unit/test_bson_uint64_models.py +++ b/tests/unit/test_bson_uint64_models.py @@ -1,9 +1,12 @@ from datetime import UTC, datetime +import pytest from bson import BSON, Decimal128 -from flex.db.model.blockchain import Asset, LpToken +from flex.db.model.blockchain import TOTAL_SUPPLY_SOURCE_INDEXER, Asset, LpToken +from flex.db.model.liquidity_pools import LpState from flex.db.model.priced import AssetPrice +from flex.db.model.transfers import AssetTransferIntent UINT64_MAX = 2**64 - 1 @@ -40,6 +43,7 @@ def test_asset_round_trips_full_uint64_identifier_and_supply() -> None: reserve="RESERVE", total_supply=0, total_supply_micros=UINT64_MAX, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, ) restored = Asset.from_dict( @@ -76,3 +80,54 @@ def test_asset_price_round_trips_full_uint64_identifiers_and_round() -> None: assert AssetPrice.encode_query({"id": UINT64_MAX}) == { "id": Decimal128(str(UINT64_MAX)), } + + +def test_transfer_intent_round_trips_full_uint64_rounds() -> None: + timestamp = datetime(2026, 1, 1, tzinfo=UTC) + intent = AssetTransferIntent( + id="airdrop:max-round", + receiver="receiver", + asset_id="1", + amount_micros="1", + note=None, + signed_transaction="signed", + txid="txid", + first_valid_round=UINT64_MAX - 1, + last_valid_round=UINT64_MAX, + status="confirmed", + confirmed_round=UINT64_MAX, + created=timestamp, + updated=timestamp, + ) + + encoded = intent.to_dict() + restored = AssetTransferIntent.from_dict( + BSON(BSON.encode(encoded)).decode(), + ) + + assert encoded["first_valid_round"] == Decimal128(str(UINT64_MAX - 1)) + assert encoded["last_valid_round"] == Decimal128(str(UINT64_MAX)) + assert encoded["confirmed_round"] == Decimal128(str(UINT64_MAX)) + assert restored.first_valid_round == UINT64_MAX - 1 + assert restored.last_valid_round == UINT64_MAX + assert restored.confirmed_round == UINT64_MAX + + +def test_lp_state_rejects_invalid_balance_before_initial_insert() -> None: + with pytest.raises(ValueError, match="asset1_reserve_micros"): + LpState( + id=1, + token_id=99, + asset1_id=7, + asset2_id=0, + dex_provider="tinyman", + address="POOL", + last_updated_round=1, + asset1_reserve_micros=True, + asset2_reserve_micros=1, + total_tokens_micros=1, + asset1_reserve=0, + asset2_reserve=0, + total_tokens=0, + token_price_algo=0, + ) diff --git a/tests/unit/test_lp_api_contract.py b/tests/unit/test_lp_api_contract.py index 3fa09e11..c3dd11ef 100644 --- a/tests/unit/test_lp_api_contract.py +++ b/tests/unit/test_lp_api_contract.py @@ -10,7 +10,12 @@ from flex.db.model.priced import AssetPrice -def _lp_price(token_id: int, *, age_seconds: int = 0) -> AssetPrice: +def _lp_price( + token_id: int, + *, + age_seconds: int = 0, + source: str = "vestige", +) -> AssetPrice: observed_at = datetime.now(UTC) - timedelta(seconds=age_seconds) return AssetPrice( id=token_id, @@ -18,7 +23,7 @@ def _lp_price(token_id: int, *, age_seconds: int = 0) -> AssetPrice: price_algo=1.25 if token_id == 8 else 2.5, price_usd=0.42 if token_id == 8 else 0.84, last_update_round=123, - source="derived_lp", + source=source, observed_at=observed_at, created=observed_at, updated=observed_at, @@ -90,7 +95,7 @@ def get_many_by_query(self, query: dict) -> list[SimpleNamespace]: self.calls.append(query) return [ _lp_price(8), - _lp_price(13), + _lp_price(13, source="derived_lp"), ] asset_prices = AssetPrices() @@ -115,11 +120,7 @@ def get_many_by_query(self, query: dict) -> list[SimpleNamespace]: "token_price_usd": 0.42, }, "5": None, - "13": { - "token_id": 13, - "token_price_algo": 2.5, - "token_price_usd": 0.84, - }, + "13": None, }, } @@ -147,6 +148,29 @@ def test_lp_single_response_remains_backward_compatible(monkeypatch) -> None: } +def test_lp_endpoint_hides_retired_derived_price_without_pool_marker( + monkeypatch, +) -> None: + asset_prices = SimpleNamespace( + get_many_by_query=lambda query: [ + _lp_price(8, source="derived_lp"), + ], + ) + monkeypatch.setattr( + api, + "db", + SimpleNamespace(asset_prices=asset_prices), + ) + + response = asyncio.run( + api.handle_get_lp_state_priced( + api.LpStatePricedRequest(lp_token_id=8), + ), + ) + + assert response is None + + def test_lp_endpoint_hides_price_older_than_max_stale(monkeypatch) -> None: asset_prices = SimpleNamespace( get_many_by_query=lambda query: [ diff --git a/tests/unit/test_lp_projection_domain.py b/tests/unit/test_lp_projection_domain.py new file mode 100644 index 00000000..95549e54 --- /dev/null +++ b/tests/unit/test_lp_projection_domain.py @@ -0,0 +1,27 @@ +from flex.domain.lp_projection import lp_balance_delta + + +def test_algo_pool_fee_debits_the_economic_algo_reserve() -> None: + delta = lp_balance_delta( + token_id=99, + asset1_id=7, + asset2_id=0, + event_asset_id=0, + event_pool_delta_micros=-1_000, + ) + + assert delta.field == "asset2_reserve_micros" + assert delta.amount == -1_000 + + +def test_token_token_pool_fee_debits_the_operational_algo_balance() -> None: + delta = lp_balance_delta( + token_id=99, + asset1_id=7, + asset2_id=8, + event_asset_id=0, + event_pool_delta_micros=-1_000, + ) + + assert delta.field == "operational_algo_balance_micros" + assert delta.amount == -1_000 diff --git a/tests/unit/test_lp_projection_repository.py b/tests/unit/test_lp_projection_repository.py index 9d948a37..5f6f399b 100644 --- a/tests/unit/test_lp_projection_repository.py +++ b/tests/unit/test_lp_projection_repository.py @@ -3,6 +3,7 @@ from datetime import UTC, datetime, timedelta from threading import RLock from types import SimpleNamespace +from unittest.mock import Mock import pytest from bson import BSON, Decimal128 @@ -150,6 +151,7 @@ def _state( def _transaction( event_id: str = "TX@POOL", *, + asa_id: int = 7, amount: int = 10, round_number: int = 100, position: int = 1, @@ -158,7 +160,7 @@ def _transaction( id=event_id, pool_address="POOL", user_address="USER", - asa_id=7, + asa_id=asa_id, delta_amount_micros=amount, confirmed_round=round_number, event_position=position, @@ -381,6 +383,23 @@ def test_uint64_identifiers_and_cursors_are_bson_safe() -> None: assert restored.event_position == maximum +def test_operational_algo_balance_is_bson_safe_and_backward_compatible() -> None: + maximum = 2**64 - 1 + state = _state() + state.operational_algo_balance_micros = maximum + + encoded = BSON.encode(state.to_dict()) + restored = LpState.from_dict(BSON(encoded).decode()) + + assert restored.operational_algo_balance_micros == maximum + + legacy_payload = state.to_dict() + legacy_payload.pop("operational_algo_balance_micros") + restored_legacy = LpState.from_dict(legacy_payload) + + assert restored_legacy.operational_algo_balance_micros == 0 + + def test_uint64_repository_queries_and_updates_are_bson_safe() -> None: maximum = 2**64 - 1 state = _state( @@ -416,6 +435,36 @@ def test_uint64_repository_queries_and_updates_are_bson_safe() -> None: ) +def test_token_token_pool_fee_updates_only_operational_algo_balance() -> None: + state = _state( + cursor=lp_round_end_order(99), + reserve=100, + ) + state.asset2_id = 8 + state.asset2_reserve_micros = 200 + state.operational_algo_balance_micros = 5_000 + repository, states, _ = _repository(state) + + outcome = repository.project( + _transaction( + event_id="TX#fee@POOL", + asa_id=0, + amount=-1_000, + ), + ) + + assert outcome.result is LpProjectionResult.APPLIED + assert outcome.state.asset1_reserve_micros == 100 + assert outcome.state.asset2_reserve_micros == 200 + assert outcome.state.operational_algo_balance_micros == 4_000 + persisted = LpState.from_dict(states.documents[0]) + assert persisted.operational_algo_balance_micros == 4_000 + assert isinstance( + states.documents[0]["operational_algo_balance_micros"], + Decimal128, + ) + + def test_event_position_is_sorted_numerically_within_a_round() -> None: assert lp_event_order(100, "TX-2", 2) < lp_event_order(100, "TX-10", 10) @@ -435,6 +484,48 @@ def test_snapshot_cannot_overwrite_a_newer_event_cursor() -> None: assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 150 +def test_balance_snapshot_does_not_publish_derived_price_fields() -> None: + observed_at = datetime(2026, 1, 1, tzinfo=UTC) + current = _state( + cursor=lp_round_end_order(100), + last_round=100, + reserve=100, + ) + current.asset1_reserve = 1.0 + current.asset2_reserve = 2.0 + current.total_tokens = 3.0 + current.token_price_algo = 4.0 + current.derived_observed_at = observed_at + current.asset2_id = 8 + current.operational_algo_balance_micros = 5_000 + repository, states, _ = _repository(current) + + incoming = _state(cursor=None, last_round=101, reserve=150) + incoming.asset1_reserve = 10.0 + incoming.asset2_reserve = 20.0 + incoming.total_tokens = 30.0 + incoming.token_price_algo = 40.0 + incoming.derived_observed_at = observed_at + timedelta(minutes=1) + incoming.asset2_id = 8 + incoming.operational_algo_balance_micros = 6_000 + + result = repository.replace_snapshot( + incoming, + observed_round=101, + ) + + assert result.asset1_reserve_micros == 150 + assert result.operational_algo_balance_micros == 6_000 + assert result.asset1_reserve == 1.0 + assert result.asset2_reserve == 2.0 + assert result.total_tokens == 3.0 + assert result.token_price_algo == 4.0 + assert result.derived_observed_at == observed_at + persisted = LpState.from_dict(states.documents[0]) + assert persisted.token_price_algo == 4.0 + assert persisted.derived_observed_at == observed_at + + def test_same_round_snapshot_with_different_balances_fails_closed() -> None: snapshot_cursor = lp_round_end_order(100) repository, states, _ = _repository( @@ -450,26 +541,48 @@ def test_same_round_snapshot_with_different_balances_fails_closed() -> None: assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 100 -def test_older_derived_calculation_cannot_overwrite_newer_price() -> None: - cursor = lp_round_end_order(100) - newer_time = datetime(2026, 1, 1, 0, 1, tzinfo=UTC) - repository, states, _ = _repository( - _state(cursor=cursor, last_round=100), +@pytest.mark.parametrize( + ("field_name", "invalid_value"), + [ + ("asset1_reserve_micros", -1), + ("asset2_reserve_micros", True), + ("total_tokens_micros", 2**64), + ("operational_algo_balance_micros", 1.5), + ], +) +def test_invalid_snapshot_is_rejected_before_mongo_write( + field_name: str, + invalid_value: object, +) -> None: + state = _state(cursor=None) + setattr(state, field_name, invalid_value) + states = SimpleNamespace( + find_one_and_update=Mock(), + ) + repository = MongoLpProjectionRepository( + states=states, # type: ignore[arg-type] + events=AtomicCollection(), # type: ignore[arg-type] ) - states.documents[0]["derived_observed_at"] = newer_time - states.documents[0]["token_price_algo"] = 2.0 - stale = _state(cursor=cursor, last_round=100) - stale.derived_observed_at = newer_time - timedelta(minutes=1) - stale.token_price_algo = 1.0 - updated = repository.update_derived_fields( - stale, - expected_cursor=cursor, + with pytest.raises(LpProjectionPersistenceError, match=field_name): + repository.replace_snapshot(state, observed_round=100) + + states.find_one_and_update.assert_not_called() + + +def test_event_marker_counterparty_is_immutable() -> None: + expected = _transaction() + conflicting = _transaction() + conflicting.user_address = "OTHER-USER" + repository, states, _ = _repository( + _state(cursor=lp_round_end_order(99)), + events=[conflicting], ) - assert updated is None - assert states.documents[0]["token_price_algo"] == 2.0 - assert states.documents[0]["derived_observed_at"] == newer_time + with pytest.raises(LpProjectionPersistenceError, match="immutable data"): + repository.project(expected) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 100 def test_expired_round_lease_uses_fencing_owner_and_checkpoint_cas() -> None: diff --git a/tests/unit/test_lp_token_identity.py b/tests/unit/test_lp_token_identity.py new file mode 100644 index 00000000..84a286b0 --- /dev/null +++ b/tests/unit/test_lp_token_identity.py @@ -0,0 +1,51 @@ +from types import SimpleNamespace + +import pytest + +from flex.data import lp_tokens +from flex.db.model.blockchain import LpToken + + +def _token(*, address: str = "POOL") -> LpToken: + return LpToken( + id=99, + pool_id=123, + asset1_id=7, + asset2_id=0, + address=address, + dex_provider="tinyman", + ) + + +def test_lp_token_registration_returns_matching_atomic_winner( + monkeypatch: pytest.MonkeyPatch, +) -> None: + candidate = _token() + manager = SimpleNamespace(get_or_create=lambda item: item) + monkeypatch.setattr( + lp_tokens, + "db", + SimpleNamespace(lp_tokens=manager), + ) + + assert lp_tokens.persist_lp_token(candidate) is candidate + + +def test_lp_token_registration_fails_closed_on_identity_conflict( + monkeypatch: pytest.MonkeyPatch, +) -> None: + candidate = _token() + manager = SimpleNamespace( + get_or_create=lambda item: _token(address="OTHER_POOL"), + ) + monkeypatch.setattr( + lp_tokens, + "db", + SimpleNamespace(lp_tokens=manager), + ) + + with pytest.raises( + lp_tokens.LpTokenIdentityConflictError, + match="different pool metadata", + ): + lp_tokens.persist_lp_token(candidate) diff --git a/tests/unit/test_lp_transaction_projection.py b/tests/unit/test_lp_transaction_projection.py index 8077e115..79bbbbc6 100644 --- a/tests/unit/test_lp_transaction_projection.py +++ b/tests/unit/test_lp_transaction_projection.py @@ -9,20 +9,25 @@ LpProjectionOutcome, LpProjectionResult, ) -from flex.db.model.liquidity_pools import LpTransaction -from flex.domain.transactions import ASSET_TRANSFER_TX +from flex.db.model.liquidity_pools import LpState, LpTransaction +from flex.domain.algorand import MAX_ALGORAND_UINT +from flex.domain.transactions import ASSET_TRANSFER_TX, PAYMENT_TX def test_lp_to_lp_transfer_updates_both_scoped_projections(monkeypatch) -> None: monkeypatch.setattr( sync_pools, - "get_all_lp_state_addresses", - lambda: {"POOL-A", "POOL-B"}, + "get_lp_tracked_assets_by_address", + lambda: { + "POOL-A": frozenset({0, 7, 99}), + "POOL-B": frozenset({0, 7, 98}), + }, ) raw_transaction = { "id": "TX", "sender": "POOL-A", "confirmed-round": 123, + "fee": 0, ASSET_TRANSFER_TX: { "asset-id": 7, "amount": 5, @@ -84,8 +89,7 @@ def get_state(self, pool_address): return states[pool_address] def update_derived_fields(self, state, *, expected_cursor): - assert state.last_event_order == expected_cursor - return state + raise AssertionError("ledger projection must not persist derived prices") repository = FakeProjectionRepository() monkeypatch.setattr( @@ -102,13 +106,20 @@ def update_derived_fields(self, state, *, expected_cursor): lambda **kwargs: repository, ) - async def unchanged(state): - return state + recalculation_calls = 0 + + async def forbidden_recalculation(state): + nonlocal recalculation_calls + recalculation_calls += 1 + raise AssertionError("ledger projection must not recalculate prices") + # Keep this sentinel even though the production symbol has been removed: + # the test fails if a future implementation reintroduces and calls it. monkeypatch.setattr( lp_states, "recalculate_lp_state_price_algo_with_micros", - unchanged, + forbidden_recalculation, + raising=False, ) result = asyncio.run( @@ -122,18 +133,20 @@ async def unchanged(state): assert states["POOL-B"].asset1_reserve_micros == 205 assert {state.address for state in result} == {"POOL-A", "POOL-B"} assert [tx.id for tx in repository.projected] == ["TX@POOL-A", "TX@POOL-B"] + assert recalculation_calls == 0 def test_lp_self_transfer_has_zero_projection(monkeypatch) -> None: monkeypatch.setattr( sync_pools, - "get_all_lp_state_addresses", - lambda: {"POOL"}, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, ) raw_transaction = { "id": "TX", "sender": "POOL", "confirmed-round": 123, + "fee": 0, ASSET_TRANSFER_TX: { "asset-id": 7, "amount": 5, @@ -148,11 +161,352 @@ def test_lp_self_transfer_has_zero_projection(monkeypatch) -> None: assert projections == [] +def test_lp_asset_transfer_projects_pool_fee_as_separate_algo_debit( + monkeypatch, +) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "USER", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert [(tx.id, tx.asa_id, tx.delta_amount_micros) for tx in projections] == [ + ("TX@POOL", 7, -5), + ("TX#fee@POOL", 0, -1_000), + ] + + +def test_token_token_pool_fee_is_a_separately_projectable_algo_event( + monkeypatch, +) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 8, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "USER", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert [(tx.id, tx.asa_id, tx.delta_amount_micros) for tx in projections] == [ + ("TX@POOL", 7, -5), + ("TX#fee@POOL", 0, -1_000), + ] + + +def test_lp_payment_projects_amount_and_fee_as_distinct_algo_debits( + monkeypatch, +) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": 1_000, + PAYMENT_TX: { + "amount": 5, + "receiver": "USER", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert [(tx.id, tx.asa_id, tx.delta_amount_micros) for tx in projections] == [ + ("TX@POOL", 0, -5), + ("TX#fee@POOL", 0, -1_000), + ] + + +def test_lp_self_transfer_still_projects_pool_fee(monkeypatch) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "POOL", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert [(tx.id, tx.pool_address, tx.asa_id, tx.delta_amount_micros) for tx in projections] == [ + ("TX#fee@POOL", "POOL", 0, -1_000), + ] + + +def test_unrelated_asset_dust_cannot_block_lp_projection(monkeypatch) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 8, 99})}, + ) + raw_transaction = { + "id": "DUST", + "sender": "ATTACKER", + "confirmed-round": 123, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 666, + "amount": 1, + "receiver": "POOL", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert projections == [] + + +def test_unrelated_clawback_cannot_block_pool_fee_projection(monkeypatch) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 8, 99})}, + ) + raw_transaction = { + "id": "UNRELATED-CLAWBACK", + "sender": "POOL", + "confirmed-round": 123, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 666, + "amount": 1, + "sender": "VICTIM", + "receiver": "ATTACKER", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert [(transaction.id, transaction.asa_id, transaction.delta_amount_micros) for transaction in projections] == [ + ("UNRELATED-CLAWBACK#fee@POOL", 0, -1_000), + ] + + +@pytest.mark.parametrize( + "invalid_fee", + [None, -1, MAX_ALGORAND_UINT + 1, 1.5, True, "1000"], +) +def test_lp_projection_rejects_invalid_algorand_fees( + monkeypatch, + invalid_fee, +) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": invalid_fee, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "USER", + }, + } + + with pytest.raises(ValueError, match="fee must be an Algorand uint64"): + asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + +@pytest.mark.parametrize( + ("field", "invalid_value"), + [ + ("amount", True), + ("amount", -1), + ("amount", MAX_ALGORAND_UINT + 1), + ("amount", "5"), + ("asset-id", True), + ("asset-id", 0), + ("asset-id", MAX_ALGORAND_UINT + 1), + ], +) +def test_lp_projection_rejects_invalid_asset_transfer_uint64_fields( + monkeypatch, + field: str, + invalid_value: object, +) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + transfer = { + "asset-id": 7, + "amount": 5, + "receiver": "USER", + } + transfer[field] = invalid_value + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": 1_000, + ASSET_TRANSFER_TX: transfer, + } + + with pytest.raises(ValueError, match=field): + asyncio.run(sync_pools.process_lp_transactions([raw_transaction])) + + +@pytest.mark.parametrize( + "invalid_round", + [True, -1, MAX_ALGORAND_UINT + 1, 1.5, "123", None], +) +def test_lp_projection_rejects_invalid_confirmed_round_before_negation( + monkeypatch, + invalid_round: object, +) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": invalid_round, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "USER", + }, + } + + with pytest.raises(ValueError, match="confirmed-round"): + asyncio.run(sync_pools.process_lp_transactions([raw_transaction])) + + +def test_lp_sync_module_has_no_raw_balance_price_publisher() -> None: + assert not hasattr(sync_pools, "update_asset_prices") + assert not hasattr(sync_pools, "create_and_update_asset_prices") + + +def test_token_token_snapshot_tracks_operational_algo_without_repricing( + monkeypatch, +) -> None: + state = LpState( + id=1, + address="POOL", + token_id=99, + asset1_id=7, + asset2_id=8, + dex_provider="tinyman", + last_updated_round=122, + asset1_reserve_micros=1, + asset2_reserve_micros=2, + total_tokens_micros=3, + operational_algo_balance_micros=4, + asset1_reserve=1.0, + asset2_reserve=2.0, + total_tokens=3.0, + token_price_algo=12.5, + ) + + async def snapshot(address, *, include_algo): + assert address == "POOL" + assert include_algo is True + return SimpleNamespace( + balances={0: 5_000, 7: 100, 8: 200, 99: 20}, + observed_round=123, + ) + + async def total_supply(asset_id): + assert asset_id == 99 + return 100 + + class FakeRepository: + def replace_snapshot(self, requested_state, *, observed_round): + assert observed_round == 123 + return requested_state + + monkeypatch.setattr(lp_states, "get_address_asset_snapshot", snapshot) + monkeypatch.setattr(lp_states, "get_asset_total_supply", total_supply) + monkeypatch.setattr( + lp_states, + "db", + SimpleNamespace( + lp_transactions=SimpleNamespace(mongodb_collection=object()), + lp_states=SimpleNamespace(mongodb_collection=object()), + ), + ) + monkeypatch.setattr( + lp_states, + "MongoLpProjectionRepository", + lambda **kwargs: FakeRepository(), + ) + + updated = asyncio.run(lp_states.update_lp_state(state)) + + assert state.asset1_reserve_micros == 1 + assert state.asset2_reserve_micros == 2 + assert state.total_tokens_micros == 3 + assert state.operational_algo_balance_micros == 4 + assert updated.asset1_reserve_micros == 100 + assert updated.asset2_reserve_micros == 200 + assert updated.total_tokens_micros == 80 + assert updated.operational_algo_balance_micros == 5_000 + assert updated.token_price_algo == 12.5 + + def test_lp_projection_rejects_clawback_semantics(monkeypatch) -> None: monkeypatch.setattr( sync_pools, - "get_all_lp_state_addresses", - lambda: {"POOL"}, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, ) raw_transaction = { "id": "TX", @@ -175,8 +529,8 @@ def test_lp_projection_rejects_clawback_semantics(monkeypatch) -> None: def test_lp_projection_ignores_unrelated_clawback(monkeypatch) -> None: monkeypatch.setattr( sync_pools, - "get_all_lp_state_addresses", - lambda: {"POOL"}, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, ) raw_transaction = { "id": "TX", diff --git a/tests/unit/test_price_background.py b/tests/unit/test_price_background.py index f5f2a295..f0219905 100644 --- a/tests/unit/test_price_background.py +++ b/tests/unit/test_price_background.py @@ -4,48 +4,42 @@ import pytest from api import background -from flex.data import lp_prices +from flex.data import lp_registry -def test_empty_asset_catalog_still_updates_lp_prices(monkeypatch) -> None: +def _empty_price_database() -> SimpleNamespace: + return SimpleNamespace( + assets=SimpleNamespace(get_all=lambda: []), + asset_prices=SimpleNamespace(get_all=lambda: []), + ) + + +def test_retired_lp_price_flag_cannot_enable_a_publisher(monkeypatch) -> None: monkeypatch.setattr(background.settings, "background_asset_prices_update", True) monkeypatch.setattr(background.settings, "background_lp_prices_update", True) - monkeypatch.setattr( - background, - "db", - SimpleNamespace( - assets=SimpleNamespace(get_all=lambda: []), - asset_prices=SimpleNamespace(get_all=lambda: []), - ), - ) + monkeypatch.setattr(background, "db", _empty_price_database()) monkeypatch.setattr(background, "get_current_round", lambda: 321) async def no_lp_definitions() -> list[dict]: return [] - updated_rounds: list[int] = [] - - async def record_lp_update(current_round: int) -> None: - updated_rounds.append(current_round) - monkeypatch.setattr( background, "get_lp_token_definitions", no_lp_definitions, ) - monkeypatch.setattr(background, "update_lp_token_prices", record_lp_update) one_shot = background.update_asset_prices_background.__wrapped__.__wrapped__ asyncio.run(one_shot()) - assert updated_rounds == [321] + assert not hasattr(background, "update_lp_token_prices") -def test_lp_registry_failure_cannot_overwrite_lp_with_external_price( +def test_lp_registry_failure_keeps_generic_refresh_fail_closed( monkeypatch, ) -> None: monkeypatch.setattr(background.settings, "background_asset_prices_update", True) - monkeypatch.setattr(background.settings, "background_lp_prices_update", True) + monkeypatch.setattr(background.settings, "background_lp_prices_update", False) monkeypatch.setattr( background, "db", @@ -64,11 +58,6 @@ async def failed_lp_registry() -> list[dict]: async def unexpected_asset_refresh(*args, **kwargs): raise AssertionError("regular asset refresh must fail closed") - updated_rounds: list[int] = [] - - async def record_lp_update(current_round: int) -> None: - updated_rounds.append(current_round) - monkeypatch.setattr( background, "get_lp_token_definitions", @@ -79,67 +68,38 @@ async def record_lp_update(current_round: int) -> None: "create_asset_price", unexpected_asset_refresh, ) - monkeypatch.setattr(background, "update_lp_token_prices", record_lp_update) one_shot = background.update_asset_prices_background.__wrapped__.__wrapped__ asyncio.run(one_shot()) - assert updated_rounds == [321] - -def test_lp_price_worker_is_disabled_independently( - monkeypatch, -) -> None: - monkeypatch.setattr(background.settings, "background_asset_prices_update", True) - monkeypatch.setattr(background.settings, "background_lp_prices_update", False) - monkeypatch.setattr( - background, - "db", - SimpleNamespace( - assets=SimpleNamespace(get_all=lambda: []), - asset_prices=SimpleNamespace(get_all=lambda: []), - ), - ) - monkeypatch.setattr(background, "get_current_round", lambda: 321) - - async def no_lp_definitions() -> list[dict]: - return [] - - async def unexpected_lp_update(current_round: int) -> None: - raise AssertionError( - f"disabled LP price worker received round {current_round}", - ) - - monkeypatch.setattr( - background, - "get_lp_token_definitions", - no_lp_definitions, - ) - monkeypatch.setattr( - background, - "update_lp_token_prices", - unexpected_lp_update, - ) - - one_shot = background.update_asset_prices_background.__wrapped__.__wrapped__ - asyncio.run(one_shot()) - - -def test_incomplete_lp_registry_fails_closed(monkeypatch) -> None: - contract = SimpleNamespace( - metadata={ +@pytest.mark.parametrize( + "metadata", + [ + { "stake_token_id": 999, "asset1_id": 7, "asset2_id": 0, }, + { + "stake_token_id": 999, + }, + ], +) +def test_incomplete_lp_registry_fails_closed( + monkeypatch, + metadata: dict, +) -> None: + contract = SimpleNamespace( + metadata=metadata, ) monkeypatch.setattr( - lp_prices, + lp_registry, "get_contracts_by_type", lambda contract_type: [contract], ) monkeypatch.setattr( - lp_prices, + lp_registry, "db", SimpleNamespace( lp_tokens=SimpleNamespace( @@ -150,7 +110,7 @@ def test_incomplete_lp_registry_fails_closed(monkeypatch) -> None: ) with pytest.raises( - lp_prices.LpTokenRegistryError, - match=r"incomplete.*999", + lp_registry.LpTokenRegistryError, + match=r"classification is incomplete.*999", ): - asyncio.run(lp_prices.get_lp_token_definitions.__wrapped__()) + asyncio.run(lp_registry.get_lp_token_definitions.__wrapped__()) diff --git a/tests/unit/test_price_router.py b/tests/unit/test_price_router.py index a312dfcf..78726a1f 100644 --- a/tests/unit/test_price_router.py +++ b/tests/unit/test_price_router.py @@ -1,26 +1,32 @@ import asyncio from datetime import UTC, datetime, timedelta -from decimal import Decimal from types import SimpleNamespace import httpx import pytest from env import settings -from flex.data import lp_prices -from flex.domain.pricing import PriceQuote, PriceSource, PriceUnavailableError +from flex.domain.pricing import PriceSource, PriceUnavailableError from flex.meta_error import MetaError from flex.providers import price_router, vestige from flex.providers.vestige import Price -def _record(*, age_seconds: int, algo: float = 2.0, usd: float = 0.5): +def _record( + *, + age_seconds: int, + algo: float = 2.0, + usd: float = 0.5, + tinyman_algo_pool_id: int | None = None, + source: str = PriceSource.VESTIGE.value, +): observed_at = datetime.now(UTC) - timedelta(seconds=age_seconds) return SimpleNamespace( id=42, price_algo=algo, price_usd=usd, - source=PriceSource.VESTIGE.value, + source=source, + tinyman_algo_pool_id=tinyman_algo_pool_id, observed_at=observed_at, updated=observed_at, ) @@ -127,71 +133,67 @@ def test_provider_outage_uses_only_bounded_stale_database_price(monkeypatch) -> assert result == Price(algo=2.0, usd=0.5) -def test_bounded_stale_quote_preserves_original_provenance(monkeypatch) -> None: - stale_age = settings.asset_prices_ttl + 1 - record = _record(age_seconds=stale_age) - _database(monkeypatch, record) +def test_provider_outage_never_uses_legacy_raw_lp_projection(monkeypatch) -> None: + _database( + monkeypatch, + _record(age_seconds=0, tinyman_algo_pool_id=999), + ) monkeypatch.setattr( vestige, "vestige_full_asset_price_not_cached", _provider_failure, ) - monkeypatch.setattr(price_router, "_asset_price_from_tinyman", _provider_failure) - - quote = asyncio.run(price_router.get_asset_price_quote(42)) + monkeypatch.setattr( + price_router, + "_asset_price_from_tinyman", + _provider_failure, + ) - assert quote.source is PriceSource.VESTIGE - assert quote.observed_at == record.observed_at - assert quote.to_legacy_floats() == (2.0, 0.5) + with pytest.raises(PriceUnavailableError, match="no acceptable fallback"): + asyncio.run(price_router.get_asset_price(42)) -def test_derived_lp_keeps_bounded_stale_dependency_timestamp(monkeypatch) -> None: - record = _record(age_seconds=settings.asset_prices_ttl + 1) - _database(monkeypatch, record) +def test_provider_outage_never_uses_derived_lp_source_without_pool_marker( + monkeypatch, +) -> None: + _database( + monkeypatch, + _record( + age_seconds=0, + source=PriceSource.DERIVED_LP.value, + ), + ) monkeypatch.setattr( vestige, "vestige_full_asset_price_not_cached", _provider_failure, ) - monkeypatch.setattr(price_router, "_asset_price_from_tinyman", _provider_failure) - asset_quote = asyncio.run(price_router.get_asset_price_quote(42)) - algo_quote = PriceQuote.from_raw( - asset_id=0, - algo=1, - usd=Decimal("0.25"), - source=PriceSource.VESTIGE, - stale_after=timedelta(minutes=5), + monkeypatch.setattr( + price_router, + "_asset_price_from_tinyman", + _provider_failure, ) - persisted = [] - async def calculate_with_quote(lp_def): - assert lp_def == {"lp_token_id": 99} - return Decimal("4"), asset_quote + with pytest.raises(PriceUnavailableError, match="no acceptable fallback"): + asyncio.run(price_router.get_asset_price(42)) - async def asset_details(asset_id: int): - assert asset_id == 99 - return SimpleNamespace(name="LP") +def test_bounded_stale_quote_preserves_original_provenance(monkeypatch) -> None: + stale_age = settings.asset_prices_ttl + 1 + record = _record(age_seconds=stale_age) + _database(monkeypatch, record) monkeypatch.setattr( - lp_prices, - "_calculate_lp_token_price_algo_with_quote", - calculate_with_quote, - ) - monkeypatch.setattr(lp_prices, "get_asset_details", asset_details) - monkeypatch.setattr(lp_prices, "_upsert_asset_price", persisted.append) - - updated = asyncio.run( - lp_prices._update_single_lp( - {"lp_token_id": 99}, - algo_quote, - current_round=123, - ), + vestige, + "vestige_full_asset_price_not_cached", + _provider_failure, ) + monkeypatch.setattr(price_router, "_asset_price_from_tinyman", _provider_failure) + + quote = asyncio.run(price_router.get_asset_price_quote(42)) - assert updated is True - assert len(persisted) == 1 - assert persisted[0].observed_at == record.observed_at - assert persisted[0].source == PriceSource.DERIVED_LP.value + assert quote.source is PriceSource.VESTIGE + assert quote.observed_at == record.observed_at + assert quote.to_legacy_floats() == (2.0, 0.5) def test_price_older_than_max_stale_is_rejected(monkeypatch) -> None: diff --git a/tests/unit/test_pricing_domain.py b/tests/unit/test_pricing_domain.py index c75d92de..ea4030a7 100644 --- a/tests/unit/test_pricing_domain.py +++ b/tests/unit/test_pricing_domain.py @@ -7,7 +7,7 @@ import pytest from algosdk.error import IndexerHTTPError -from pymongo.errors import PyMongoError +from pymongo.errors import DuplicateKeyError, PyMongoError from flex.application.price_refresh import ( PriceDataError, @@ -15,9 +15,9 @@ validate_provider_quote, ) from flex.data import asset_prices as asset_price_data -from flex.data import lp_prices as lp_price_data from flex.db.model.priced import AssetPrice from flex.domain.pricing import ( + MAX_OBSERVATION_CLOCK_SKEW, InvalidPriceError, PriceQuote, PriceSource, @@ -160,6 +160,26 @@ def test_provider_boundary_rejects_misattributed_quote() -> None: ) +def test_provider_boundary_rejects_future_dated_quote() -> None: + quote = PriceQuote.from_raw( + asset_id=42, + algo="1", + usd="0.25", + source=PriceSource.VESTIGE, + observed_at=datetime.now(UTC) + MAX_OBSERVATION_CLOCK_SKEW + timedelta(seconds=1), + stale_after=timedelta(minutes=1), + ) + + with pytest.raises(PriceDataError, match="too far in the future"): + validate_provider_quote( + quote, + asset_id=42, + source=PriceSource.VESTIGE, + fresh_for=timedelta(minutes=1), + observed_round=123, + ) + + @pytest.mark.parametrize( "overrides", [ @@ -208,6 +228,29 @@ def test_standalone_freshness_uses_the_same_inclusive_boundary() -> None: ) +def test_future_observation_is_never_treated_as_fresh() -> None: + now = datetime(2026, 1, 1, 12, tzinfo=UTC) + observed_at = now + MAX_OBSERVATION_CLOCK_SKEW + timedelta(microseconds=1) + quote = PriceQuote.from_raw( + asset_id=42, + algo="1", + usd="0.25", + source=PriceSource.VESTIGE, + observed_at=observed_at, + stale_after=timedelta(minutes=5), + ) + + assert ( + is_observation_stale( + observed_at, + fresh_for=timedelta(minutes=5), + now=now, + ) + is True + ) + assert quote.is_stale(now=now) is True + + def _stored_asset_price( *, updated: datetime, @@ -349,6 +392,11 @@ async def fetch_quote(*args, **kwargs) -> PriceQuote: {"observed_at": {"$exists": False}}, {"observed_at": None}, {"observed_at": {"$lte": observed_at}}, + { + "observed_at": { + "$gt": refreshed.updated + MAX_OBSERVATION_CLOCK_SKEW, + } + }, ], } ) @@ -359,6 +407,37 @@ async def fetch_quote(*args, **kwargs) -> PriceQuote: collection.insert_one.assert_not_called() +def test_future_dated_asset_price_is_rejected_before_database_io( + monkeypatch, +) -> None: + future = datetime.now(UTC) + MAX_OBSERVATION_CLOCK_SKEW + timedelta(seconds=1) + price = _stored_asset_price( + updated=future, + observed_at=future, + ) + collection = SimpleNamespace( + update_one=Mock(), + find_one=Mock(), + insert_one=Mock(), + ) + monkeypatch.setattr( + asset_price_data, + "db", + SimpleNamespace( + asset_prices=SimpleNamespace( + mongodb_collection=collection, + ), + ), + ) + + with pytest.raises(InvalidPriceError, match="too far in the future"): + asset_price_data._upsert_asset_price(price) + + collection.update_one.assert_not_called() + collection.find_one.assert_not_called() + collection.insert_one.assert_not_called() + + def test_older_asset_price_observation_cannot_replace_newer_record( monkeypatch, ) -> None: @@ -392,6 +471,45 @@ def test_older_asset_price_observation_cannot_replace_newer_record( collection.insert_one.assert_not_called() +def test_newer_initial_price_retries_after_concurrent_older_insert( + monkeypatch, +) -> None: + observed_at = datetime(2026, 1, 1, 12, tzinfo=UTC) + newer = _stored_asset_price( + updated=observed_at, + observed_at=observed_at, + ) + collection = SimpleNamespace( + update_one=Mock( + side_effect=[ + SimpleNamespace(matched_count=0), + SimpleNamespace(matched_count=1), + ] + ), + find_one=Mock(return_value=None), + insert_one=Mock(side_effect=DuplicateKeyError("concurrent insert")), + ) + monkeypatch.setattr( + asset_price_data, + "db", + SimpleNamespace( + asset_prices=SimpleNamespace( + mongodb_collection=collection, + ), + ), + ) + + persisted = asset_price_data._upsert_asset_price(newer) + + assert persisted is True + assert collection.update_one.call_count == 2 + first_selector, first_update = collection.update_one.call_args_list[0].args + retry_selector, retry_update = collection.update_one.call_args_list[1].args + assert retry_selector == first_selector + assert retry_update == {"$set": first_update["$set"]} + assert collection.update_one.call_args_list[1].kwargs == {"upsert": False} + + def test_batch_refresh_isolates_expected_asset_metadata_failure(monkeypatch) -> None: async def batch_prices(asset_ids: list[int]) -> dict[int, vestige.Price]: return {asset_id: vestige.Price(algo=0.25, usd=1.0) for asset_id in asset_ids} @@ -491,76 +609,3 @@ async def asset_details(asset_id: int): ) assert result == [winner] - - -def test_lp_data_adapter_passes_raw_uint64_values_to_exact_domain_pricer(monkeypatch) -> None: - asset1_id = 7 - lp_token_id = 99 - reserve_micros = 2**53 + 1 - pool_balance_micros = 100 - total_supply_micros = reserve_micros + pool_balance_micros - responses = iter( - [ - { - "params": { - "creator": "FACTORY", - "reserve": "POOL", - "total": total_supply_micros, - "decimals": 8, - }, - }, - { - "assets": [ - {"asset-id": asset1_id, "amount": reserve_micros}, - {"asset-id": lp_token_id, "amount": pool_balance_micros}, - ], - "amount": 0, - }, - ], - ) - run_sync_calls: list[tuple[object, tuple[object, ...]]] = [] - - async def run_sync(func, *args): - run_sync_calls.append((func, args)) - return next(responses) - - async def get_details(asset_id: int): - assert asset_id == asset1_id - return SimpleNamespace(decimals=6) - - async def get_price_quote(asset_id: int): - assert asset_id == asset1_id - return PriceQuote.from_raw( - asset_id=asset1_id, - algo=Decimal("0.5"), - usd=Decimal("0.125"), - source=PriceSource.VESTIGE, - stale_after=timedelta(minutes=5), - ) - - exact_pricer = Mock(return_value=Decimal("1")) - monkeypatch.setattr(lp_price_data, "_run_sync", run_sync) - monkeypatch.setattr(lp_price_data, "get_asset_details", get_details) - monkeypatch.setattr( - lp_price_data.price_router, - "get_asset_price_quote", - get_price_quote, - ) - monkeypatch.setattr(lp_price_data, "calculate_lp_token_price_algo_exact", exact_pricer) - - result = asyncio.run( - lp_price_data.calculate_lp_token_price_algo( - {"lp_token_id": lp_token_id, "asset1_id": asset1_id}, - ), - ) - - assert result == Decimal("1") - assert run_sync_calls[1][1] == ("POOL",) - exact_pricer.assert_called_once_with( - asset1_price_algo=Decimal("0.5"), - asset1_reserve_micros=reserve_micros, - asset1_decimals=6, - total_lp_supply_micros=total_supply_micros, - pool_lp_balance_micros=pool_balance_micros, - lp_token_decimals=8, - ) diff --git a/tests/unit/test_stats_pricing.py b/tests/unit/test_stats_pricing.py new file mode 100644 index 00000000..068ae5ff --- /dev/null +++ b/tests/unit/test_stats_pricing.py @@ -0,0 +1,49 @@ +import asyncio +from datetime import UTC, datetime +from types import SimpleNamespace + +from flex.data import stats +from flex.db.model.pools import PoolType +from flex.db.model.priced import AssetPrice + + +def test_tvl_never_uses_legacy_raw_lp_projection(monkeypatch) -> None: + legacy_price = AssetPrice( + id=42, + name="LEGACY", + price_algo=999.0, + price_usd=999.0, + last_update_round=123, + tinyman_algo_pool_id=777, + source="tinyman", + observed_at=datetime.now(UTC), + ) + pool = SimpleNamespace( + pool_id=1, + stake_token=SimpleNamespace(id=42), + total_staked=10, + ) + requested_asset_ids = [] + + async def safe_price(asset_id: int): + requested_asset_ids.append(asset_id) + return SimpleNamespace(price_usd=2.0) + + monkeypatch.setattr( + stats, + "db", + SimpleNamespace( + pool_states=SimpleNamespace( + get_many=lambda **query: [pool], + ), + asset_prices=SimpleNamespace(get_all=lambda: [legacy_price]), + ), + ) + monkeypatch.setattr(stats, "get_asset_price", safe_price) + + total = asyncio.run( + stats.calculate_total_tvl_usd_for_type(PoolType.FARMING), + ) + + assert total == 20.0 + assert requested_asset_ids == [42] diff --git a/tests/unit/test_tinyman_price_projection.py b/tests/unit/test_tinyman_price_projection.py deleted file mode 100644 index 1340717d..00000000 --- a/tests/unit/test_tinyman_price_projection.py +++ /dev/null @@ -1,99 +0,0 @@ -import asyncio -from datetime import UTC, datetime, timedelta -from types import SimpleNamespace - -import pytest - -from flex.data import tinyman_lps -from flex.domain.pricing import ( - InvalidLiquidityPoolError, - PriceQuote, - PriceSource, -) - - -def _lp_state( - *, - asset_reserve: int = 4_000_000, - observed_at: datetime | None = None, -): - return SimpleNamespace( - id=555, - token_id=99, - asset1_id=7, - asset2_id=0, - is_algo_pool=True, - asset1_reserve_micros=asset_reserve, - asset2_reserve_micros=2_000_000, - total_tokens_micros=2_000_000, - token_price_algo=0, - last_updated_round=123, - last_event_order="00000000000000000123:~", - updated=observed_at or datetime(2026, 1, 1, tzinfo=UTC), - ) - - -def _algo_quote(*, observed_at: datetime | None = None) -> PriceQuote: - return PriceQuote.from_raw( - asset_id=0, - algo=1, - usd="0.5", - source=PriceSource.VESTIGE, - stale_after=timedelta(minutes=5), - observed_at=observed_at, - ) - - -def test_tinyman_projection_persists_validated_provenance(monkeypatch) -> None: - pool_observed_at = datetime(2026, 1, 1, tzinfo=UTC) - state = _lp_state(observed_at=pool_observed_at) - persisted = [] - - async def asset_details(asset_id: int): - assert asset_id == 7 - return SimpleNamespace(decimals=6, name="ASSET") - - monkeypatch.setattr(tinyman_lps, "get_asset_details", asset_details) - monkeypatch.setattr( - tinyman_lps, - "_upsert_asset_price", - persisted.append, - ) - - result = asyncio.run( - tinyman_lps.update_tinyman_algo_asset_price( - state, - algo_quote=_algo_quote( - observed_at=pool_observed_at + timedelta(minutes=1), - ), - ), - ) - - assert state.token_price_algo == 0 - assert persisted == [result] - assert result.price_algo == 0.5 - assert result.price_usd == 0.25 - assert result.source == PriceSource.TINYMAN.value - assert result.observed_at == pool_observed_at - - -def test_tinyman_projection_rejects_empty_reserve_without_writing( - monkeypatch, -) -> None: - state = _lp_state(asset_reserve=0) - persisted = [] - monkeypatch.setattr( - tinyman_lps, - "_upsert_asset_price", - persisted.append, - ) - - with pytest.raises(InvalidLiquidityPoolError, match="must be positive"): - asyncio.run( - tinyman_lps.update_tinyman_algo_asset_price( - state, - algo_quote=_algo_quote(), - ), - ) - - assert persisted == [] From 32077be5ec382c9ddb7f6dfe45e002c37b55a25f Mon Sep 17 00:00:00 2001 From: wackloner Date: Sun, 19 Jul 2026 22:27:01 +0700 Subject: [PATCH 10/10] harden public financial release --- .claude/AGENTS.md | 5 +- .github/workflows/ci.yml | 28 ++ .gitignore | 26 ++ BOARD.md | 15 +- CLAUDE.md | 9 +- Makefile | 10 +- README.md | 17 +- SECURITY.md | 4 + ...audit-architecture-financial-2026-07-19.md | 260 ++++++++---------- 9 files changed, 219 insertions(+), 155 deletions(-) diff --git a/.claude/AGENTS.md b/.claude/AGENTS.md index ae098ecf..d29f6f22 100644 --- a/.claude/AGENTS.md +++ b/.claude/AGENTS.md @@ -30,13 +30,16 @@ work status; it does not override repository safety rules or the current user re scripts unless the user explicitly requests that external action. - Validate on-chain and provider payloads before persistence. - Preserve idempotency for transaction replay and financial background jobs. +- Treat confirmed payouts and complete manifests as terminal states. +- The raw-balance LP publisher is retired. Do not recreate it; token-token fee + ALGO is operational balance, not an economic reserve. - Do not silently substitute zero for unavailable chain state or price data. ## Quality Gate ```bash pipenv verify -pipenv sync --dev +make sync make quality ``` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfc3b5a4..17ead154 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,28 @@ jobs: --exit-code 1 \ "cometa-backend:${COMETA_IMAGE_TAG:-local}" + secret-scan: + name: Secret history + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + - name: Fetch every published ref + run: git fetch --force --prune origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' + - name: Scan reachable Git history + run: | + docker run --rm \ + -v "$PWD:/repo" \ + ghcr.io/trufflesecurity/trufflehog@sha256:59b244249d1a1aef4baa24fe73d3c931616264482580d806d77f6c74d26b3e42 \ + git file:///repo \ + --results=verified,unknown \ + --fail \ + --fail-on-scan-errors \ + --no-update \ + --github-actions + mongo-integration: name: MongoDB financial invariants runs-on: ubuntu-latest @@ -113,14 +135,20 @@ jobs: if: ${{ always() }} needs: - python-matrix + - configuration - mongo-integration + - secret-scan runs-on: ubuntu-latest timeout-minutes: 2 steps: - name: Require Python and MongoDB checks env: MATRIX_RESULT: ${{ needs.python-matrix.result }} + CONFIGURATION_RESULT: ${{ needs.configuration.result }} MONGO_RESULT: ${{ needs.mongo-integration.result }} + SECRET_SCAN_RESULT: ${{ needs.secret-scan.result }} run: | test "$MATRIX_RESULT" = "success" + test "$CONFIGURATION_RESULT" = "success" test "$MONGO_RESULT" = "success" + test "$SECRET_SCAN_RESULT" = "success" diff --git a/.gitignore b/.gitignore index 4302ce41..bc8df320 100644 --- a/.gitignore +++ b/.gitignore @@ -107,6 +107,14 @@ celerybeat.pid .env.* !.env.example .env.main +*.mnemonic +*.seed +*.pem +*.key +*.p12 +*.pfx +*.jks +*.keystore .venv env/ venv/ @@ -114,6 +122,13 @@ ENV/ env.bak/ venv.bak/ +# Database exports and operator backups +dump/ +backups/ +*.bson +*.dump +*.sql.gz + # Spyder project settings .spyderproject .spyproject @@ -242,3 +257,14 @@ test-results/ test_refund.py check_vestige_hack.py sample.py +# Retired credential-bearing operator helper; history is scrubbed separately. +/verify_pool.sh + +# Local operator scratch files; never publish from this checkout. +/analyze_logs.sh +/download_and_analyze_logs.sh +/log_analyzer.py +/docs/impeccable-analysis.md +/docs/pagination-plan.md +/scripts/recover_contracts.py +/scripts/restore_missing.py diff --git a/BOARD.md b/BOARD.md index 778d971e..0a574d51 100644 --- a/BOARD.md +++ b/BOARD.md @@ -8,7 +8,7 @@ - **Statuses**: `todo` | `in_progress` | `blocked` | `done` - **Priorities**: `critical` | `high` | `medium` | `low` - **Tags**: `security` | `backend` | `infra` | `dx` | `arch` | `perf` -- Next available ID: **CB-086** +- Next available ID: **CB-097** ## Active @@ -24,8 +24,19 @@ | CB-081 | Fence expired sync workers | done | critical | backend, arch | A worker cannot commit a financial round after its lease expires, with unit and real-Mongo regressions | | CB-082 | Verify Mongo financial invariants | done | high | backend, infra | CI proves CAS replay, marker repair, BSON promotion, uniqueness, and lease fencing against a disposable pinned MongoDB | | CB-083 | Refresh public engineering docs | done | medium | dx, arch | Runtime commands, Python support, architecture boundaries, API shapes, and cross-project contract are current | -| CB-084 | Disable unverified LP pricing | done | critical | backend, arch | Legacy raw-account-balance LP pricing is independently default-off until DEX economic reserves are verified | +| CB-084 | Disable unverified LP pricing | done | critical | backend, arch | Raw-account-balance publisher is removed; startup purges its legacy rows and every stored-price read rejects them | | CB-085 | Remove repository credibility drift | done | medium | dx, arch | Public claims match verified behavior, local sync selects Python 3.12, and the unrelated EVM sample is removed | +| CB-086 | Reconcile legacy lottery payouts | done | critical | security, backend | Pre-intent lottery draws fail closed until manual reconciliation; new draws use durable payout states | +| CB-087 | Preserve terminal payout states | done | high | backend, arch | Confirmed transfer intents and complete airdrop manifests cannot regress under stale concurrent workers | +| CB-088 | Separate LP ledger from pricing | done | critical | backend, arch | Raw balances never publish prices; fees are replay-safe events and operational ALGO is isolated from reserves | +| CB-089 | Migrate canonical asset supply | done | high | backend, arch | Financial reads trust only Indexer-provenanced base units, migrate atomically, and fail closed on duplicate asset IDs | +| CB-090 | Claim staking lottery entitlements atomically | done | critical | backend, arch | Concurrent and crash-recovery paths converge on one draw generation in real MongoDB | +| CB-091 | Bound outbound signer fees and network | done | critical | security, backend | Suggested and persisted transactions enforce a configured fee ceiling and canonical genesis before network I/O | +| CB-092 | Validate Algorand numeric boundaries | done | high | backend, arch | Indexer events and snapshots reject coercion, negatives, duplicates, and uint64 overflow before persistence | +| CB-093 | Reserve one-of-one lottery inventory | todo | high | backend, arch | A re-enabled lottery atomically reserves NFT inventory and reconciles release/finalization | +| CB-094 | Replace retired LP pricing with verified adapters | todo | high | backend, arch | DEX-specific app state proves economic reserves; donation and excess-balance adversarial tests pass | +| CB-095 | Reject poisoned price chronology | done | high | backend, arch | Future-dated quotes fail before persistence and a valid quote replaces legacy future timestamps | +| CB-096 | Enforce standalone financial indexes | done | high | backend, arch | Operator airdrops and LP discovery require unique immutable business keys before concurrent upserts | ## Completed milestones diff --git a/CLAUDE.md b/CLAUDE.md index 85d255b1..d1dbb6a9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,7 +37,7 @@ docs/ — Architecture decisions, operations, and audit reports ```bash # Local development pipenv verify -pipenv sync --dev +make sync make run # production-equivalent startup, including indexes/workers make run-api # API-only Uvicorn reload; no migrations or workers @@ -67,9 +67,14 @@ scripts/redeploy.sh # pull + rebuild + restart the backend service - Persist maintained Flex financial `uint64` fields through the BSON codecs in `flex/db/bson.py`; do not copy legacy int64/float compatibility shapes - Outbound transfers must persist immutable signed intent before broadcast and reconcile on-chain before completion +- Legacy lottery draws without a matching durable operation remain `reconciliation_required` - LP events must enter through the complete-round preflight and `MongoLpProjectionRepository` +- Keep token-token fee funding in `operational_algo_balance_micros`; never mix it into economic reserves +- Trust `total_supply_micros` only with `total_supply_source=indexer`; otherwise + refetch and persist both fields atomically before a financial supply read - Keep `SYNC_STAKING_POOLS=false` until full Algorand application-group validation is implemented -- Keep `BACKGROUND_LP_PRICES_UPDATE=false` until DEX-specific economic reserves are verified +- Raw LP account balances never publish prices; `BACKGROUND_LP_PRICES_UPDATE` + is a retired compatibility setting and cannot restore the removed publisher - New pricing and transaction invariants belong in pure modules under `flex/domain/` - Run the strict mypy target before changing `core/circuit_breaker.py` or `flex/domain/` diff --git a/Makefile b/Makefile index 2abd4d73..bd64858d 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ PYTHON_LINT_PATHS := \ core/circuit_breaker.py core/cometa.py core/decorators.py core/util.py \ flex/__init__.py flex/api.py flex/application flex/blockchain/asset_transfers.py \ flex/blockchain/contract_state.py flex/blockchain/info.py flex/data/asset_prices.py \ - flex/data/lp_prices.py flex/data/lp_states.py flex/data/pool_state.py \ - flex/data/tinyman_lps.py flex/data/transactions.py \ + flex/data/lp_registry.py flex/data/lp_states.py flex/data/lp_tokens.py flex/data/pool_state.py \ + flex/data/transactions.py \ flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/base_entity.py \ flex/db/classes/bson_uint64.py flex/db/classes/collection_manager.py \ flex/db/indexes.py flex/db/lp_projection.py \ @@ -18,7 +18,7 @@ PYTHON_LINT_PATHS := \ PYTHON_MODERN_PATHS := \ api/background.py api/nft_lottery.py api/wallet.py core/circuit_breaker.py flex/application \ flex/blockchain/asset_transfers.py \ - flex/blockchain/contract_state.py flex/data/asset_prices.py flex/data/lp_prices.py \ + flex/blockchain/contract_state.py flex/data/asset_prices.py flex/data/lp_registry.py flex/data/lp_tokens.py \ flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/bson_uint64.py \ flex/db/lp_projection.py flex/db/sync_coordinator.py \ flex/db/model/airdrop.py flex/db/model/priced.py flex/db/model/transfers.py \ @@ -28,8 +28,8 @@ PYTHON_FORMAT_PATHS := \ api/background.py api/nft_lottery.py api/wallet.py app.py blockchain/indexer.py bot/log.py \ core/circuit_breaker.py core/cometa.py core/util.py \ flex/api.py flex/application flex/blockchain/asset_transfers.py flex/blockchain/contract_state.py flex/blockchain/info.py \ - flex/data/asset_prices.py flex/data/lp_prices.py \ - flex/data/lp_states.py flex/data/pool_state.py flex/data/tinyman_lps.py \ + flex/data/asset_prices.py flex/data/lp_registry.py \ + flex/data/lp_states.py flex/data/lp_tokens.py flex/data/pool_state.py \ flex/data/transactions.py flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/base_entity.py \ flex/db/classes/bson_uint64.py flex/db/classes/collection_manager.py flex/db/indexes.py flex/db/lp_projection.py \ flex/db/sync_coordinator.py flex/db/model/airdrop.py flex/db/model/blockchain.py \ diff --git a/README.md b/README.md index 7170eb45..f1c7a5aa 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,11 @@ MongoDB—into stable, query-oriented API models for the product frontend. | Engineering concern | Implementation | | --- | --- | -| **Financial precision** | Prices use validated `Decimal` value objects; maintained LP projection and sync fields retain the full Algorand `uint64` domain through BSON-safe codecs. | -| **Crash-safe projections** | Scoped event cursors, per-state compare-and-set writes, immutable markers, and a fenced round checkpoint make LP replay convergent across crashes and competing workers. | -| **Replay-safe payouts** | Airdrops and NFT transfers persist immutable signed intent before broadcast, reconcile on-chain outcomes, and allocate integer base units exactly. | +| **Financial precision** | Prices use validated `Decimal` value objects; LP balances, fees, and canonical asset supply retain the full Algorand `uint64` domain through BSON-safe storage. | +| **Crash-safe projections** | Scoped event cursors, fee-specific IDs, per-state compare-and-set writes, immutable markers, and a fenced round checkpoint make LP replay convergent across crashes and competing workers. | +| **Replay-safe payouts** | Airdrops and NFT transfers persist immutable signed intent before broadcast, cap signer fees, bind the configured genesis hash, keep terminal states monotonic, and fail closed on pre-intent lottery history. | | **Resilient price routing** | Vestige and Tinyman payloads are validated with provenance and bounded staleness; retry classification and a guarded Vestige refresh prevent failure storms. | -| **Operational boundaries** | Selected blocking chain calls leave the event loop; deterministic failures use bounded retry; unverified staking, LP projection, and legacy LP pricing paths are disabled by default. | +| **Operational boundaries** | Selected blocking chain calls leave the event loop; deterministic failures use bounded retry; unverified staking is fail-closed and the raw-balance LP price publisher has been removed. | | **Versioned chain decoding** | Reach 0.1.11 state is decoded natively from Algorand with explicit per-version layouts, exact-width integers, and fail-closed schema validation. | | **Supply-chain hardening** | The digest-pinned Alpine image is multi-stage, non-root, and Python-only; CI smoke-tests it and rejects high/critical vulnerabilities or embedded secrets. | @@ -84,8 +84,9 @@ intentionally open items, is in | --- | --- | | Provider quote → stored price | Positive, finite decimal values with source and observation timestamp | | Cached price → API response | Explicit freshness window; expired data is rejected instead of silently relabelled | -| Chain event → LP read model | Full-block preflight, uint64-safe CAS cursor, marker repair, and round fencing | -| Asset payout → Algorand | Persist signed intent first; rebroadcast identical bytes; reconcile before completion | +| Chain event → LP read model | Full-block preflight, fee-aware uint64 ledger, CAS cursor, marker repair, and round fencing | +| Raw LP account balance → price | Prohibited; economic reserves require a verified DEX-specific adapter | +| Asset payout → Algorand | Validate genesis and fee ceiling; persist signed intent first; rebroadcast identical bytes; reconcile before completion | | Selected sync chain SDK → async request path | Bounded executor hand-off | | Permanent provider error → retry loop | Typed classification prevents pointless retries | | Half-open circuit → provider | A single probe prevents a recovery stampede | @@ -153,7 +154,9 @@ This single command runs: CI repeats those checks on Python 3.12 and 3.14 for every pull request and every push to `main`, verifies the lockfile and Compose configuration, builds and smoke-tests the production image, scans it with Trivy, and exercises financial -repository invariants against a digest-pinned MongoDB service. The focused +repository invariants against a digest-pinned MongoDB service. A pinned +TruffleHog gate fetches and scans every published Git ref for verified or unresolved +credentials and feeds the stable required `python` status. The focused coverage ratchet is currently 75%; it measures maintained domain and infrastructure modules rather than presenting a misleading whole-repository number. diff --git a/SECURITY.md b/SECURITY.md index 1d277907..0f458eb2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -33,3 +33,7 @@ replay. Tests must use generated accounts and non-production credentials. Never commit mnemonics, API keys, `.env` files, database exports, recovery artifacts, or unredacted logs. If a secret reaches Git history, revoke or rotate it immediately; deleting the current file is not sufficient. + +GitHub secret scanning and push protection are enabled. CI additionally scans +the full reachable history with a pinned detector; do not bypass that gate with +an allowlist unless the value is a documented, non-secret test fixture. diff --git a/docs/audit/01-audit-architecture-financial-2026-07-19.md b/docs/audit/01-audit-architecture-financial-2026-07-19.md index 191cf19e..9fff7df6 100644 --- a/docs/audit/01-audit-architecture-financial-2026-07-19.md +++ b/docs/audit/01-audit-architecture-financial-2026-07-19.md @@ -1,183 +1,165 @@ -# Внутренний мультиагентный аудит архитектуры и финансовой корректности +# Мультиагентный аудит архитектуры и финансовой корректности **Дата:** 2026-07-19 - -**База аудита:** `main@2c1cdad` - -**Ветка исправлений:** `audit/financial-correctness` +**База:** состояние `main` до ветки `audit/financial-correctness`; история +репозитория затем была очищена, поэтому старые SHA намеренно не используются. ## Резюме -Аудит проводился с позиции production fintech backend, а не косметической -подготовки GitHub-профиля. На исходной базе были риски повторной выплаты, -потери точности и некорректного replay при конкурирующих воркерах. На ветке -исправлений денежные операции переведены на integer base units, устойчивые -business keys, immutable intents и compare-and-set проекции. - -## Executive summary - -This internal review used separate agent roles for financial correctness, -MongoDB concurrency, CI and supply-chain posture, security and history, and -public API documentation. The branch replaces unsafe payout and LP state -transitions with durable intents, exact base-unit allocation, conditional -writes, fencing, and real-Mongo race tests. It deliberately keeps unverified -staking projection and legacy LP pricing disabled. Remaining authority, -async-persistence, and runtime-hardening work is listed below instead of hidden -behind a maturity score. - -Качественная сводка критических границ: - -| Область | Исходный риск | Текущий контроль | -| --- | --- | --- | -| Финансовая корректность | double-pay и float allocation | replay-safe выплаты, exact allocation, fail-closed legacy | -| Конкурентность и recovery | blind RMW и недоказанный replay | CAS, marker repair, round leases, реальные Mongo-тесты | -| Security boundaries | смешанные read/signing полномочия | секреты вне кода; auth/signing вынесены в следующий milestone | -| Инженерная проверяемость | mocks не доказывали BSON/Mongo | Python 3.12/3.14, strict typing, 262 теста, Mongo integration CI | - -MongoDB гарантирует атомарность одной операции над одним документом, поэтому -проектор строится вокруг conditional update, а не blind read-modify-write. -Междокументная одновременная видимость остаётся отдельной задачей для -replica-set transactions. См. [MongoDB atomicity](https://www.mongodb.com/docs/manual/core/write-operations-atomicity/) +Аудит проводился как review production fintech backend, а не косметическая +подготовка профиля. Независимые потоки проверяли выплаты, MongoDB concurrency, +LP accounting, price provenance, Algorand boundaries, supply chain, Git history, +CI и публичную документацию. Повторяющиеся находки перепроверялись тестами и +исправлялись только после воспроизведения. + +Ключевой результат: денежные операции используют integer base units, +неизменяемые business IDs, persisted signed intents и одно-документные CAS. +Непроверенные источники цены и staking projection остаются fail-closed. + +| Граница | Контроль | +| --- | --- | +| Выплата | exact allocation, durable intent, bounded signer fee, on-chain reconciliation | +| Staking draw | CAS entitlement, одна generation, crash-repair того же draw ID | +| LP ledger | strict uint64 input, ordered cursor, marker repair, fenced round lease | +| Цена | provenance + freshness + clock-skew guard; raw pool balances запрещены | +| Проверка | Python 3.12/3.14, 372 fast tests, 26 real-Mongo tests, 83.19% focused coverage | + +MongoDB гарантирует атомарность одной операции над одним документом; поэтому +денежные инварианты размещены внутри одного CAS aggregate, а не blind +read-modify-write. Междокументная одновременная видимость потребует replica-set +transactions. См. [atomicity](https://www.mongodb.com/docs/manual/core/write-operations-atomicity/) и [transactions](https://www.mongodb.com/docs/manual/core/transactions/). ## Ранжированные находки -### 1. Critical — повторная выплата и неточное распределение airdrop — исправлено +### 1. Critical — секрет оставался в достижимой Git history — исправлено в репозитории -**Почему:** сбой после broadcast, но до записи результата, позволял повторно -отправить актив; float-доли не гарантировали сохранение целого бюджета. +**Почему:** удаление файла из HEAD не отзывает значение и не удаляет старый +blob; его можно восстановить из любого достижимого commit. -**Исправление:** immutable signed intent сохраняется до broadcast и сверяется -по `operation_id`; неопределённый результат reconciled по txid -(`flex/application/asset_transfers.py:179-298`). Airdrop резервирует неизменяемый -manifest до первой отправки (`flex/tools/airdrop.py:332-405`), а largest-remainder -allocation сохраняет бюджет до последней base unit -(`flex/domain/allocation.py:36-80`). +**Исправление:** чувствительные исторические пути удалены из всех публикуемых +refs через `git-filter-repo`; old-object и fresh-clone проверки входят в +процедуру публикации. CI сканирует все публикуемые refs digest-pinned +TruffleHog (`.github/workflows/ci.yml:84`). Ротация ранее использованного +credential остаётся обязательным внешним действием владельца. -### 2. Critical — LP double-apply, stale-read race и истёкший lease — исправлено +### 2. Critical — double-pay window, float allocation и signer fee — исправлено -**Почему:** blind read-modify-write терял обновления; конкурентный replay мог -принять свежий marker за corruption из-за старого snapshot; истёкший worker -мог завершить round. +**Почему:** crash после broadcast до Mongo update допускал повторную отправку; +float shares не сохраняли бюджет; доверенный Algod мог предложить чрезмерную +комиссию. -**Исправление:** per-state CAS cursor, marker-last recovery и повторное чтение -при конкурентном marker (`flex/db/lp_projection.py:53-149`). Завершение round -требует неистёкший lease (`flex/db/sync_coordinator.py:63-100`). Управляемая -гонка на настоящем Mongo доказывает exactly-once delta -(`tests/integration/test_mongo_financial_projection.py:130-155`). +**Исправление:** immutable signed intent сохраняется до первого broadcast и +reconciled по txid (`flex/application/asset_transfers.py:227`). Airdrop заранее +фиксирует полный manifest, а largest-remainder allocation сохраняет каждую base +unit. Confirmed intent и complete manifest терминальны. Gateway до подписи и +повторно перед broadcast проверяет genesis, signature, lease и configured fee +floor/ceiling (`flex/blockchain/asset_transfers.py:50`). Algorand minimum fee описан в +[официальной документации](https://dev.algorand.co/concepts/transactions/fees/). -### 3. Critical — LP price manipulation через raw account balance — исправлено +### 3. Critical — staking lottery создавала две независимые liabilities — исправлено -**Почему:** donation или protocol excess на адресе пула мог попасть в -«экономический резерв» и исказить цену. +**Почему:** прежний `read recent draws → insert` позволял двум workers создать +разные draw IDs; idempotency выплаты не объединяет разные business operations. -**Исправление:** legacy worker отделён от обычного price refresh и default-off -через `BACKGROUND_LP_PRICES_UPDATE=false` (`env.py:57-64`, -`api/background.py:243-251`). Включать только после DEX-specific проверки -app state и economic reserves. +**Исправление:** один entitlement на `(lottery_name, wallet)` атомарно меняет +`next_eligible_at`, `generation` и active draw. Crash recovery продолжает тот же +draw и replay prize selection (`api/nft_lottery.py:275`). Гонка 32 вызовов и crash +между entitlement/draw writes проверены на настоящем MongoDB. One-of-one NFT +inventory ещё требует отдельной атомарной reservation, поэтому публичные +lottery routes остаются disabled. -### 4. High — staking classifier принимал непроверенные переводы — mitigated +### 4. Critical — LP replay мог потерять или повторить баланс — исправлено -**Почему:** перевод рядом с application call не доказывает stake; без проверки -полной transaction group можно создать ложное состояние. +**Почему:** blind RMW, stale snapshots и истёкший worker нарушали exactly-once +projection. -**Исправление сейчас:** `SYNC_STAKING_POOLS=false`, а попытка включения -завершается fail-closed (`flex/sync_pools.py:351-367`). **Следующий фикс:** -типизированный parser полной Algorand group, проверка app ID, selector, -sender/receiver, asset и group order, затем adversarial fixtures. +**Исправление:** per-state cursor CAS, marker-last repair и re-read после +конкурентного marker (`flex/db/lp_projection.py:52`). Round commit fenced +неистёкшим lease. Fee pool-sender — отдельное replay-safe ALGO событие; +token-token operational ALGO не смешивается с economic reserves. Indexer +amounts, IDs, rounds, duplicates и snapshots проверяются до negation и Mongo +write (`flex/sync_pools.py:78`, `flex/data/lp_states.py:113`). -### 5. High — browser-visible shared key не является авторизацией — открыто +### 5. Critical — raw account balance мог манипулировать LP price — исправлено -**Почему:** `X-API-Key` сравнивается корректно, но один общий клиентский token -не подтверждает пользователя и не разделяет права -(`core/auth.py:8-13`, `app.py:323-369`). +**Почему:** donation, minimum-balance funding или protocol excess не являются +экономическими reserves DEX. -**Конкретный фикс:** registration авторизовать wallet-signature challenge с -nonce, expiry и replay table; `/contracts/refresh-cache` оставить только -server-to-server роли с отдельным secret и audit log. До этого считать текущий -token compatibility/rate-control механизмом, не security boundary. +**Исправление:** LP projector стал ledger-only; raw-balance publisher удалён. +Startup очищает обе legacy provenance signatures, а readers независимо их +отклоняют. LP registry прекращает весь refresh, если не классифицирован хотя бы +один farm stake token. Provider observation за пределами clock-skew budget +отклоняется до записи; уже сохранённое далёкое future value считается invalid +и заменяется корректной котировкой. Новый источник допускается только после +DEX-specific app state verification. Canonical supply доверяется лишь с +`total_supply_source=indexer`. -### 6. High — blocking persistence остаётся в async routes — открыто +### 6. High — shared browser key не является пользовательской авторизацией — открыто -**Почему:** sync PyMongo/provider вызовы в event loop увеличивают tail latency -всех запросов при деградации Mongo или DEX (`app.py:418-428`, -`app.py:529-535`, `app.py:559-561`). +**Почему:** один `X-API-Key` не доказывает wallet ownership и не разделяет роли +(`core/auth.py:8`, `app.py:323`). -**Конкретный фикс:** ввести async repository ports с timeout/cancellation; -переходно — `asyncio.to_thread` вокруг целого repository call и -thread-safe cache, плюс saturation/load test. +**Конкретный фикс:** wallet-signature challenge с nonce, expiry и replay table; +server-to-server maintenance role с отдельным secret и audit log. До этого +shared key считается compatibility/rate-control механизмом. -### 7. High — Mongo invariants раньше проверялись только fake-коллекциями — исправлено +### 7. High — blocking persistence остаётся в async routes — открыто -**Почему:** mocks не проверяют BSON numeric comparison, unique-index races, -`$inc` promotion и реальные `find_one_and_update` semantics. +**Почему:** sync PyMongo/provider calls в event loop увеличивают tail latency +всех запросов при деградации Mongo или DEX (`app.py:418`, `app.py:529`). -**Исправление:** отдельный digest-pinned MongoDB CI job -(`.github/workflows/ci.yml:84-109`) проверяет concurrent CAS, marker repair, -legacy int64→Decimal128, `uint64` max, fail-closed duplicates и fencing -(`tests/integration/test_mongo_financial_projection.py:130-336`). Стабильный -required context `python` агрегирует matrix и Mongo job -(`.github/workflows/ci.yml:111-126`). +**Конкретный фикс:** async repository ports с timeout/cancellation; переходно — +`asyncio.to_thread` вокруг целого repository call и saturation test. -### 8. Medium — mutable stateful Docker images — открыто +### 8. High — legacy staking classifier не доказывает transaction group — mitigated -**Почему:** `mongo` и `algorand/algod:latest` могут поменять major/runtime при -обычном rebuild поверх persistent volumes (`docker-compose.yml:29-61`). +**Почему:** соседний transfer без проверки app ID, selector и group order не +доказывает stake. -**Конкретный фикс:** после проверки реального VPS зафиксировать оба образа как -`tag@sha256`, описать backup/restore и downgrade, а CI должен отклонять bare -tags и `latest`. Не подменять production digest без data-format rehearsal. +**Текущий контроль:** `SYNC_STAKING_POOLS=false`, а включение отклоняется. +Следующий фикс — типизированный parser полной Algorand group и adversarial +fixtures. Это отдельно от исправленного lottery entitlement. -### 9. Medium — container smoke не запускает production entrypoint — открыто +### 9. High — mocks не доказывали Mongo/BSON invariants — исправлено -**Почему:** CI заменяет entrypoint на shell и импортирует `app`, поэтому не -доказывает запуск `scripts/run.sh`, Uvicorn, healthcheck и graceful shutdown -(`.github/workflows/ci.yml:55-70`). +**Почему:** fake collections не воспроизводят Decimal128 comparison, unique +index races, `$inc` promotion и `find_one_and_update`. -**Конкретный фикс:** поднять disposable Mongo, запустить образ штатно с -безопасными feature flags, дождаться healthy, запросить `/status`, затем -проверить SIGTERM и вывести logs при сбое. +**Исправление:** digest-pinned MongoDB CI job проверяет 26 integration scenarios: +CAS replay, marker repair, uint64 max, legacy int64 promotion, terminal payout +races, uniqueness, staking entitlement и lease fencing +(`.github/workflows/ci.yml:106`). Stable `python` context агрегирует Python +matrix, container configuration/image scan, Mongo и all-ref secret scan +(`.github/workflows/ci.yml:133`). -### 10. Medium — contract registration не атомарен по business key — открыто +### 10. Medium — runtime/container proof остаётся неполным — открыто -**Почему:** check-then-insert допускает конкурентные дубликаты, а notification -после записи является незарегистрированным side effect -(`app.py:327-365`, `core/db/contracts.py:16-25`). +**Почему:** production smoke заменяет entrypoint shell-командой, а stateful +Mongo/Algod image rollout требует отдельной data-format rehearsal. -**Конкретный фикс:** unique index по contract `id`, atomic upsert с -immutable-field conflict check и transactional outbox для уведомления. +**Конкретный фикс:** disposable stack должен запустить штатный entrypoint, +дождаться health, проверить `/status`, SIGTERM и logs. Production digests +фиксировать только после backup/restore и downgrade rehearsal. ## Milestones -1. **M0 — money safety (готово):** findings 1–3 исправлены; finding 4 - переведён в fail-closed. Добавлены NFT idempotency - (`api/wallet.py:14-37`), on-chain reconciliation и regressions. -2. **M1 — persistence proof (готово):** finding 7, Python matrix, BSON boundary, - stable required check. Production остаётся на 3.12; 3.14 — compatibility - gate. Python 3.14.6 является актуальным maintenance release - ([Python.org](https://www.python.org/downloads/release/python-3146/)). -3. **M2 — authority and atomic workflows (следующий):** findings 5 и 10. -4. **M3 — runtime hardening:** findings 6, 8 и 9; затем SLO/metrics и controlled - deploy rehearsal. - -## Вклад независимых агентов - -- три financial-review потока независимо подтвердили payout/LP классы ошибок; -- adversarial Mongo/BSON review воспроизвёл stale-read race, пропущенный - первоначальным concurrency-тестом; -- CI/GitHub review обнаружил drift обязательного status context после matrix; -- dependency/container review проверил lock, image posture и runtime smoke; -- docs/API review сравнил публичные обещания с фактическими route shapes и - cross-project consumer contract. - -Форматирование и стиль намеренно не включались в findings: их обеспечивает -Ruff. Приоритет аудита — correctness, security, data integrity и доказуемое -recovery-поведение. +1. **M0 — money safety (готово):** findings 2–5; точные суммы, terminal states, + fee/network policy, CAS entitlements, strict uint64 boundaries. +2. **M1 — proof and history (готово в коде):** findings 1 и 9; Python + 3.12/3.14, real Mongo, immutable scanner, clean-history procedure. +3. **M2 — authority (следующий):** finding 6, atomic contract registration, + transactional outbox и one-of-one NFT inventory. +4. **M3 — runtime (следующий):** findings 7, 8 и 10; затем SLO, metrics и + controlled deploy rehearsal. + +Python 3.14.6 проверяется как forward-compatibility gate и является текущим +maintenance release ([Python.org](https://www.python.org/downloads/release/python-3146/)); +production-equivalent environment остаётся на Python 3.12. ## Воспроизводимость ```bash -git log --oneline 2c1cdad..HEAD make sync make quality @@ -186,6 +168,8 @@ MONGODB_TEST_URI=mongodb://127.0.0.1:27017 \ pipenv run pytest tests/integration -m integration -v ``` -`make quality` проверен в чистом окружении Python 3.14.6; -production-equivalent `make sync` явно выбирает Python 3.12. CI повторяет обе -версии и включает real-Mongo job в стабильный required context `python`. +Отдельные чистые environments Python 3.12.8 и 3.14.6 дали одинаковый результат: +372 passed, 26 integration skipped. Все 26 integration tests прошли на +standalone MongoDB отдельно. Форматирование не включалось в findings: его +обеспечивает Ruff; review приоритизировал correctness, security, data integrity +и recovery semantics.