diff --git a/.claude/AGENTS.md b/.claude/AGENTS.md index 3cac4d4e..d29f6f22 100644 --- a/.claude/AGENTS.md +++ b/.claude/AGENTS.md @@ -30,19 +30,32 @@ work status; it does not override repository safety rules or the current user re scripts unless the user explicitly requests that external action. - Validate on-chain and provider payloads before persistence. - Preserve idempotency for transaction replay and financial background jobs. +- Treat confirmed payouts and complete manifests as terminal states. +- The raw-balance LP publisher is retired. Do not recreate it; token-token fee + ALGO is operational balance, not an economic reserve. - Do not silently substitute zero for unavailable chain state or price data. ## Quality Gate ```bash pipenv verify -pipenv sync --dev +make sync make quality ``` CI validates Compose but does not certify the image as immutable-deploy ready: -the private Node sidecar package-auth blocker is documented in `README.md`. Keep the -focused lint, type, and coverage ratchets honest. +it runs the full quality gate on Python 3.12 and 3.14, builds the digest-pinned +Python-only image, smoke-tests the non-root runtime, scans it with Trivy, and +checks financial persistence invariants against a disposable MongoDB service. +Keep the focused lint, type, and coverage ratchets honest. + +`make run` uses the production-equivalent `python app.py` entrypoint, including +critical indexes, migrations only when `MIGRATE=true`, and configured workers. +Use `make run-api` only for API-only hot reload. + +Financial projector design and known standalone-Mongo limits are documented in +`docs/architecture/lp-projection.md`; outbound payout recovery is documented in +`docs/architecture/outbound-asset-transfers.md`. ## Cross-Project Changes diff --git a/.env.example b/.env.example index c192fa84..1722a701 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,7 @@ REKEYED_MNEMONIC= ALGOD_ADDRESS=http://algod:8080 ALGOD_TOKEN= ALGO_INDEXER_ADDRESS=https://mainnet-idx.algonode.cloud +OUTBOUND_ASSET_TRANSFER_MAX_FEE_MICROALGOS=1000 # API authentication API_PASSWORD= @@ -21,6 +22,11 @@ MONGODB_USERNAME= MONGODB_PASSWORD= NEW_DB_NAME=cometa-updated +# Financial projectors are opt-in until their chain classifiers are verified. +SYNC_NEW_POOLS=true +SYNC_LIQUIDITY_POOLS=false +SYNC_STAKING_POOLS=false + # Notifications # Syntactically valid, non-secret local placeholder; replace in production. TELEGRAM_BOT_API_TOKEN=123456:test-token @@ -32,5 +38,7 @@ FARM_CREATION_FEE=0 FARM_FLAT_ALGO_CREATION_FEE=0 # Pricing resilience +BACKGROUND_ASSET_PRICES_UPDATE=true +BACKGROUND_LP_PRICES_UPDATE=false ASSET_PRICES_TTL=120 ASSET_PRICES_MAX_STALE=3600 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e8b02cd..17ead154 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,18 +13,23 @@ permissions: contents: read jobs: - python: + python-matrix: + name: Python ${{ matrix.python-version }} runs-on: ubuntu-latest timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.14"] steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: - python-version: "3.12" + python-version: ${{ matrix.python-version }} cache: pipenv - run: python -m pip install --disable-pip-version-check pipenv==2024.4.0 - run: pipenv verify - - run: pipenv sync --dev + - run: pipenv sync --dev --python "${{ matrix.python-version }}" - name: Lint maintained modules run: make lint - name: Check formatting @@ -75,3 +80,75 @@ jobs: --severity HIGH,CRITICAL \ --exit-code 1 \ "cometa-backend:${COMETA_IMAGE_TAG:-local}" + + secret-scan: + name: Secret history + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + - name: Fetch every published ref + run: git fetch --force --prune origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' + - name: Scan reachable Git history + run: | + docker run --rm \ + -v "$PWD:/repo" \ + ghcr.io/trufflesecurity/trufflehog@sha256:59b244249d1a1aef4baa24fe73d3c931616264482580d806d77f6c74d26b3e42 \ + git file:///repo \ + --results=verified,unknown \ + --fail \ + --fail-on-scan-errors \ + --no-update \ + --github-actions + + mongo-integration: + name: MongoDB financial invariants + runs-on: ubuntu-latest + timeout-minutes: 10 + services: + mongodb: + image: mongo:7.0.28@sha256:4510cf3d7050003e958745adb25d2deb3fb907430716162d9cc1a92eda2a6047 + ports: + - 27017:27017 + options: >- + --health-cmd "mongosh --quiet --eval 'quit(db.adminCommand(\"ping\").ok ? 0 : 2)'" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + env: + MONGODB_TEST_URI: mongodb://127.0.0.1:27017 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + cache: pipenv + - run: python -m pip install --disable-pip-version-check pipenv==2024.4.0 + - run: pipenv sync --dev --python "3.12" + - name: Test financial repositories against MongoDB + run: pipenv run pytest tests/integration -m integration -v + + python: + name: python + if: ${{ always() }} + needs: + - python-matrix + - configuration + - mongo-integration + - secret-scan + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: Require Python and MongoDB checks + env: + MATRIX_RESULT: ${{ needs.python-matrix.result }} + CONFIGURATION_RESULT: ${{ needs.configuration.result }} + MONGO_RESULT: ${{ needs.mongo-integration.result }} + SECRET_SCAN_RESULT: ${{ needs.secret-scan.result }} + run: | + test "$MATRIX_RESULT" = "success" + test "$CONFIGURATION_RESULT" = "success" + test "$MONGO_RESULT" = "success" + test "$SECRET_SCAN_RESULT" = "success" diff --git a/.gitignore b/.gitignore index 4302ce41..bc8df320 100644 --- a/.gitignore +++ b/.gitignore @@ -107,6 +107,14 @@ celerybeat.pid .env.* !.env.example .env.main +*.mnemonic +*.seed +*.pem +*.key +*.p12 +*.pfx +*.jks +*.keystore .venv env/ venv/ @@ -114,6 +122,13 @@ ENV/ env.bak/ venv.bak/ +# Database exports and operator backups +dump/ +backups/ +*.bson +*.dump +*.sql.gz + # Spyder project settings .spyderproject .spyproject @@ -242,3 +257,14 @@ test-results/ test_refund.py check_vestige_hack.py sample.py +# Retired credential-bearing operator helper; history is scrubbed separately. +/verify_pool.sh + +# Local operator scratch files; never publish from this checkout. +/analyze_logs.sh +/download_and_analyze_logs.sh +/log_analyzer.py +/docs/impeccable-analysis.md +/docs/pagination-plan.md +/scripts/recover_contracts.py +/scripts/restore_missing.py diff --git a/BOARD.md b/BOARD.md index 359197ba..0a574d51 100644 --- a/BOARD.md +++ b/BOARD.md @@ -1,6 +1,6 @@ # Cometa Backend — Task Board -> Last updated: 2026-07-18 +> Last updated: 2026-07-19 ## Conventions @@ -8,16 +8,35 @@ - **Statuses**: `todo` | `in_progress` | `blocked` | `done` - **Priorities**: `critical` | `high` | `medium` | `low` - **Tags**: `security` | `backend` | `infra` | `dx` | `arch` | `perf` -- Next available ID: **CB-078** +- Next available ID: **CB-097** ## Active | ID | Task | Status | Priority | Tags | Definition of done | | --- | --- | --- | --- | --- | --- | | CB-073 | Public repository hardening | in_progress | critical | security, dx | Credentials rotated, history sanitized, secret protection enabled, clean-clone scan passes | -| CB-074 | Atomic event projection | todo | critical | backend, arch | Crash-safe inbox/projector with duplicate, replay, and recovery tests | +| CB-074 | Atomic event projection | in_progress | critical | backend, arch | LP projector is crash-safe; replace the disabled legacy staking projector with verified grouped events and recovery tests | | CB-075 | Isolate transaction signing | todo | high | security, arch | Read-only API boundary; authenticated policy-limited signing service | | CB-076 | Async persistence boundary | todo | high | backend, perf | Storage outages cannot block the event loop; timeouts and readiness covered | +| CB-078 | Replay-safe outbound asset payouts | done | critical | security, backend, arch | Exact allocations, immutable airdrop manifests, persisted signed intents, on-chain reconciliation, and regression tests | +| CB-079 | Crash-safe LP projection | done | critical | backend, arch | Decimal128 balances, ordered per-state CAS cursor, fenced round checkpoint, snapshot guards, and crash/concurrency tests | +| CB-080 | Financial read-model regressions | done | medium | backend | Reward decimals use the reward asset and request ordering never mutates cached contract state | +| CB-081 | Fence expired sync workers | done | critical | backend, arch | A worker cannot commit a financial round after its lease expires, with unit and real-Mongo regressions | +| CB-082 | Verify Mongo financial invariants | done | high | backend, infra | CI proves CAS replay, marker repair, BSON promotion, uniqueness, and lease fencing against a disposable pinned MongoDB | +| CB-083 | Refresh public engineering docs | done | medium | dx, arch | Runtime commands, Python support, architecture boundaries, API shapes, and cross-project contract are current | +| CB-084 | Disable unverified LP pricing | done | critical | backend, arch | Raw-account-balance publisher is removed; startup purges its legacy rows and every stored-price read rejects them | +| CB-085 | Remove repository credibility drift | done | medium | dx, arch | Public claims match verified behavior, local sync selects Python 3.12, and the unrelated EVM sample is removed | +| CB-086 | Reconcile legacy lottery payouts | done | critical | security, backend | Pre-intent lottery draws fail closed until manual reconciliation; new draws use durable payout states | +| CB-087 | Preserve terminal payout states | done | high | backend, arch | Confirmed transfer intents and complete airdrop manifests cannot regress under stale concurrent workers | +| CB-088 | Separate LP ledger from pricing | done | critical | backend, arch | Raw balances never publish prices; fees are replay-safe events and operational ALGO is isolated from reserves | +| CB-089 | Migrate canonical asset supply | done | high | backend, arch | Financial reads trust only Indexer-provenanced base units, migrate atomically, and fail closed on duplicate asset IDs | +| CB-090 | Claim staking lottery entitlements atomically | done | critical | backend, arch | Concurrent and crash-recovery paths converge on one draw generation in real MongoDB | +| CB-091 | Bound outbound signer fees and network | done | critical | security, backend | Suggested and persisted transactions enforce a configured fee ceiling and canonical genesis before network I/O | +| CB-092 | Validate Algorand numeric boundaries | done | high | backend, arch | Indexer events and snapshots reject coercion, negatives, duplicates, and uint64 overflow before persistence | +| CB-093 | Reserve one-of-one lottery inventory | todo | high | backend, arch | A re-enabled lottery atomically reserves NFT inventory and reconciles release/finalization | +| CB-094 | Replace retired LP pricing with verified adapters | todo | high | backend, arch | DEX-specific app state proves economic reserves; donation and excess-balance adversarial tests pass | +| CB-095 | Reject poisoned price chronology | done | high | backend, arch | Future-dated quotes fail before persistence and a valid quote replaces legacy future timestamps | +| CB-096 | Enforce standalone financial indexes | done | high | backend, arch | Operator airdrops and LP discovery require unique immutable business keys before concurrent upserts | ## Completed milestones @@ -27,8 +46,9 @@ | Precision-safe pricing | done | Decimal observations, provenance, freshness policy, guarded legacy boundary | | Provider resilience | done | Typed fallback errors, bounded stale data, retry classification, circuit breaker | | Replay identity | done | Deterministic nested event IDs and collection-level uniqueness constraints | +| LP financial ledger | done | Marker-gap recovery, uint64-safe BSON operations, full-block preflight, snapshot coverage guards, and fenced round CAS | | Container baseline | done | Digest-pinned Alpine base, multi-stage non-root runtime, healthcheck, image exclusions, Trivy CI gate | -| API hardening | done | Fail-closed header authentication, trusted hosts, explicit CORS policy, bounded LP/asset/wallet requests | +| API request hardening | done | Fail-closed configured header checks, trusted hosts, explicit CORS policy, bounded selectors and wallet expansion | | Native Reach decoding | done | Versioned global/local codecs, exact-width integers, deterministic layout tests, no private npm runtime | | Legacy runtime removal | done | CB-077: Node/Reach sidecar and production source bind mount removed | diff --git a/CLAUDE.md b/CLAUDE.md index 145386d5..d1dbb6a9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ Backend for Cometa — an Algorand DeFi platform handling liquidity pools, token ## Stack -- **Language**: Python 3.12, Pipenv +- **Language**: Python 3.12 production runtime; Python 3.14 compatibility CI; Pipenv - **Framework**: FastAPI + Uvicorn - **Database**: MongoDB (pymongo) - **Cache**: process-local TTL caches (Redis migration is roadmap work) @@ -16,19 +16,20 @@ Backend for Cometa — an Algorand DeFi platform handling liquidity pools, token ## Project Structure ``` -app.py — FastAPI application, routes, startup +app.py — FastAPI composition, routes, startup, and workers env.py — Settings via pydantic-settings (from .env) -api/ — Core API: background tasks, DB models, stats, swaps, wallets, NFT lottery -blockchain/ — Algorand node/indexer interaction utilities -core/ — Shared core logic -dexes/ — DEX integrations (HumbleSwap, Vestige, etc.) -flex/ — Flex module (API + data) +api/ — Product API, background work, wallets, and disabled lottery surface +blockchain/ — Algorand node/indexer adapters +core/ — Shared authentication, persistence, and resilience +dexes/ — DEX-specific integrations +flex/application/ — Financial use-case orchestration +flex/domain/ — Pure allocation, pricing, projection, and identity rules +flex/db/ — Mongo models, BSON codecs, repositories, and indexes +flex/tools/ — Operator tools such as manifest-driven airdrops bot/ — Telegram bot logic -farcaster/ — Farcaster integration -scripts/ — Deployment & management shell scripts -airdrop/ — Airdrop tooling -marketplaces/ — NFT marketplace integrations -metapunks/ — MetaPunks-specific logic +scripts/ — Deployment and management shell scripts +tests/ — Unit tests plus opt-in real-service integration tests +docs/ — Architecture decisions, operations, and audit reports ``` ## Key Commands @@ -36,8 +37,9 @@ metapunks/ — MetaPunks-specific logic ```bash # Local development pipenv verify -pipenv sync --dev -pipenv run uvicorn app:app --reload --port 8000 +make sync +make run # production-equivalent startup, including indexes/workers +make run-api # API-only Uvicorn reload; no migrations or workers # Quality gate make quality @@ -57,9 +59,22 @@ scripts/redeploy.sh # pull + rebuild + restart the backend service - Background tasks in `api/background.py` — use exponential backoff for retries - Decode only explicitly supported Reach contract versions in `flex/blockchain/contract_state.py` - Route registration and process orchestration stay in `app.py`; Flex routes live in `flex/api.py` -- MongoDB models in `api/db_model.py` +- Legacy contract persistence models live in `core/db/model.py`; `api/db_model.py` + contains only the public contract-type enum. Maintained Flex models and + repositories live under `flex/db/`. - Asset prices use process-local TTL caches — see `dexes/` for provider calls - Preserve financial values as `Decimal` or integer base units until an explicit compatibility boundary +- Persist maintained Flex financial `uint64` fields through the BSON codecs in + `flex/db/bson.py`; do not copy legacy int64/float compatibility shapes +- Outbound transfers must persist immutable signed intent before broadcast and reconcile on-chain before completion +- Legacy lottery draws without a matching durable operation remain `reconciliation_required` +- LP events must enter through the complete-round preflight and `MongoLpProjectionRepository` +- Keep token-token fee funding in `operational_algo_balance_micros`; never mix it into economic reserves +- Trust `total_supply_micros` only with `total_supply_source=indexer`; otherwise + refetch and persist both fields atomically before a financial supply read +- Keep `SYNC_STAKING_POOLS=false` until full Algorand application-group validation is implemented +- Raw LP account balances never publish prices; `BACKGROUND_LP_PRICES_UPDATE` + is a retired compatibility setting and cannot restore the removed publisher - New pricing and transaction invariants belong in pure modules under `flex/domain/` - Run the strict mypy target before changing `core/circuit_breaker.py` or `flex/domain/` @@ -91,8 +106,8 @@ See parent `~/dev/cometa/CLAUDE.md` for the canonical API contract table. That f ## Testing -Tests are organized by boundary under `tests/unit/`. Run the same fast checks used -by CI before committing: +Fast tests are organized by boundary under `tests/unit/`; real-service checks live +under `tests/integration/`. Run the same local quality gate used by CI before committing: ```bash make sync @@ -103,7 +118,10 @@ make quality - Use `httpx.AsyncClient` with ASGI transport for endpoint integration tests - Keep domain tests pure and deterministic; inject clocks and provider functions - Priority: event replay/idempotency, price freshness, contract CRUD, authorization +- Exercise crash boundaries, concurrent replay, BSON `uint64` limits, and + Indexer-lag cutovers for financial projections - Use a dedicated test database only for integration tests; never point tests at production +- Set `MONGODB_TEST_URI` only to run the opt-in MongoDB integration suite - Every bug fix should include a regression test when practical ## Commit Discipline @@ -118,9 +136,10 @@ make quality Tasks in `BOARD.md`. Format: pantheon. -## Available MCP Tools +## Optional Diagnostic Tools -- **MongoDB MCP** — direct database queries in Claude sessions. Use for debugging: `db.contracts.find({active: true})`, inspecting collections, verifying data integrity. Connection: `mongodb://localhost:27017/cometa` -- **Algorand MCP** — on-chain state verification, account info, asset lookups on mainnet -- **Vestige MCP** — DEX price data, pool states, trading pairs for Algorand DeFi -- **Codex MCP** — second-opinion code review via GPT-5.x. Run `review` after writing significant code +MongoDB, Algorand, and DEX inspection connectors may be available in some agent +sessions. Treat them as optional and read-only by default. Derive database and +network targets from the active environment; never assume localhost is a safe +database and never sign or broadcast a transaction without explicit task-scoped +authorization. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 076d98df..207079bd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,16 +13,21 @@ make sync make run ``` +`make run` uses the production-equivalent entrypoint. Use `make run-api` for +API-only hot reload when migrations and background workers are intentionally +out of scope. + Before opening a pull request, run the same gate used by CI: ```bash make quality ``` -Use `make format` to apply Ruff formatting. New domain code should use Python -3.12 type hints and pass strict mypy checks. Keep I/O in adapters or application -services; prefer pure functions and immutable value objects for pricing, -transaction parsing, and other financial rules. +Use `make format` to apply Ruff formatting. New domain code targets Python 3.12, +passes strict mypy, and must remain green under the Python 3.14 compatibility +job. Keep I/O in adapters or application services; prefer pure functions and +immutable value objects for pricing, transaction parsing, and other financial +rules. ## Tests @@ -31,7 +36,9 @@ using the `test_.py` naming pattern. Test both the successful result and the failure boundary, especially for: - integer or `Decimal` amount handling; +- BSON round trips across the full Algorand `uint64` range; - duplicate, replayed, reordered, or nested chain events; +- crashes between durable intent, broadcast, reconciliation, marker, and cursor writes; - stale prices and provider fallback exhaustion; - retries, timeouts, and circuit-breaker transitions; - authentication and storage uniqueness. @@ -39,6 +46,14 @@ and the failure boundary, especially for: Mock external networks at the adapter boundary. Never use production credentials, wallets, or mutable production data in tests. +MongoDB semantics that mocks cannot prove belong under `tests/integration/`. +Run them only against a disposable database: + +```bash +MONGODB_TEST_URI=mongodb://127.0.0.1:27017 \ + pipenv run pytest tests/integration -m integration -v +``` + ## API compatibility The frontend lives in `../metafarm-frontend` and calls this service through diff --git a/Makefile b/Makefile index 209a122d..bd64858d 100644 --- a/Makefile +++ b/Makefile @@ -1,36 +1,54 @@ PYTHON_LINT_PATHS := \ - api/background.py app.py blockchain/indexer.py bot/log.py env.py telegram_bot.py \ + api/background.py api/nft_lottery.py api/wallet.py app.py blockchain/indexer.py bot/log.py env.py telegram_bot.py \ core/circuit_breaker.py core/cometa.py core/decorators.py core/util.py \ - flex/__init__.py flex/api.py flex/application flex/blockchain/contract_state.py flex/data/asset_prices.py \ - flex/data/lp_prices.py flex/data/lp_states.py flex/data/pool_state.py \ - flex/data/tinyman_lps.py flex/data/transactions.py \ - flex/db/classes/collection_manager.py flex/db/indexes.py \ - flex/db/model/liquidity_pools.py \ - flex/db/model/priced.py flex/domain flex/providers/pact.py flex/providers/price_router.py \ - flex/providers/vestige.py flex/sync_pools.py \ - scripts/verify_algorand_credentials.py tests + flex/__init__.py flex/api.py flex/application flex/blockchain/asset_transfers.py \ + flex/blockchain/contract_state.py flex/blockchain/info.py flex/data/asset_prices.py \ + flex/data/lp_registry.py flex/data/lp_states.py flex/data/lp_tokens.py flex/data/pool_state.py \ + flex/data/transactions.py \ + flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/base_entity.py \ + flex/db/classes/bson_uint64.py flex/db/classes/collection_manager.py \ + flex/db/indexes.py flex/db/lp_projection.py \ + flex/db/sync_coordinator.py flex/db/model/airdrop.py flex/db/model/blockchain.py flex/db/model/liquidity_pools.py \ + flex/db/model/pools.py flex/db/model/priced.py flex/db/model/transfers.py flex/domain flex/providers/pact.py \ + flex/providers/price_router.py \ + flex/providers/vestige.py flex/sync_pools.py flex/sync_state.py \ + flex/migrations/fix_dex_providers.py \ + flex/tools/airdrop.py scripts/verify_algorand_credentials.py tests PYTHON_MODERN_PATHS := \ - api/background.py core/circuit_breaker.py flex/application flex/blockchain/contract_state.py \ - flex/data/asset_prices.py flex/data/lp_prices.py flex/db/model/priced.py \ - flex/domain flex/providers/pact.py flex/providers/price_router.py tests/unit + api/background.py api/nft_lottery.py api/wallet.py core/circuit_breaker.py flex/application \ + flex/blockchain/asset_transfers.py \ + flex/blockchain/contract_state.py flex/data/asset_prices.py flex/data/lp_registry.py flex/data/lp_tokens.py \ + flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/bson_uint64.py \ + flex/db/lp_projection.py flex/db/sync_coordinator.py \ + flex/db/model/airdrop.py flex/db/model/priced.py flex/db/model/transfers.py \ + flex/domain flex/providers/pact.py flex/providers/price_router.py tests/unit tests/integration PYTHON_FORMAT_PATHS := \ - api/background.py app.py blockchain/indexer.py bot/log.py core/circuit_breaker.py core/cometa.py core/util.py \ - flex/api.py flex/application flex/blockchain/contract_state.py flex/data/asset_prices.py flex/data/lp_prices.py \ - flex/data/lp_states.py flex/data/pool_state.py flex/data/tinyman_lps.py \ - flex/data/transactions.py flex/db/indexes.py flex/db/model/liquidity_pools.py \ - flex/db/model/priced.py flex/domain flex/providers/pact.py flex/providers/price_router.py \ - flex/providers/vestige.py flex/sync_pools.py telegram_bot.py \ - tests/conftest.py tests/unit + api/background.py api/nft_lottery.py api/wallet.py app.py blockchain/indexer.py bot/log.py \ + core/circuit_breaker.py core/cometa.py core/util.py \ + flex/api.py flex/application flex/blockchain/asset_transfers.py flex/blockchain/contract_state.py flex/blockchain/info.py \ + flex/data/asset_prices.py flex/data/lp_registry.py \ + flex/data/lp_states.py flex/data/lp_tokens.py flex/data/pool_state.py \ + flex/data/transactions.py flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/base_entity.py \ + flex/db/classes/bson_uint64.py flex/db/classes/collection_manager.py flex/db/indexes.py flex/db/lp_projection.py \ + flex/db/sync_coordinator.py flex/db/model/airdrop.py flex/db/model/blockchain.py \ + flex/db/model/liquidity_pools.py flex/db/model/pools.py flex/db/model/priced.py flex/db/model/transfers.py \ + flex/domain flex/providers/pact.py flex/providers/price_router.py \ + flex/providers/vestige.py flex/sync_pools.py flex/sync_state.py telegram_bot.py \ + flex/migrations/fix_dex_providers.py \ + flex/tools/airdrop.py tests/conftest.py tests/unit tests/integration -.PHONY: sync run lint format format-check typecheck test quality +.PHONY: sync run run-api lint format format-check typecheck test quality sync: pipenv verify - pipenv sync --dev + pipenv sync --dev --python 3.12 run: + pipenv run python app.py + +run-api: pipenv run uvicorn app:app --reload --port 8000 lint: @@ -50,8 +68,14 @@ test: pipenv run pytest tests \ --cov=core.circuit_breaker \ --cov=core.decorators \ + --cov=flex.application.asset_transfers \ + --cov=flex.db.asset_transfer_intents \ + --cov=flex.db.lp_projection \ + --cov=flex.db.sync_coordinator \ --cov=flex.db.classes.collection_manager \ --cov=flex.blockchain.contract_state \ + --cov=flex.domain.allocation \ + --cov=flex.domain.lp_projection \ --cov=flex.domain.pricing \ --cov=flex.domain.transactions \ --cov=flex.providers.pact \ diff --git a/README.md b/README.md index 7013dfad..f1c7a5aa 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@

CI - Python 3.12 + Python 3.12 and 3.14 FastAPI Algorand mainnet API status @@ -37,10 +37,11 @@ MongoDB—into stable, query-oriented API models for the product frontend. | Engineering concern | Implementation | | --- | --- | -| **Financial precision** | Prices use validated `Decimal` value objects, carry source and observation time, and cross the legacy `float` boundary only explicitly. | -| **Deterministic replay identity** | Nested Algorand transfers receive stable, projection-scoped event IDs; unique indexes guard price and event-marker collections. Crash-atomic projection remains roadmap work. | +| **Financial precision** | Prices use validated `Decimal` value objects; LP balances, fees, and canonical asset supply retain the full Algorand `uint64` domain through BSON-safe storage. | +| **Crash-safe projections** | Scoped event cursors, fee-specific IDs, per-state compare-and-set writes, immutable markers, and a fenced round checkpoint make LP replay convergent across crashes and competing workers. | +| **Replay-safe payouts** | Airdrops and NFT transfers persist immutable signed intent before broadcast, cap signer fees, bind the configured genesis hash, keep terminal states monotonic, and fail closed on pre-intent lottery history. | | **Resilient price routing** | Vestige and Tinyman payloads are validated with provenance and bounded staleness; retry classification and a guarded Vestige refresh prevent failure storms. | -| **Operational boundaries** | Selected latency-sensitive SDK calls leave the event loop through executors; wallet fan-out is cached and bounded; background workers reconcile chain state without coupling reads to refresh latency. | +| **Operational boundaries** | Selected blocking chain calls leave the event loop; deterministic failures use bounded retry; unverified staking is fail-closed and the raw-balance LP price publisher has been removed. | | **Versioned chain decoding** | Reach 0.1.11 state is decoded natively from Algorand with explicit per-version layouts, exact-width integers, and fail-closed schema validation. | | **Supply-chain hardening** | The digest-pinned Alpine image is multi-stage, non-root, and Python-only; CI smoke-tests it and rejects high/critical vulnerabilities or embedded secrets. | @@ -70,6 +71,12 @@ farm and distribution versions map packed Algorand global and local state into the existing API view shape without private npm packages or a second runtime. The decision and extension rules are documented in [`docs/architecture/native-reach-state.md`](docs/architecture/native-reach-state.md). +Financial write paths are documented separately in +[`docs/architecture/outbound-asset-transfers.md`](docs/architecture/outbound-asset-transfers.md) +and [`docs/architecture/lp-projection.md`](docs/architecture/lp-projection.md). +The latest internal multi-agent engineering review, including resolved and +intentionally open items, is in +[`docs/audit/01-audit-architecture-financial-2026-07-19.md`](docs/audit/01-audit-architecture-financial-2026-07-19.md). ### Reliability boundaries @@ -77,8 +84,10 @@ The decision and extension rules are documented in | --- | --- | | Provider quote → stored price | Positive, finite decimal values with source and observation timestamp | | Cached price → API response | Explicit freshness window; expired data is rejected instead of silently relabelled | -| Chain event → read model | Deterministic identity plus unique persistence constraints | -| Sync SDK → async request path | Bounded executor hand-off | +| Chain event → LP read model | Full-block preflight, fee-aware uint64 ledger, CAS cursor, marker repair, and round fencing | +| Raw LP account balance → price | Prohibited; economic reserves require a verified DEX-specific adapter | +| Asset payout → Algorand | Validate genesis and fee ceiling; persist signed intent first; rebroadcast identical bytes; reconcile before completion | +| Selected sync chain SDK → async request path | Bounded executor hand-off | | Permanent provider error → retry loop | Typed classification prevents pointless retries | | Half-open circuit → provider | A single probe prevents a recovery stampede | | Reach bytes → public view | Exact key, step, size, tag, and contract-version validation | @@ -87,7 +96,8 @@ The decision and extension rules are documented in ### Requirements -- Python 3.12 and [Pipenv](https://pipenv.pypa.io/) +- Python 3.12 and [Pipenv](https://pipenv.pypa.io/) for the production-equivalent environment +- Python 3.14 is also exercised by CI as a forward-compatibility gate - MongoDB - access to an Algorand node/indexer @@ -100,10 +110,13 @@ pipenv run python -c \ 'from algosdk import account, mnemonic; key, _ = account.generate_account(); print(mnemonic.from_private_key(key))' # Put this throwaway phrase in ALGO_MNEMONIC. Never fund or reuse the account. # Point MONGODB_HOST, ALGOD_ADDRESS, and ALGO_INDEXER_ADDRESS at dev services. -make run +make run-api ``` -`make run` starts Uvicorn with reload on port `8000`. A development mnemonic is +`make run-api` is the safe API-only development loop. `make run` executes the +production-equivalent entrypoint: critical indexes, +optional migrations when `MIGRATE=true`, configured workers, and Uvicorn. For +the full entrypoint, review every worker flag first. A development mnemonic is still required by legacy Python transaction adapters; use a generated, unfunded account only. @@ -121,6 +134,10 @@ docker compose up -d --build docker compose logs -f app ``` +This Compose stack starts persistent MongoDB and a full Algorand node. Inspect +the network, volume paths, and validated `MONGODB_IMAGE`/`ALGOD_IMAGE` values +before using it outside an isolated development host. + ## Quality gate ```bash @@ -134,9 +151,13 @@ This single command runs: - the complete Python test suite with branch coverage, including deterministic Reach state-codec and security-boundary tests. -CI repeats those checks on every pull request and every push to `main`, verifies -the lockfile and Compose configuration, builds and smoke-tests the production -image, and scans it with Trivy. The focused coverage ratchet is currently 75%; +CI repeats those checks on Python 3.12 and 3.14 for every pull request and every +push to `main`, verifies the lockfile and Compose configuration, builds and +smoke-tests the production image, scans it with Trivy, and exercises financial +repository invariants against a digest-pinned MongoDB service. A pinned +TruffleHog gate fetches and scans every published Git ref for verified or unresolved +credentials and feeds the stable required `python` status. The focused +coverage ratchet is currently 75%; it measures maintained domain and infrastructure modules rather than presenting a misleading whole-repository number. @@ -151,8 +172,8 @@ Useful individual targets are `make lint`, `make format-check`, | `GET /contracts` | Farm and distribution catalog | | `GET /contracts/user/{address}` | Contracts associated with a wallet | | `GET /contracts/farm/enriched` | Contracts enriched with asset metadata and prices | -| `POST /assets/price` | Batch asset pricing | -| `POST /lp/state/priced` | Batched LP reserve and price data | +| `POST /assets/price` | Read stored bounded-fresh prices; optional `ids` accepts up to 250 values, while omission returns all stored projections | +| `POST /lp/state/priced` | Read LP token prices; missing or stale batch entries are returned as `null` | | `GET /stats/tvl` | Protocol TVL snapshot | The production API intentionally disables interactive OpenAPI pages. Endpoint @@ -173,7 +194,8 @@ flex/domain/ Pure pricing and transaction invariants flex/providers/ Market-data provider adapters flex/db/ MongoDB models, repositories, and indexes tests/unit/ Fast regression and boundary tests -scripts/ Deployment and safe operational utilities +tests/integration/ Opt-in tests against disposable real services +scripts/ Deployment and legacy operational utilities ``` ## Configuration and security diff --git a/SECURITY.md b/SECURITY.md index 1d277907..0f458eb2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -33,3 +33,7 @@ replay. Tests must use generated accounts and non-production credentials. Never commit mnemonics, API keys, `.env` files, database exports, recovery artifacts, or unredacted logs. If a secret reaches Git history, revoke or rotate it immediately; deleting the current file is not sufficient. + +GitHub secret scanning and push protection are enabled. CI additionally scans +the full reachable history with a pinned detector; do not bypass that gate with +an allowlist unless the value is a documented, non-secret test fixture. diff --git a/StakingRewards.sol b/StakingRewards.sol deleted file mode 100644 index 3d80d147..00000000 --- a/StakingRewards.sol +++ /dev/null @@ -1,144 +0,0 @@ -// SPDX-License-Identifier: MIT -pragma solidity ^0.8.24; - -contract StakingRewards { - IERC20 public immutable stakingToken; - IERC20 public immutable rewardsToken; - - address public owner; - - // Duration of rewards to be paid out (in seconds) - uint256 public duration; - // Timestamp of when the rewards finish - uint256 public finishAt; - // Minimum of last updated time and reward finish time - uint256 public updatedAt; - // Reward to be paid out per second - uint256 public rewardRate; - // Sum of (reward rate * dt * 1e18 / total supply) - uint256 public rewardPerTokenStored; - // User address => rewardPerTokenStored - mapping(address => uint256) public userRewardPerTokenPaid; - // User address => rewards to be claimed - mapping(address => uint256) public rewards; - - // Total staked - uint256 public totalSupply; - // User address => staked amount - mapping(address => uint256) public balanceOf; - - constructor(address _stakingToken, address _rewardToken) { - owner = msg.sender; - stakingToken = IERC20(_stakingToken); - rewardsToken = IERC20(_rewardToken); - } - - modifier onlyOwner() { - require(msg.sender == owner, "not authorized"); - _; - } - - modifier updateReward(address _account) { - rewardPerTokenStored = rewardPerToken(); - updatedAt = lastTimeRewardApplicable(); - - if (_account != address(0)) { - rewards[_account] = earned(_account); - userRewardPerTokenPaid[_account] = rewardPerTokenStored; - } - - _; - } - - function lastTimeRewardApplicable() public view returns (uint256) { - return _min(finishAt, block.timestamp); - } - - function rewardPerToken() public view returns (uint256) { - if (totalSupply == 0) { - return rewardPerTokenStored; - } - - return rewardPerTokenStored - + (rewardRate * (lastTimeRewardApplicable() - updatedAt) * 1e18) - / totalSupply; - } - - function stake(uint256 _amount) external updateReward(msg.sender) { - require(_amount > 0, "amount = 0"); - stakingToken.transferFrom(msg.sender, address(this), _amount); - balanceOf[msg.sender] += _amount; - totalSupply += _amount; - } - - function withdraw(uint256 _amount) external updateReward(msg.sender) { - require(_amount > 0, "amount = 0"); - balanceOf[msg.sender] -= _amount; - totalSupply -= _amount; - stakingToken.transfer(msg.sender, _amount); - } - - function earned(address _account) public view returns (uint256) { - return ( - ( - balanceOf[_account] - * (rewardPerToken() - userRewardPerTokenPaid[_account]) - ) / 1e18 - ) + rewards[_account]; - } - - function getReward() external updateReward(msg.sender) { - uint256 reward = rewards[msg.sender]; - if (reward > 0) { - rewards[msg.sender] = 0; - rewardsToken.transfer(msg.sender, reward); - } - } - - function setRewardsDuration(uint256 _duration) external onlyOwner { - require(finishAt < block.timestamp, "reward duration not finished"); - duration = _duration; - } - - function notifyRewardAmount(uint256 _amount) - external - onlyOwner - updateReward(address(0)) - { - if (block.timestamp >= finishAt) { - rewardRate = _amount / duration; - } else { - uint256 remainingRewards = (finishAt - block.timestamp) * rewardRate; - rewardRate = (_amount + remainingRewards) / duration; - } - - require(rewardRate > 0, "reward rate = 0"); - require( - rewardRate * duration <= rewardsToken.balanceOf(address(this)), - "reward amount > balance" - ); - - finishAt = block.timestamp + duration; - updatedAt = block.timestamp; - } - - function _min(uint256 x, uint256 y) private pure returns (uint256) { - return x <= y ? x : y; - } -} - -interface IERC20 { - function totalSupply() external view returns (uint256); - function balanceOf(address account) external view returns (uint256); - function transfer(address recipient, uint256 amount) - external - returns (bool); - function allowance(address owner, address spender) - external - view - returns (uint256); - function approve(address spender, uint256 amount) external returns (bool); - function transferFrom(address sender, address recipient, uint256 amount) - external - returns (bool); -} diff --git a/api/background.py b/api/background.py index bde3f626..41672287 100644 --- a/api/background.py +++ b/api/background.py @@ -25,7 +25,7 @@ is_asset_price_stale, update_asset_price, ) -from flex.data.lp_prices import get_lp_token_definitions, update_lp_token_prices +from flex.data.lp_registry import get_lp_token_definitions from flex.migrations import migrate_background from flex.sync_pools import sync_pools_loop @@ -161,7 +161,8 @@ async def update_asset_prices_background(): all_assets = db.assets.get_all() - # Exclude LP tokens — they are priced separately by update_lp_token_prices() + # LP tokens have no verified economic-reserve adapter. Keep them out of the + # generic provider refresh instead of deriving prices from account balances. try: lp_defs = await get_lp_token_definitions() lp_token_ids = {d["lp_token_id"] for d in lp_defs} @@ -240,11 +241,10 @@ async def update_asset_prices_background(): failures, ) - # LP token prices: calculate from on-chain reserves - try: - await update_lp_token_prices(current_round) - except Exception as e: - logger.error(f"LP token price update failed: {e}") + if settings.background_lp_prices_update: + logger.error( + "BACKGROUND_LP_PRICES_UPDATE is retired and ignored; raw pool-account balances cannot publish prices", + ) def run_background(): diff --git a/api/nft_lottery.py b/api/nft_lottery.py index 0062f56b..785b5402 100644 --- a/api/nft_lottery.py +++ b/api/nft_lottery.py @@ -1,33 +1,65 @@ import logging +import math import random import time from dataclasses import dataclass -from datetime import datetime -from enum import Enum -from typing import Optional +from datetime import UTC, datetime, timedelta +from enum import StrEnum +from hashlib import sha256 +from typing import Any +from uuid import uuid4 from algosdk.v2client import indexer from dataclasses_json import dataclass_json +from pymongo import ASCENDING, ReturnDocument +from pymongo.errors import DuplicateKeyError, OperationFailure import flex from api.swaps import SwapInfo -from api.wallet import send_nft, cometa_public_key +from api.wallet import send_nft from blockchain.nfts import get_nft_info -from blockchain.node import init_algod_client, get_current_round +from blockchain.node import init_algod_client from core.db.db_manager import DbManager +from core.db.mongodb import get_db_collection from env import settings - +from flex.blockchain.base import cometa_public_key MIN_DRAW_INTERVAL = 60 * 60 * 24 # 24 hours +LOTTERY_DRAW_ID_INDEX_NAME = "id_nonempty_unique" +LOTTERY_DRAW_ID_FILTER = { + "id": { + "$type": "string", + "$gt": "", + } +} +LEGACY_LOTTERY_DRAW_ID_FILTER = {"id": {"$type": "string"}} + -class LotteryType(str, Enum): - SWAP = 'swap' - STAKING = 'staking' +class LotteryType(StrEnum): + SWAP = "swap" + STAKING = "staking" def __str__(self): return self.value +class LotteryPayoutStatus(StrEnum): + PENDING = "pending" + PREPARED = "prepared" + CONFIRMED = "confirmed" + UNRESOLVED = "unresolved" + RECONCILIATION_REQUIRED = "reconciliation_required" + + +class StakingEntitlementStatus(StrEnum): + RESERVED = "reserved" + PREPARED = "prepared" + MATERIALIZED = "materialized" + CONFIRMED = "confirmed" + NO_PRIZE = "no_prize" + UNRESOLVED = "unresolved" + + @dataclass_json @dataclass class NftLottery: @@ -36,31 +68,41 @@ class NftLottery: min_amount: int probability: float available_nfts: list[int] - win_title: str = 'You have won a prize NFT!' - max_amount: Optional[int] = None - type: Optional[str] = None - only_for_buy: Optional[bool] = None - pool_id: Optional[int] = None + win_title: str = "You have won a prize NFT!" + max_amount: int | None = None + type: str | None = None + only_for_buy: bool | None = None + pool_id: int | None = None def is_eligible(self, entity_id: int, amount: float) -> bool: - return (entity_id == self.asset_id or entity_id == self.pool_id) and amount >= self.min_amount and ( - self.max_amount is None or amount <= self.max_amount) + return ( + (entity_id == self.asset_id or entity_id == self.pool_id) + and amount >= self.min_amount + and (self.max_amount is None or amount <= self.max_amount) + ) @dataclass_json @dataclass class LotteryDraw: wallet: str - prize: Optional[int] - timestamp: Optional[float] = None - lottery_name: Optional[str] = None + prize: int | None + timestamp: float | None = None + lottery_name: str | None = None claimed: bool = False - created_date: Optional[datetime] = None - send_error: Optional[str] = None + created_date: datetime | None = None + send_error: str | None = None + id: str | None = None + payout_operation_id: str | None = None + payout_txid: str | None = None + confirmed_round: int | None = None + payout_status: str | None = None + entitlement_id: str | None = None + entitlement_generation: int | None = None def __post_init__(self): if self.timestamp: - self.created_date = datetime.fromtimestamp(self.timestamp) + self.created_date = datetime.fromtimestamp(self.timestamp, tz=UTC) @dataclass_json @@ -71,17 +113,623 @@ class LotteryParticipant: last_draw_block: int -nft_lotteries = DbManager[NftLottery](settings.db_name, 'nft_lotteries', 'name', NftLottery) -lottery_draws = DbManager[LotteryDraw](settings.db_name, 'lottery_draws', 'swap_txid', LotteryDraw) -lottery_participants = DbManager[LotteryParticipant](settings.db_name, 'lottery_participants', 'address', LotteryParticipant) +nft_lotteries = DbManager[NftLottery](settings.db_name, "nft_lotteries", "name", NftLottery) +lottery_draws = DbManager[LotteryDraw](settings.db_name, "lottery_draws", "id", LotteryDraw) +lottery_participants = DbManager[LotteryParticipant]( + settings.db_name, "lottery_participants", "address", LotteryParticipant +) +lottery_entitlements = get_db_collection( + settings.db_name, + "lottery_entitlements", +) algod_client = init_algod_client() -indexer_client = indexer.IndexerClient(indexer_token=settings.algod_token, indexer_address=settings.algo_indexer_address) +indexer_client = indexer.IndexerClient( + indexer_token=settings.algod_token, indexer_address=settings.algo_indexer_address +) logger = logging.getLogger(__name__) -def draw_id(lottery: NftLottery) -> Optional[int]: +def _is_lottery_draw_id_index( + index: dict[str, Any], + *, + partial_filter: dict[str, Any], +) -> bool: + return ( + list(index.get("key", [])) == [("id", ASCENDING)] + and index.get("unique") is True + and index.get("partialFilterExpression") == partial_filter + ) + + +def _duplicate_lottery_draw_ids() -> list[dict[str, Any]]: + return list( + lottery_draws.collection.aggregate( + [ + {"$match": LOTTERY_DRAW_ID_FILTER}, + { + "$group": { + "_id": "$id", + "count": {"$sum": 1}, + "document_ids": {"$push": "$_id"}, + } + }, + {"$match": {"count": {"$gt": 1}}}, + {"$limit": 10}, + ], + allowDiskUse=True, + ) + ) + + +def ensure_lottery_indexes() -> None: + """Upgrade the draw identity index without a uniqueness gap or data loss.""" + + collection = lottery_draws.collection + indexes = collection.index_information() + desired_index = indexes.get(LOTTERY_DRAW_ID_INDEX_NAME) + legacy_index = indexes.get("id_unique") + + if desired_index is not None and not _is_lottery_draw_id_index( + desired_index, + partial_filter=LOTTERY_DRAW_ID_FILTER, + ): + raise RuntimeError( + f"lottery_draws index {LOTTERY_DRAW_ID_INDEX_NAME!r} has unexpected options; " + "inspect it manually before startup" + ) + + if legacy_index is not None and not ( + _is_lottery_draw_id_index( + legacy_index, + partial_filter=LEGACY_LOTTERY_DRAW_ID_FILTER, + ) + or _is_lottery_draw_id_index( + legacy_index, + partial_filter=LOTTERY_DRAW_ID_FILTER, + ) + ): + raise RuntimeError("lottery_draws index 'id_unique' has unexpected options; inspect it manually before startup") + + if desired_index is None and not ( + legacy_index is not None + and _is_lottery_draw_id_index( + legacy_index, + partial_filter=LOTTERY_DRAW_ID_FILTER, + ) + ): + duplicate_groups = _duplicate_lottery_draw_ids() + if duplicate_groups: + raise RuntimeError( + "lottery_draws contains duplicate non-empty draw IDs; " + f"preserved {len(duplicate_groups)} duplicate group(s) for explicit reconciliation" + ) + try: + collection.create_index( + "id", + unique=True, + name=LOTTERY_DRAW_ID_INDEX_NAME, + partialFilterExpression=LOTTERY_DRAW_ID_FILTER, + ) + except DuplicateKeyError as exc: + raise RuntimeError( + "lottery_draws uniqueness changed while building its index; " + "all conflicting records were preserved for explicit reconciliation" + ) from exc + except OperationFailure as exc: + if exc.code != 11000: + raise + raise RuntimeError( + "lottery_draws uniqueness changed while building its index; " + "all conflicting records were preserved for explicit reconciliation" + ) from exc + desired_index = collection.index_information().get(LOTTERY_DRAW_ID_INDEX_NAME) + if desired_index is None or not _is_lottery_draw_id_index( + desired_index, + partial_filter=LOTTERY_DRAW_ID_FILTER, + ): + raise RuntimeError("lottery_draws unique draw ID index was not installed") + + if legacy_index is not None and _is_lottery_draw_id_index( + legacy_index, + partial_filter=LEGACY_LOTTERY_DRAW_ID_FILTER, + ): + # The replacement already protects every non-empty ID. Dropping the + # narrower legacy definition now cannot introduce a uniqueness gap. + try: + collection.drop_index("id_unique") + except OperationFailure as exc: + if exc.code != 27: + raise + else: + logger.info("Upgraded lottery_draws.id unique index without modifying draw records") + replacement = collection.index_information().get(LOTTERY_DRAW_ID_INDEX_NAME) + if replacement is None or not _is_lottery_draw_id_index( + replacement, + partial_filter=LOTTERY_DRAW_ID_FILTER, + ): + raise RuntimeError("lottery_draws replacement identity index disappeared during the legacy index upgrade") + + +def _create_draw( + *, + lottery_name: str, + prize: int | None, + wallet: str, + timestamp: float, +) -> LotteryDraw: + ensure_lottery_indexes() + return lottery_draws.create( + LotteryDraw( + id=uuid4().hex, + lottery_name=lottery_name, + prize=prize, + wallet=wallet, + timestamp=timestamp, + payout_status=LotteryPayoutStatus.PENDING, + ) + ) + + +def _staking_entitlement_id(lottery_name: str, wallet: str) -> str: + identity = f"staking-lottery:{lottery_name}:{wallet}" + return sha256(identity.encode()).hexdigest() + + +def _valid_draw_timestamp(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + return None + return float(value) + + +def _seed_staking_entitlement( + *, + lottery_name: str, + wallet: str, + now: datetime, +) -> str: + """Initialize the CAS guard from any pre-migration draw.""" + + entitlement_id = _staking_entitlement_id(lottery_name, wallet) + seed: dict[str, Any] = { + "_id": entitlement_id, + "lottery_name": lottery_name, + "wallet": wallet, + "generation": 0, + "next_eligible_at": datetime.fromtimestamp(0, tz=UTC), + "created": now, + "updated": now, + } + latest = lottery_draws.collection.find_one( + { + "wallet": wallet, + "lottery_name": lottery_name, + }, + sort=[("timestamp", -1)], + ) + outstanding = lottery_draws.collection.find_one( + { + "wallet": wallet, + "lottery_name": lottery_name, + "prize": {"$ne": None}, + "claimed": {"$in": [False, None]}, + }, + sort=[("timestamp", -1)], + ) + active_document = outstanding or latest + if active_document is not None: + draw, _ = _backfill_draw_id(active_document) + active_timestamp = _valid_draw_timestamp(draw.timestamp) + latest_timestamp = _valid_draw_timestamp(latest.get("timestamp")) if latest is not None else active_timestamp + if active_timestamp is None or latest_timestamp is None: + draw_timestamp = now.timestamp() + status = StakingEntitlementStatus.UNRESOLVED + next_eligible_at = now + timedelta(seconds=MIN_DRAW_INTERVAL) + else: + draw_timestamp = active_timestamp + status = ( + StakingEntitlementStatus.NO_PRIZE + if draw.prize is None + else (StakingEntitlementStatus.CONFIRMED if draw.claimed else StakingEntitlementStatus.UNRESOLVED) + ) + try: + next_eligible_at = datetime.fromtimestamp( + latest_timestamp, + tz=UTC, + ) + timedelta(seconds=MIN_DRAW_INTERVAL) + except (OverflowError, OSError, ValueError): + draw_timestamp = now.timestamp() + status = StakingEntitlementStatus.UNRESOLVED + next_eligible_at = now + timedelta(seconds=MIN_DRAW_INTERVAL) + seed["next_eligible_at"] = next_eligible_at + seed["active"] = { + "draw_id": draw.id, + "draw_timestamp": draw_timestamp, + "prize": draw.prize, + "status": status, + } + + try: + lottery_entitlements.update_one( + {"_id": entitlement_id}, + {"$setOnInsert": seed}, + upsert=True, + ) + except DuplicateKeyError: + # Another worker installed the same single-document guard. + pass + return entitlement_id + + +def _claim_staking_entitlement( + *, + lottery_name: str, + wallet: str, + now: datetime, +) -> dict[str, Any] | None: + """Claim one rolling-window liability or recover its unfinished draw.""" + + entitlement_id = _seed_staking_entitlement( + lottery_name=lottery_name, + wallet=wallet, + now=now, + ) + draw_id_value = uuid4().hex + draw_timestamp = now.timestamp() + claimed = lottery_entitlements.find_one_and_update( + { + "_id": entitlement_id, + "next_eligible_at": {"$lte": now}, + "$or": [ + {"active": {"$exists": False}}, + { + "active.status": { + "$in": [ + StakingEntitlementStatus.CONFIRMED, + StakingEntitlementStatus.NO_PRIZE, + ] + } + }, + ], + }, + { + "$inc": {"generation": 1}, + "$set": { + "next_eligible_at": now + timedelta(seconds=MIN_DRAW_INTERVAL), + "active": { + "draw_id": draw_id_value, + "draw_timestamp": draw_timestamp, + "reserved_at": now, + "status": StakingEntitlementStatus.RESERVED, + }, + "updated": now, + }, + }, + return_document=ReturnDocument.AFTER, + ) + if claimed is not None: + return claimed + + current = lottery_entitlements.find_one({"_id": entitlement_id}) + if not isinstance(current, dict): + raise RuntimeError("staking lottery entitlement disappeared") + active = current.get("active") + if isinstance(active, dict) and active.get("status") in { + StakingEntitlementStatus.MATERIALIZED, + StakingEntitlementStatus.UNRESOLVED, + }: + active_draw_id = active.get("draw_id") + draw_document = ( + lottery_draws.collection.find_one({"id": active_draw_id}) if isinstance(active_draw_id, str) else None + ) + if ( + isinstance(draw_document, dict) + and draw_document.get("claimed") is True + and draw_document.get("payout_status") == LotteryPayoutStatus.CONFIRMED + ): + repaired = lottery_entitlements.find_one_and_update( + { + "_id": entitlement_id, + "generation": current["generation"], + "active.draw_id": active_draw_id, + "active.status": active["status"], + }, + { + "$set": { + "active.status": StakingEntitlementStatus.CONFIRMED, + "updated": now, + } + }, + return_document=ReturnDocument.AFTER, + ) + if repaired is not None: + return _claim_staking_entitlement( + lottery_name=lottery_name, + wallet=wallet, + now=now, + ) + if isinstance(active, dict) and active.get("status") in { + StakingEntitlementStatus.RESERVED, + StakingEntitlementStatus.PREPARED, + }: + # A crash or competing request may leave the one winning reservation + # unfinished. Helping it cannot create a second business entitlement. + return current + return None + + +def _prepare_staking_prize( + entitlement: dict[str, Any], + lottery: NftLottery, +) -> dict[str, Any]: + active = entitlement.get("active") + if not isinstance(active, dict) or not isinstance(active.get("draw_id"), str): + raise RuntimeError("staking lottery entitlement has no active draw") + if active.get("status") == StakingEntitlementStatus.RESERVED: + prize_id = draw_id(lottery) + prepared = lottery_entitlements.find_one_and_update( + { + "_id": entitlement["_id"], + "generation": entitlement["generation"], + "active.draw_id": active["draw_id"], + "active.status": StakingEntitlementStatus.RESERVED, + }, + { + "$set": { + "active.prize": prize_id, + "active.status": StakingEntitlementStatus.PREPARED, + "updated": datetime.now(UTC), + } + }, + return_document=ReturnDocument.AFTER, + ) + if prepared is not None: + return prepared + entitlement = lottery_entitlements.find_one({"_id": entitlement["_id"]}) + if not isinstance(entitlement, dict): + raise RuntimeError("staking lottery entitlement disappeared during prize reservation") + + active = entitlement.get("active") + if not isinstance(active, dict) or active.get("status") not in { + StakingEntitlementStatus.PREPARED, + StakingEntitlementStatus.MATERIALIZED, + StakingEntitlementStatus.NO_PRIZE, + }: + raise RuntimeError("staking lottery prize reservation is not recoverable") + return entitlement + + +def _materialize_staking_draw( + entitlement: dict[str, Any], + lottery: NftLottery, + wallet: str, +) -> LotteryDraw: + active = entitlement["active"] + draw_id_value = active["draw_id"] + draw_timestamp = active["draw_timestamp"] + prize_id = active.get("prize") + generation = entitlement["generation"] + ensure_lottery_indexes() + payload = LotteryDraw( + id=draw_id_value, + lottery_name=lottery.name, + prize=prize_id, + wallet=wallet, + timestamp=draw_timestamp, + payout_status=LotteryPayoutStatus.PENDING, + entitlement_id=entitlement["_id"], + entitlement_generation=generation, + ).to_dict() + try: + document = lottery_draws.collection.find_one_and_update( + {"id": draw_id_value}, + {"$setOnInsert": payload}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + except DuplicateKeyError: + document = lottery_draws.collection.find_one({"id": draw_id_value}) + if not isinstance(document, dict): + raise RuntimeError("staking lottery draw could not be materialized") + immutable = ( + document.get("id"), + document.get("lottery_name"), + document.get("wallet"), + document.get("prize"), + document.get("timestamp"), + document.get("entitlement_id"), + document.get("entitlement_generation"), + ) + expected = ( + draw_id_value, + lottery.name, + wallet, + prize_id, + draw_timestamp, + entitlement["_id"], + generation, + ) + if immutable != expected: + raise RuntimeError("staking lottery draw ID belongs to different immutable data") + + status = StakingEntitlementStatus.NO_PRIZE if prize_id is None else StakingEntitlementStatus.MATERIALIZED + updated = lottery_entitlements.update_one( + { + "_id": entitlement["_id"], + "generation": generation, + "active.draw_id": draw_id_value, + "active.status": { + "$in": [ + StakingEntitlementStatus.PREPARED, + status, + ] + }, + }, + { + "$set": { + "active.status": status, + "updated": datetime.now(UTC), + } + }, + ) + if updated.matched_count != 1: + current = lottery_entitlements.find_one({"_id": entitlement["_id"]}) + current_active = current.get("active") if isinstance(current, dict) else None + if not ( + isinstance(current_active, dict) + and current_active.get("draw_id") == draw_id_value + and current_active.get("status") + in { + status, + StakingEntitlementStatus.CONFIRMED, + StakingEntitlementStatus.UNRESOLVED, + } + ): + raise RuntimeError("staking lottery entitlement changed during draw materialization") + return LotteryDraw.from_dict(document) + + +def _get_or_create_staking_draw( + lottery: NftLottery, + wallet: str, + *, + now: datetime, +) -> LotteryDraw | None: + entitlement = _claim_staking_entitlement( + lottery_name=lottery.name, + wallet=wallet, + now=now, + ) + if entitlement is None: + return None + prepared = _prepare_staking_prize(entitlement, lottery) + return _materialize_staking_draw(prepared, lottery, wallet) + + +def _update_staking_entitlement_status( + draw: LotteryDraw, + status: StakingEntitlementStatus, +) -> None: + if draw.entitlement_id is None or draw.entitlement_generation is None or draw.id is None: + return + selector: dict[str, Any] = { + "_id": draw.entitlement_id, + "generation": draw.entitlement_generation, + "active.draw_id": draw.id, + } + if status != StakingEntitlementStatus.CONFIRMED: + selector["active.status"] = { + "$ne": StakingEntitlementStatus.CONFIRMED, + } + try: + result = lottery_entitlements.update_one( + selector, + { + "$set": { + "active.status": status, + "updated": datetime.now(UTC), + } + }, + ) + except Exception: + logger.exception( + "Could not update staking entitlement %s generation %s to %s; future draws remain fail-closed", + draw.entitlement_id, + draw.entitlement_generation, + status, + ) + return + if result.matched_count != 1: + current = lottery_entitlements.find_one( + { + "_id": draw.entitlement_id, + "generation": draw.entitlement_generation, + "active.draw_id": draw.id, + } + ) + current_active = current.get("active") if isinstance(current, dict) else None + if isinstance(current_active, dict) and current_active.get("status") in { + status, + StakingEntitlementStatus.CONFIRMED, + }: + return + logger.error( + "Could not update staking entitlement %s generation %s to %s; future draws remain fail-closed", + draw.entitlement_id, + draw.entitlement_generation, + status, + ) + + +def _backfill_draw_id(document: dict[str, Any]) -> tuple[LotteryDraw, Any]: + document_id = document.get("_id") + if document_id is None: + raise RuntimeError("lottery draw is missing its MongoDB identity") + + for attempt in range(4): + original_id = document.get("id") + missing_draw_id = not isinstance(original_id, str) or not original_id + draw_id_value = ( + f"legacy-{sha256(f'lottery-draw:{document_id}'.encode()).hexdigest()}" if missing_draw_id else original_id + ) + original_status = document.get("payout_status") + missing_payout_status = not isinstance(original_status, str) or not original_status + claimed_value = document.get("claimed") + missing_claimed = "claimed" not in document or claimed_value is None + if claimed_value is not None and not isinstance(claimed_value, bool): + raise RuntimeError("lottery draw has an invalid claimed state") + if not missing_draw_id and not missing_payout_status and not missing_claimed: + break + if attempt == 3: + raise RuntimeError("lottery draw kept changing during migration") + + expected_operation_id = f"nft:lottery:{draw_id_value}" + payout_status = ( + LotteryPayoutStatus.CONFIRMED + if claimed_value is True + else ( + LotteryPayoutStatus.PREPARED + if document.get("payout_operation_id") == expected_operation_id + else LotteryPayoutStatus.RECONCILIATION_REQUIRED + ) + ) + migration_query: dict[str, Any] = {"_id": document_id} + for field_name in ( + "id", + "payout_status", + "claimed", + "payout_operation_id", + ): + migration_query[field_name] = ( + {"$eq": document[field_name]} if field_name in document else {"$exists": False} + ) + migrated = lottery_draws.collection.find_one_and_update( + migration_query, + { + "$set": { + "id": draw_id_value, + "payout_status": payout_status, + **({"claimed": False} if missing_claimed else {}), + } + }, + return_document=ReturnDocument.AFTER, + ) + if migrated is not None: + document = migrated + break + updated = lottery_draws.collection.find_one({"_id": document_id}) + if updated is None: + raise RuntimeError("lottery draw disappeared during migration") + document = updated + + draw = LotteryDraw.from_dict(document) + if not isinstance(draw.id, str) or not draw.id or not draw.payout_status or not isinstance(draw.claimed, bool): + raise RuntimeError("lottery draw migration did not persist its identity and payout state") + return draw, document_id + + +def draw_id(lottery: NftLottery) -> int | None: if random.random() > lottery.probability: return None while True: @@ -90,11 +738,11 @@ def draw_id(lottery: NftLottery) -> Optional[int]: # TODO: refactor to get balance once res = random.choice(lottery.available_nfts) data = indexer_client.lookup_account_assets(address=cometa_public_key, asset_id=res) - assets = data.get('assets', []) - if len(assets) > 0 and assets[0].get('amount', 0) > 0: + assets = data.get("assets", []) + if len(assets) > 0 and assets[0].get("amount", 0) > 0: # drawn nft persists in the wallet break - logger.info(f'NFT {res} is not in the wallet, drawing again') + logger.info(f"NFT {res} is not in the wallet, drawing again") lottery.available_nfts.remove(res) nft_lotteries.update(lottery) return res @@ -108,33 +756,32 @@ class NftPrize: title: str -def get_nft_prize(lottery: NftLottery, asa_id: Optional[int]) -> Optional[NftPrize]: +def get_nft_prize(lottery: NftLottery, asa_id: int | None) -> NftPrize | None: if asa_id is None: return None prize_info = get_nft_info(asa_id) - PINATA_URL = 'https://gateway.pinata.cloud/ipfs/' - image_url = prize_info.image_url.replace('ipfs://', PINATA_URL) + PINATA_URL = "https://gateway.pinata.cloud/ipfs/" + image_url = prize_info.image_url.replace("ipfs://", PINATA_URL) - return NftPrize(asa_id=asa_id, - name=prize_info.name, - image_url=image_url, - title=lottery.win_title) + return NftPrize(asa_id=asa_id, name=prize_info.name, image_url=image_url, title=lottery.win_title) -def draw_prize(lottery: NftLottery, address: str) -> Optional[NftPrize]: - logger.debug(f'Drawing lottery {lottery.name} for {address}') +def draw_prize(lottery: NftLottery, address: str) -> NftPrize | None: + logger.debug(f"Drawing lottery {lottery.name} for {address}") prize_id = draw_id(lottery) - lottery_draws.create(LotteryDraw(lottery_name=lottery.name, - prize=prize_id, - wallet=address, - timestamp=time.time())) + _create_draw( + lottery_name=lottery.name, + prize=prize_id, + wallet=address, + timestamp=time.time(), + ) return get_nft_prize(lottery, prize_id) -def lottery_for_swap(swap: SwapInfo) -> Optional[NftPrize]: - lotteries = nft_lotteries.get_many({'type': LotteryType.SWAP}) - logger.debug(f'Swap lotteries cnt = {len(lotteries)}') +def lottery_for_swap(swap: SwapInfo) -> NftPrize | None: + lotteries = nft_lotteries.get_many({"type": LotteryType.SWAP}) + logger.debug(f"Swap lotteries cnt = {len(lotteries)}") prize = None for lottery in lotteries: @@ -146,104 +793,227 @@ def lottery_for_swap(swap: SwapInfo) -> Optional[NftPrize]: return prize -async def lottery_for_staking(pool_id: int, address: str) -> Optional[NftPrize]: - lotteries = nft_lotteries.get_many({'type': LotteryType.STAKING, 'pool_id': pool_id}) +async def lottery_for_staking(pool_id: int, address: str) -> NftPrize | None: + lotteries = nft_lotteries.get_many({"type": LotteryType.STAKING, "pool_id": pool_id}) if not lotteries: - logger.info(f'No lotteries found for pools_id {pool_id}') + logger.info(f"No lotteries found for pools_id {pool_id}") return None - logger.debug(f'Lotteries found for pools_id {pool_id}: {lotteries}') + logger.debug(f"Lotteries found for pools_id {pool_id}: {lotteries}") pool_state = flex.db.pool_states.get_one(pool_id=pool_id) if pool_state is None: - logger.error(f'Pool state not found for pool {pool_id}') + logger.error(f"Pool state not found for pool {pool_id}") return None user_state = flex.db.user_states.get_one(address=address) if user_state is None: - logger.error(f'User state not found for address {address}') + logger.error(f"User state not found for address {address}") return None address_stake_micros = user_state.pool_by_address.get(pool_state.address) if address_stake_micros is None: - logger.info(f'No staking found for address {address} in pool {pool_id}') + logger.info(f"No staking found for address {address} in pool {pool_id}") return None lottery = None - for l in lotteries: - if len(l.available_nfts) == 0: - logger.warning(f'NFTS are OVER for lottery {l.name}') - nft_lotteries.remove(l) - logger.info(f'Lottery {l.name} removed') + for candidate in lotteries: + if len(candidate.available_nfts) == 0: + logger.warning(f"NFTS are OVER for lottery {candidate.name}") + nft_lotteries.remove(candidate) + logger.info(f"Lottery {candidate.name} removed") continue - if address_stake_micros >= l.min_amount and (lottery is None or l.probability > lottery.probability): - lottery = l + if address_stake_micros >= candidate.min_amount and ( + lottery is None or candidate.probability > lottery.probability + ): + lottery = candidate if lottery is None: - logger.info(f'No lottery found for address {address} in pool {pool_id}') + logger.info(f"No lottery found for address {address} in pool {pool_id}") return None - logger.info(f'Lottery {lottery.name} for pool {pool_id} and address {address} started') + logger.info(f"Lottery {lottery.name} for pool {pool_id} and address {address} started") - address_draws = lottery_draws.get_many({'wallet': address, 'lottery_name': lottery.name}) - now_timestamp = time.time() - # TODO: optimize the check, get only last timestamp - if len(address_draws) > 0: - last_draw_timestamp = max([d.timestamp for d in address_draws]) - if now_timestamp - last_draw_timestamp < MIN_DRAW_INTERVAL: - logger.info(f'Lottery {lottery.name} for pool {pool_id} and address {address} already drawn recently') - return None - - prize_id = draw_id(lottery) - lottery_draws.create( - LotteryDraw( - lottery_name=lottery.name, - prize=prize_id, - wallet=address, - timestamp=now_timestamp - ) + draw = _get_or_create_staking_draw( + lottery, + address, + now=datetime.now(UTC), ) + if draw is None: + logger.info(f"Lottery {lottery.name} for pool {pool_id} and address {address} already drawn recently") + return None + prize_id = draw.prize - logger.info(f'The prize is {prize_id}') + logger.info(f"The prize is {prize_id}") if prize_id is not None: prize_info = get_nft_prize(lottery, prize_id) - logger.info(f'Prize info: {prize_info}') + logger.info(f"Prize info: {prize_info}") return prize_info return None def send_all_prizes(): - logger.info('Sending all failed NFT prizes...') + logger.info("Sending all failed NFT prizes...") + ensure_lottery_indexes() res = [] sent_count = 0 error_count = 0 - for draw in lottery_draws.get_many({'claimed': False, 'prize': {'$ne': None}}): - info = { - 'wallet': draw.wallet, - 'prize': draw.prize, - 'lottery': draw.lottery_name - } + documents = list( + lottery_draws.collection.find( + { + "claimed": {"$in": [False, None]}, + "prize": {"$ne": None}, + } + ) + ) + for document in documents: + try: + draw, document_id = _backfill_draw_id(document) + except Exception as exc: + document_id = document.get("_id") + info = { + "wallet": document.get("wallet"), + "prize": document.get("prize"), + "lottery": document.get("lottery_name"), + "error": f"lottery draw migration failed: {exc}", + } + if document_id is not None: + lottery_draws.collection.update_one( + {"_id": document_id, "claimed": False}, + {"$set": {"send_error": str(info["error"])[:500]}}, + ) + error_count += 1 + logger.exception("Skipped malformed lottery draw") + res.append(info) + continue + if draw.claimed or draw.prize is None: + continue + info = {"wallet": draw.wallet, "prize": draw.prize, "lottery": draw.lottery_name} + payable_statuses = ( + LotteryPayoutStatus.PENDING, + LotteryPayoutStatus.PREPARED, + LotteryPayoutStatus.UNRESOLVED, + ) + if draw.payout_status not in payable_statuses: + error = ( + "legacy lottery payout requires manual reconciliation" + if draw.payout_status == LotteryPayoutStatus.RECONCILIATION_REQUIRED + else f"lottery payout has non-payable status {draw.payout_status!r}" + ) + info["error"] = error + lottery_draws.collection.update_one( + { + "_id": document_id, + "claimed": False, + "payout_status": draw.payout_status, + }, + {"$set": {"send_error": error}}, + ) + error_count += 1 + logger.warning("Skipped non-payable NFT payout: %s", info) + res.append(info) + continue try: - send_nft(draw.wallet, draw.prize) - info['sent'] = datetime.utcnow() - draw.claimed = True - sent_count += 1 + idempotency_key = f"lottery:{draw.id}" + payout_operation_id = f"nft:{idempotency_key}" + if draw.payout_operation_id not in (None, payout_operation_id): + raise RuntimeError("lottery draw belongs to a different payout operation") + prepared = lottery_draws.collection.update_one( + { + "_id": document_id, + "claimed": False, + "payout_operation_id": {"$in": [None, payout_operation_id]}, + "payout_status": { + "$in": list(payable_statuses), + }, + }, + { + "$set": { + "payout_operation_id": payout_operation_id, + "payout_status": LotteryPayoutStatus.PREPARED, + "send_error": None, + } + }, + ) + if prepared.matched_count != 1: + current = lottery_draws.collection.find_one({"_id": document_id}) + if current is not None and current.get("claimed") is True: + info["already_claimed"] = True + info["txid"] = current.get("payout_txid") + _update_staking_entitlement_status( + LotteryDraw.from_dict(current), + StakingEntitlementStatus.CONFIRMED, + ) + logger.info("NFT payout completed by another worker: %s", info) + res.append(info) + continue + raise RuntimeError("lottery draw changed before payout reservation") + receipt = send_nft( + draw.wallet, + draw.prize, + idempotency_key=idempotency_key, + ) + info["txid"] = receipt.txid + info["sent"] = datetime.now(UTC) + claimed = lottery_draws.collection.find_one_and_update( + { + "_id": document_id, + "claimed": False, + "payout_operation_id": receipt.operation_id, + "payout_status": { + "$ne": LotteryPayoutStatus.RECONCILIATION_REQUIRED, + }, + }, + { + "$set": { + "claimed": True, + "payout_operation_id": receipt.operation_id, + "payout_txid": receipt.txid, + "confirmed_round": receipt.confirmed_round, + "payout_status": LotteryPayoutStatus.CONFIRMED, + "send_error": None, + } + }, + return_document=ReturnDocument.AFTER, + ) + if claimed is None: + claimed = lottery_draws.collection.find_one({"_id": document_id}) + if claimed is None or not claimed.get("claimed") or claimed.get("payout_txid") != receipt.txid: + raise RuntimeError("lottery draw changed while recording its confirmed payout") + info["already_claimed"] = True + else: + sent_count += 1 + _update_staking_entitlement_status( + LotteryDraw.from_dict(claimed), + StakingEntitlementStatus.CONFIRMED, + ) except Exception as e: - info['error'] = str(e) - draw.send_error = str(e) + info["error"] = str(e) + _update_staking_entitlement_status( + draw, + StakingEntitlementStatus.UNRESOLVED, + ) + lottery_draws.collection.update_one( + { + "_id": document_id, + "claimed": False, + "payout_operation_id": payout_operation_id, + "payout_status": {"$ne": LotteryPayoutStatus.RECONCILIATION_REQUIRED}, + }, + { + "$set": { + "payout_status": LotteryPayoutStatus.UNRESOLVED, + "send_error": str(e)[:500], + } + }, + ) error_count += 1 - logger.info(f'Sent NFT: {info}') + logger.info(f"Sent NFT: {info}") - lottery_draws.update(draw) res.append(info) - - return { - 'sent_count': sent_count, - 'error_count': error_count, - 'results': res - } + return {"sent_count": sent_count, "error_count": error_count, "results": res} diff --git a/api/wallet.py b/api/wallet.py index 96899c21..1fdb8679 100644 --- a/api/wallet.py +++ b/api/wallet.py @@ -1,29 +1,37 @@ -import logging - -from algosdk import mnemonic, account -from algosdk.transaction import AssetTransferTxn, wait_for_confirmation +"""Wallet-side asset transfer operations.""" -from blockchain.node import init_algod_client -from env import settings +import logging -cometa_private_key = mnemonic.to_private_key(settings.algo_mnemonic) -cometa_public_key = account.address_from_private_key(cometa_private_key) +from flex.application.asset_transfers import ( + AssetTransferReceipt, + AssetTransferRequest, +) +from flex.application.transfer_runtime import get_asset_transfer_service -algod = init_algod_client() logger = logging.getLogger(__name__) -def send_nft(address: str, nft_id: int, amount: int = 1) -> None: - logger.info(f'Sending {amount} NFT {nft_id} to {address}') - params = algod.suggested_params() - txn = AssetTransferTxn( - sender=cometa_public_key, - sp=params, - receiver=address, - amt=amount, - index=nft_id) - stxn = txn.sign(cometa_private_key) +def send_nft( + address: str, + nft_id: int, + amount: int = 1, + *, + idempotency_key: str, +) -> AssetTransferReceipt: + """Send an NFT once for a stable business operation.""" - txid = algod.send_transaction(stxn) - wait_for_confirmation(algod, txid) - logger.info(f'Sent {amount} NFT {nft_id} to {address} with tx {txid}') + receipt = get_asset_transfer_service().execute( + AssetTransferRequest( + operation_id=f"nft:{idempotency_key}", + receiver=address, + asset_id=nft_id, + amount_micros=amount, + ) + ) + logger.info( + "NFT transfer %s confirmed in round %s as %s", + receipt.operation_id, + receipt.confirmed_round, + receipt.txid, + ) + return receipt diff --git a/app.py b/app.py index a8c9b477..f38cc54e 100644 --- a/app.py +++ b/app.py @@ -423,7 +423,9 @@ async def get_contracts( without_old_pools: bool = True, include_address_pools: Optional[str] = None, ) -> List[ContractInfo]: - contracts = get_contracts_by_type(type) + # The repository result is TTL-cached. Work on a copy so `new_first` + # cannot reverse shared cache state for later requests. + contracts = list(get_contracts_by_type(type)) if include_address_pools and ( include_address_pools in settings.special_addresses diff --git a/docs/architecture/lp-projection.md b/docs/architecture/lp-projection.md new file mode 100644 index 00000000..42ea0e05 --- /dev/null +++ b/docs/architecture/lp-projection.md @@ -0,0 +1,97 @@ +# Crash-Safe LP Projection + +The LP projector treats MongoDB’s LP-state document as the authoritative +idempotency boundary. This is required because the deployed standalone MongoDB +cannot atomically update both `lp_states` and `lp_transactions`. + +## Invariants + +Each scoped transfer has a fixed-width cursor: + +```text +:: +``` + +Events are sorted by that cursor. One `find_one_and_update` atomically applies +the Decimal128 balance delta and advances `last_event_order`. The audit marker +is inserted afterward. If the process dies in that gap, replay sees the cursor, +repairs the missing marker, and does not apply the delta again. + +Before the first write, the complete block batch is checked for its expected +round, deterministic order, duplicate IDs, and conflicting payloads. LP +self-transfer amount legs are discarded as net zero, but their network fee is +still projected. Every pool-paid fee is a separate, stable +`#fee@` ALGO event, so retry cannot combine or repeat it. +Clawback and close semantics fail closed when they affect a tracked LP account. +Indexer rounds, asset IDs, amounts, balances, and fees reject booleans, +coercible strings, negatives, and overflow before any delta is negated or any +repository write begins. + +The repository fails closed when: + +- an event ID exists with different immutable data; +- an event marker exists while its state cursor is still behind; +- a later cursor has passed an unrecorded event; +- a balance would leave the Algorand uint64 range; +- legacy root aliases make an inner event ambiguous; +- duplicate LP token IDs or addresses exist; +- one LP-token ASA is discovered with conflicting immutable pool metadata. + +Balances, issued LP supply, IDs, rounds, positions, fees, and deltas are Python +integers encoded as BSON Decimal128 where MongoDB int64 is insufficient. +Token-token pools keep fee-funding ALGO in +`operational_algo_balance_micros`, separate from the two economic asset +balances. ASA total supply is read directly from Indexer base units and +persisted as a decimal string with `total_supply_source=indexer`. A legacy or +generic rewrite without that provenance is refetched and migrated atomically +before a financial read may use it. A unique asset ID index makes concurrent +creation fail closed. LP-token registration likewise uses a unique Decimal128 +ASA ID plus atomic `get-or-create`, then compares pool ID, assets, address, and +DEX provider against the persisted winner. + +## Snapshots and pricing isolation + +Account balances and `current-round` come from the same Indexer response. An +authoritative snapshot advances the cursor to that round’s end sentinel. A +stale snapshot cannot overwrite a newer event, and conflicting balances for the +same snapshot round require reconciliation. Snapshots update only integer ledger +fields and cursors: they never calculate or publish a price. +Duplicate holdings and out-of-range values are rejected at the Indexer adapter; +issued supply is range-checked before subtraction. Refresh builds an immutable +candidate state, and the repository repeats validation before issuing its +single-document CAS, so a failed snapshot cannot partially mutate in-memory or +persisted balances. + +Indexer account balances are useful for reconciliation but are not proof of a +DEX’s economic reserves. Donations, minimum-balance funding, and protocol excess +can all be present. The ledger projector therefore has no price-publishing +dependency. Public LP prices come from the separately validated `asset_prices` +read model with source and freshness checks. The raw-balance publisher has been +removed; startup deletes its identifiable legacy rows and read paths +independently reject them. A future DEX adapter must derive economic reserves +from verified protocol state before it can become a price source. + +## Worker ordering + +The singleton `sync_states/main` document provides a Mongo lease and a fenced, +compare-and-set round checkpoint. An expired worker can duplicate work but +cannot regress the checkpoint; per-state cursors make that replay convergent. +Cutover advances only through the minimum round fully covered by every LP +cursor. A mid-round event cursor covers the preceding round, not the unfinished +one, so an Indexer lag cannot skip its remaining events. +Repeated deterministic failure uses bounded exponential backoff and terminates +the worker instead of hot-looping. + +`SYNC_LIQUIDITY_POOLS` remains opt-in until an operational snapshot cutover is +performed. `SYNC_STAKING_POOLS` remains disabled because the legacy classifier +does not validate complete Algorand application groups. Enabling it is rejected +at runtime rather than risking financial corruption. + +Standalone MongoDB still cannot make an LP-to-LP transfer visible in two pool +documents simultaneously. Replay guarantees convergence; strict cross-pool +visibility would require a replica-set transaction or a one-document ledger +aggregate. + +`BACKGROUND_LP_PRICES_UPDATE` remains accepted only for deployment +compatibility and cannot restore the removed publisher. Enabling LP ledger sync +does not bypass this boundary. diff --git a/docs/architecture/outbound-asset-transfers.md b/docs/architecture/outbound-asset-transfers.md new file mode 100644 index 00000000..59a29c16 --- /dev/null +++ b/docs/architecture/outbound-asset-transfers.md @@ -0,0 +1,89 @@ +# Replay-Safe Outbound Asset Transfers + +Cometa treats every payout as a durable business operation, not as a retryable +SDK call. Callers provide a stable operation ID such as +`airdrop::

` or `nft:lottery:`. + +Before the first broadcast, the service stores: + +- immutable receiver, ASA, amount, and note; +- the signed Algorand transaction and its transaction ID; +- its first/last valid rounds and execution status. + +A retry loads and rebroadcasts the same signed payload. Before network I/O, the +adapter verifies its signature, transaction ID, sender, receiver, ASA, amount, +note, lease, validity window, fee ceiling, configured network genesis, and +absence of close, clawback, group, or rekey fields. Suggested parameters are +normalized to a flat protocol-minimum fee only after Algod's reported minimum +is at least the protocol floor and both fee ceiling and genesis match policy. +This closes the crash window where Algorand accepted a +transaction but MongoDB did not record the result, prevents node-provided fees +from draining the signer, and stops a corrupted intent from sending another +valid treasury transaction. + +ASA IDs and amounts remain decimal strings for compatibility; validity and +confirmation rounds use BSON Decimal128 codecs. Both representations preserve +the full Algorand uint64 domain rather than relying on signed BSON int64. +Conflicting immutable intent or manifest IDs abort index setup; the application +never deletes financial evidence automatically. + +A deterministic Algorand lease adds defense in depth, but persistence of the +exact signed transaction is the primary idempotency mechanism. If the +transaction expires without confirmed status, the service stops and requires +on-chain reconciliation before any replacement can be authorized. + +## Airdrop invariants + +`send_airdrop` uses exact rational arithmetic and the largest-remainder method, +so recipient base units sum to the declared budget exactly. The first run also +reserves an immutable SHA-256 manifest covering the asset, total, complete +recipient set, allocations, and selected notes. Reusing an `airdrop_id` with a +different manifest fails before any broadcast. + +`send_airdrop` is also safe when invoked directly by an operator script rather +than through FastAPI startup: before any reward read or signing, it fail-closes +on duplicate manifest IDs or reward operation IDs and installs their unique +indexes. Concurrent workers therefore converge on one reward record as well as +one transfer intent. + +Legacy campaigns have no trustworthy complete-recipient manifest. They +therefore fail closed until explicitly reviewed and migrated. Legacy reward +transactions must be confirmed on-chain and match the stored sender, receiver, +ASA, and amount before they can be marked complete. + +Operational retries must always reuse the original operation or airdrop ID. +`AirdropIncompleteError` reports unresolved recipients and transaction IDs; +never invent a replacement ID to bypass reconciliation. +`complete` is terminal: a slower worker cannot overwrite it with `partial`. +Likewise, attempt, submitted, and error updates cannot regress a confirmed +transfer intent. + +## Lottery payouts + +Every new lottery draw receives an immutable ID and starts in `pending`. The +payout moves through `prepared` to `confirmed`; uncertain attempts remain +`unresolved` and reuse the same operation. + +Staking lotteries first claim a single entitlement document keyed by +`(lottery_name, wallet)`. A compare-and-set advances its rolling 24-hour window, +generation, and active draw ID atomically. Prize selection and draw insertion +are replay-repaired from that reservation, so crashes and competing workers +converge on one liability. An unresolved prize blocks the next generation until +reconciliation; no-prize and confirmed generations are terminal. + +Pre-intent legacy draws are different: an old worker may have broadcast and +crashed before recording `claimed`. They receive a deterministic ID but move to +`reconciliation_required`, never directly to payment. Automatic resume is +allowed only when the draw already references the exact durable operation ID, +which proves that any broadcast used the persisted-intent path. Manual +reconciliation must otherwise attach verified on-chain evidence or explicitly +authorize an unpaid draw. + +The exact draw is marked claimed with a conditional update only after +confirmation. Multiple workers may race safely: they resolve to the same +persisted transaction and cannot update another draw. + +Lottery inventory itself is still a separate authority boundary. A future +re-enabled lottery must reserve each one-of-one NFT atomically before exposing +the feature; the public lottery routes remain disabled until that control and +product reconciliation are reviewed. diff --git a/docs/audit/01-audit-architecture-financial-2026-07-19.md b/docs/audit/01-audit-architecture-financial-2026-07-19.md new file mode 100644 index 00000000..9fff7df6 --- /dev/null +++ b/docs/audit/01-audit-architecture-financial-2026-07-19.md @@ -0,0 +1,175 @@ +# Мультиагентный аудит архитектуры и финансовой корректности + +**Дата:** 2026-07-19 +**База:** состояние `main` до ветки `audit/financial-correctness`; история +репозитория затем была очищена, поэтому старые SHA намеренно не используются. + +## Резюме + +Аудит проводился как review production fintech backend, а не косметическая +подготовка профиля. Независимые потоки проверяли выплаты, MongoDB concurrency, +LP accounting, price provenance, Algorand boundaries, supply chain, Git history, +CI и публичную документацию. Повторяющиеся находки перепроверялись тестами и +исправлялись только после воспроизведения. + +Ключевой результат: денежные операции используют integer base units, +неизменяемые business IDs, persisted signed intents и одно-документные CAS. +Непроверенные источники цены и staking projection остаются fail-closed. + +| Граница | Контроль | +| --- | --- | +| Выплата | exact allocation, durable intent, bounded signer fee, on-chain reconciliation | +| Staking draw | CAS entitlement, одна generation, crash-repair того же draw ID | +| LP ledger | strict uint64 input, ordered cursor, marker repair, fenced round lease | +| Цена | provenance + freshness + clock-skew guard; raw pool balances запрещены | +| Проверка | Python 3.12/3.14, 372 fast tests, 26 real-Mongo tests, 83.19% focused coverage | + +MongoDB гарантирует атомарность одной операции над одним документом; поэтому +денежные инварианты размещены внутри одного CAS aggregate, а не blind +read-modify-write. Междокументная одновременная видимость потребует replica-set +transactions. См. [atomicity](https://www.mongodb.com/docs/manual/core/write-operations-atomicity/) +и [transactions](https://www.mongodb.com/docs/manual/core/transactions/). + +## Ранжированные находки + +### 1. Critical — секрет оставался в достижимой Git history — исправлено в репозитории + +**Почему:** удаление файла из HEAD не отзывает значение и не удаляет старый +blob; его можно восстановить из любого достижимого commit. + +**Исправление:** чувствительные исторические пути удалены из всех публикуемых +refs через `git-filter-repo`; old-object и fresh-clone проверки входят в +процедуру публикации. CI сканирует все публикуемые refs digest-pinned +TruffleHog (`.github/workflows/ci.yml:84`). Ротация ранее использованного +credential остаётся обязательным внешним действием владельца. + +### 2. Critical — double-pay window, float allocation и signer fee — исправлено + +**Почему:** crash после broadcast до Mongo update допускал повторную отправку; +float shares не сохраняли бюджет; доверенный Algod мог предложить чрезмерную +комиссию. + +**Исправление:** immutable signed intent сохраняется до первого broadcast и +reconciled по txid (`flex/application/asset_transfers.py:227`). Airdrop заранее +фиксирует полный manifest, а largest-remainder allocation сохраняет каждую base +unit. Confirmed intent и complete manifest терминальны. Gateway до подписи и +повторно перед broadcast проверяет genesis, signature, lease и configured fee +floor/ceiling (`flex/blockchain/asset_transfers.py:50`). Algorand minimum fee описан в +[официальной документации](https://dev.algorand.co/concepts/transactions/fees/). + +### 3. Critical — staking lottery создавала две независимые liabilities — исправлено + +**Почему:** прежний `read recent draws → insert` позволял двум workers создать +разные draw IDs; idempotency выплаты не объединяет разные business operations. + +**Исправление:** один entitlement на `(lottery_name, wallet)` атомарно меняет +`next_eligible_at`, `generation` и active draw. Crash recovery продолжает тот же +draw и replay prize selection (`api/nft_lottery.py:275`). Гонка 32 вызовов и crash +между entitlement/draw writes проверены на настоящем MongoDB. One-of-one NFT +inventory ещё требует отдельной атомарной reservation, поэтому публичные +lottery routes остаются disabled. + +### 4. Critical — LP replay мог потерять или повторить баланс — исправлено + +**Почему:** blind RMW, stale snapshots и истёкший worker нарушали exactly-once +projection. + +**Исправление:** per-state cursor CAS, marker-last repair и re-read после +конкурентного marker (`flex/db/lp_projection.py:52`). Round commit fenced +неистёкшим lease. Fee pool-sender — отдельное replay-safe ALGO событие; +token-token operational ALGO не смешивается с economic reserves. Indexer +amounts, IDs, rounds, duplicates и snapshots проверяются до negation и Mongo +write (`flex/sync_pools.py:78`, `flex/data/lp_states.py:113`). + +### 5. Critical — raw account balance мог манипулировать LP price — исправлено + +**Почему:** donation, minimum-balance funding или protocol excess не являются +экономическими reserves DEX. + +**Исправление:** LP projector стал ledger-only; raw-balance publisher удалён. +Startup очищает обе legacy provenance signatures, а readers независимо их +отклоняют. LP registry прекращает весь refresh, если не классифицирован хотя бы +один farm stake token. Provider observation за пределами clock-skew budget +отклоняется до записи; уже сохранённое далёкое future value считается invalid +и заменяется корректной котировкой. Новый источник допускается только после +DEX-specific app state verification. Canonical supply доверяется лишь с +`total_supply_source=indexer`. + +### 6. High — shared browser key не является пользовательской авторизацией — открыто + +**Почему:** один `X-API-Key` не доказывает wallet ownership и не разделяет роли +(`core/auth.py:8`, `app.py:323`). + +**Конкретный фикс:** wallet-signature challenge с nonce, expiry и replay table; +server-to-server maintenance role с отдельным secret и audit log. До этого +shared key считается compatibility/rate-control механизмом. + +### 7. High — blocking persistence остаётся в async routes — открыто + +**Почему:** sync PyMongo/provider calls в event loop увеличивают tail latency +всех запросов при деградации Mongo или DEX (`app.py:418`, `app.py:529`). + +**Конкретный фикс:** async repository ports с timeout/cancellation; переходно — +`asyncio.to_thread` вокруг целого repository call и saturation test. + +### 8. High — legacy staking classifier не доказывает transaction group — mitigated + +**Почему:** соседний transfer без проверки app ID, selector и group order не +доказывает stake. + +**Текущий контроль:** `SYNC_STAKING_POOLS=false`, а включение отклоняется. +Следующий фикс — типизированный parser полной Algorand group и adversarial +fixtures. Это отдельно от исправленного lottery entitlement. + +### 9. High — mocks не доказывали Mongo/BSON invariants — исправлено + +**Почему:** fake collections не воспроизводят Decimal128 comparison, unique +index races, `$inc` promotion и `find_one_and_update`. + +**Исправление:** digest-pinned MongoDB CI job проверяет 26 integration scenarios: +CAS replay, marker repair, uint64 max, legacy int64 promotion, terminal payout +races, uniqueness, staking entitlement и lease fencing +(`.github/workflows/ci.yml:106`). Stable `python` context агрегирует Python +matrix, container configuration/image scan, Mongo и all-ref secret scan +(`.github/workflows/ci.yml:133`). + +### 10. Medium — runtime/container proof остаётся неполным — открыто + +**Почему:** production smoke заменяет entrypoint shell-командой, а stateful +Mongo/Algod image rollout требует отдельной data-format rehearsal. + +**Конкретный фикс:** disposable stack должен запустить штатный entrypoint, +дождаться health, проверить `/status`, SIGTERM и logs. Production digests +фиксировать только после backup/restore и downgrade rehearsal. + +## Milestones + +1. **M0 — money safety (готово):** findings 2–5; точные суммы, terminal states, + fee/network policy, CAS entitlements, strict uint64 boundaries. +2. **M1 — proof and history (готово в коде):** findings 1 и 9; Python + 3.12/3.14, real Mongo, immutable scanner, clean-history procedure. +3. **M2 — authority (следующий):** finding 6, atomic contract registration, + transactional outbox и one-of-one NFT inventory. +4. **M3 — runtime (следующий):** findings 7, 8 и 10; затем SLO, metrics и + controlled deploy rehearsal. + +Python 3.14.6 проверяется как forward-compatibility gate и является текущим +maintenance release ([Python.org](https://www.python.org/downloads/release/python-3146/)); +production-equivalent environment остаётся на Python 3.12. + +## Воспроизводимость + +```bash +make sync +make quality + +# Только disposable MongoDB, никогда production: +MONGODB_TEST_URI=mongodb://127.0.0.1:27017 \ + pipenv run pytest tests/integration -m integration -v +``` + +Отдельные чистые environments Python 3.12.8 и 3.14.6 дали одинаковый результат: +372 passed, 26 integration skipped. Все 26 integration tests прошли на +standalone MongoDB отдельно. Форматирование не включалось в findings: его +обеспечивает Ruff; review приоритизировал correctness, security, data integrity +и recovery semantics. diff --git a/env.py b/env.py index 217df01e..60dde2cc 100644 --- a/env.py +++ b/env.py @@ -16,6 +16,11 @@ class Settings(BaseSettings): algod_address: str algod_token: str algo_indexer_address: str + outbound_asset_transfer_max_fee_microalgos: int = Field( + default=1_000, + ge=1_000, + le=1_000_000, + ) server_port: int workers_num: int @@ -57,12 +62,18 @@ class Settings(BaseSettings): background_user_pools_update: bool = False background_pools_update: bool = False background_asset_prices_update: bool = True # Enable background update of asset prices + # Retained only so existing deployments can roll forward. The raw-balance + # publisher has been removed and this flag cannot enable LP pricing. + background_lp_prices_update: bool = False asset_price_update_batch_size: int = Field(default=10, gt=0) asset_price_api_call_delay: float = Field(default=1, ge=0) asset_price_batch_delay: float = Field(default=2.0, ge=0) sync_new_pools: bool = True sync_liquidity_pools: bool = False + # Legacy stake projection does not yet validate complete application groups. + # Keep it fail-closed until the authoritative event classifier lands. + sync_staking_pools: bool = False update_contract_caches: bool = True update_contracts_chunk_size: int = Field(default=10, ge=1, le=100) @@ -74,6 +85,8 @@ class Settings(BaseSettings): old_pool_end_date_days_ago: int = 30 sync_lag_max_rounds: int = 1000 # 1 hour sync_behind_seconds_threshold: int = 60 + sync_round_max_attempts: int = Field(default=5, ge=1, le=100) + sync_retry_max_seconds: float = Field(default=30, gt=0, le=300) reset_and_resync_pool_states: bool = False @@ -82,7 +95,6 @@ class Settings(BaseSettings): asset_prices_ttl: int = Field(default=120, gt=0) # 2 minutes. asset_prices_max_stale: int = Field(default=3600, gt=0) asset_prices_update_interval: int = 60 # Run the background update every 60 seconds - lp_prices_update_interval: int = 300 # LP pricing via algod every 5 minutes lp_token_prices_ttl: int = 30 total_tvl_ttl: int = 30 diff --git a/flex/api.py b/flex/api.py index 31f12009..b2b85e4e 100644 --- a/flex/api.py +++ b/flex/api.py @@ -287,8 +287,9 @@ async def _fetch_asset_safe(aid): except Exception as e: logger.error(f"Batch price creation failed: {e}") - # LP prices are now in asset_prices collection (populated by background worker). - # Keep lp_states key as empty dict for frontend compatibility. + # This route only consumes independently validated stored observations. + # No active publisher derives LP prices from pool-account balances. + # Keep lp_states empty for frontend compatibility. lp_states_dict = {} assets_dict = {a.id: a.to_details().to_dict() for a in assets_list} diff --git a/flex/application/asset_transfers.py b/flex/application/asset_transfers.py new file mode 100644 index 00000000..e8774eea --- /dev/null +++ b/flex/application/asset_transfers.py @@ -0,0 +1,418 @@ +"""Idempotent application service for outbound Algorand asset transfers.""" + +from dataclasses import dataclass +from typing import Protocol + +from flex.db.model.transfers import AssetTransferIntent +from flex.domain.algorand import ( + MAX_ALGORAND_UINT, + require_algorand_uint64, +) + +MAX_NOTE_BYTES = 1_000 +MAX_OPERATION_ID_BYTES = 200 + + +class AssetTransferError(RuntimeError): + """Base class for outbound transfer failures.""" + + +class InvalidAssetTransferError(ValueError): + """Raised before persistence when a transfer request is invalid.""" + + +class AssetTransferConflictError(AssetTransferError): + """Raised when an idempotency key is reused for a different transfer.""" + + +class AssetTransferPendingError(AssetTransferError): + """Raised when broadcast outcome is ambiguous and must be reconciled.""" + + def __init__(self, operation_id: str, txid: str) -> None: + super().__init__(f"transfer {operation_id!r} is unresolved; reconcile transaction {txid}") + self.operation_id = operation_id + self.txid = txid + + +class AssetTransferExpiredError(AssetTransferError): + """Raised when an unconfirmed persisted transaction is no longer valid.""" + + def __init__(self, operation_id: str, txid: str) -> None: + super().__init__( + f"transfer {operation_id!r} expired unconfirmed; verify transaction {txid} on-chain before replacement" + ) + self.operation_id = operation_id + self.txid = txid + + +class TransferStatus: + PREPARED = "prepared" + SUBMITTED = "submitted" + CONFIRMED = "confirmed" + + +@dataclass(frozen=True, slots=True) +class AssetTransferRequest: + operation_id: str + receiver: str + asset_id: int + amount_micros: int + note: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.operation_id, str) or not self.operation_id.strip(): + raise InvalidAssetTransferError("operation_id must be a non-empty string") + if len(self.operation_id.encode()) > MAX_OPERATION_ID_BYTES: + raise InvalidAssetTransferError("operation_id is too long") + if not isinstance(self.receiver, str) or not self.receiver.strip(): + raise InvalidAssetTransferError("receiver must be a non-empty string") + if isinstance(self.asset_id, bool) or not isinstance(self.asset_id, int): + raise InvalidAssetTransferError("asset_id must be an integer") + if not 0 < self.asset_id <= MAX_ALGORAND_UINT: + raise InvalidAssetTransferError("asset_id is outside the Algorand uint64 range") + if isinstance(self.amount_micros, bool) or not isinstance(self.amount_micros, int): + raise InvalidAssetTransferError("amount_micros must be an integer") + if not 0 < self.amount_micros <= MAX_ALGORAND_UINT: + raise InvalidAssetTransferError("amount_micros is outside the Algorand uint64 range") + if self.note is not None: + if not isinstance(self.note, str): + raise InvalidAssetTransferError("note must be a string") + if len(self.note.encode()) > MAX_NOTE_BYTES: + raise InvalidAssetTransferError("note exceeds Algorand's 1000-byte limit") + + +@dataclass(frozen=True, slots=True) +class PreparedAssetTransfer: + signed_transaction: str + txid: str + first_valid_round: int + last_valid_round: int + + def __post_init__(self) -> None: + try: + require_algorand_uint64( + self.first_valid_round, + "first_valid_round", + ) + require_algorand_uint64( + self.last_valid_round, + "last_valid_round", + ) + except ValueError as exc: + raise InvalidAssetTransferError(str(exc)) from exc + if self.first_valid_round > self.last_valid_round: + raise InvalidAssetTransferError( + "first_valid_round cannot exceed last_valid_round", + ) + + +@dataclass(frozen=True, slots=True) +class AssetTransferReceipt: + operation_id: str + txid: str + confirmed_round: int + already_confirmed: bool + + def __post_init__(self) -> None: + try: + require_algorand_uint64( + self.confirmed_round, + "confirmed_round", + ) + except ValueError as exc: + raise InvalidAssetTransferError(str(exc)) from exc + + +@dataclass(frozen=True, slots=True) +class ConfirmedAssetTransfer: + txid: str + sender: str + receiver: str + asset_id: int + amount_micros: int + confirmed_round: int + + def __post_init__(self) -> None: + try: + require_algorand_uint64( + self.asset_id, + "asset_id", + positive=True, + ) + require_algorand_uint64( + self.amount_micros, + "amount_micros", + positive=True, + ) + require_algorand_uint64( + self.confirmed_round, + "confirmed_round", + ) + except ValueError as exc: + raise InvalidAssetTransferError(str(exc)) from exc + + +@dataclass(frozen=True, slots=True) +class TransferReconciliation: + operation_id: str + txid: str + status: str + confirmed_round: int | None + + +class AssetTransferIntentRepository(Protocol): + def get(self, operation_id: str) -> AssetTransferIntent | None: ... + + def reserve(self, intent: AssetTransferIntent) -> AssetTransferIntent: ... + + def record_attempt(self, operation_id: str, txid: str) -> AssetTransferIntent: ... + + def mark_submitted(self, operation_id: str, txid: str) -> AssetTransferIntent: ... + + def mark_confirmed( + self, + operation_id: str, + txid: str, + confirmed_round: int, + ) -> AssetTransferIntent: ... + + def record_error( + self, + operation_id: str, + txid: str, + error: str, + ) -> AssetTransferIntent: ... + + +class AssetTransferGateway(Protocol): + def prepare(self, request: AssetTransferRequest) -> PreparedAssetTransfer: ... + + def broadcast( + self, + prepared: PreparedAssetTransfer, + request: AssetTransferRequest, + ) -> str: ... + + def wait_for_confirmation(self, txid: str) -> int: ... + + def lookup_confirmed_round(self, txid: str) -> int | None: ... + + def lookup_confirmed_transfer(self, txid: str) -> ConfirmedAssetTransfer | None: ... + + def current_round(self) -> int: ... + + +@dataclass(slots=True) +class AssetTransferService: + repository: AssetTransferIntentRepository + gateway: AssetTransferGateway + + def validate(self, request: AssetTransferRequest) -> None: + """Reject an idempotency-key conflict without touching the network.""" + + intent = self.repository.get(request.operation_id) + if intent is not None: + self._assert_same_transfer(intent, request) + + def lookup_confirmed_round(self, txid: str) -> int | None: + """Look up a legacy transaction without creating a new intent.""" + + return self.gateway.lookup_confirmed_round(txid) + + def lookup_confirmed_transfer(self, txid: str) -> ConfirmedAssetTransfer | None: + """Return authoritative on-chain fields for legacy reconciliation.""" + + return self.gateway.lookup_confirmed_transfer(txid) + + def execute(self, request: AssetTransferRequest) -> AssetTransferReceipt: + intent = self.repository.get(request.operation_id) + if intent is None: + prepared = self.gateway.prepare(request) + intent = self.repository.reserve( + AssetTransferIntent( + id=request.operation_id, + receiver=request.receiver, + asset_id=str(request.asset_id), + amount_micros=str(request.amount_micros), + note=request.note, + signed_transaction=prepared.signed_transaction, + txid=prepared.txid, + first_valid_round=prepared.first_valid_round, + last_valid_round=prepared.last_valid_round, + status=TransferStatus.PREPARED, + ) + ) + + self._assert_same_transfer(intent, request) + if intent.status == TransferStatus.CONFIRMED: + return self._confirmed_receipt(intent, already_confirmed=True) + + if self.gateway.current_round() > intent.last_valid_round: + try: + confirmed_round = self.gateway.lookup_confirmed_round(intent.txid) + except Exception as exc: + intent = self.repository.record_error( + intent.id, + intent.txid, + f"expired transaction lookup unavailable after {type(exc).__name__}", + ) + if intent.status == TransferStatus.CONFIRMED: + return self._confirmed_receipt(intent, already_confirmed=True) + raise AssetTransferExpiredError(intent.id, intent.txid) from exc + if confirmed_round is not None: + intent = self.repository.mark_confirmed( + intent.id, + intent.txid, + confirmed_round, + ) + return self._confirmed_receipt(intent, already_confirmed=True) + + intent = self.repository.record_error( + intent.id, + intent.txid, + "signed transaction expired before confirmation; manual reconciliation required", + ) + if intent.status == TransferStatus.CONFIRMED: + return self._confirmed_receipt(intent, already_confirmed=True) + raise AssetTransferExpiredError(intent.id, intent.txid) + + intent = self.repository.record_attempt(intent.id, intent.txid) + if intent.status == TransferStatus.CONFIRMED: + return self._confirmed_receipt(intent, already_confirmed=True) + broadcast_error: Exception | None = None + try: + returned_txid = self.gateway.broadcast( + PreparedAssetTransfer( + signed_transaction=intent.signed_transaction, + txid=intent.txid, + first_valid_round=intent.first_valid_round, + last_valid_round=intent.last_valid_round, + ), + request, + ) + if returned_txid != intent.txid: + raise AssetTransferError("Algorand node returned a different transaction ID") + intent = self.repository.mark_submitted(intent.id, intent.txid) + if intent.status == TransferStatus.CONFIRMED: + return self._confirmed_receipt(intent, already_confirmed=True) + except Exception as exc: + # A transport failure can happen after the node accepted the + # transaction. Confirmation of the persisted txid is authoritative. + broadcast_error = exc + + try: + confirmed_round = self.gateway.wait_for_confirmation(intent.txid) + except Exception as exc: + failure_kind = type(broadcast_error or exc).__name__ + intent = self.repository.record_error( + intent.id, + intent.txid, + f"confirmation unresolved after {failure_kind}", + ) + if intent.status == TransferStatus.CONFIRMED: + return self._confirmed_receipt(intent, already_confirmed=True) + raise AssetTransferPendingError(intent.id, intent.txid) from exc + + intent = self.repository.mark_confirmed( + intent.id, + intent.txid, + confirmed_round, + ) + return self._confirmed_receipt(intent, already_confirmed=False) + + def reconcile(self, operation_id: str) -> TransferReconciliation: + intent = self.repository.get(operation_id) + if intent is None: + raise AssetTransferError(f"transfer {operation_id!r} does not exist") + if intent.status == TransferStatus.CONFIRMED: + receipt = self._confirmed_receipt(intent, already_confirmed=True) + return TransferReconciliation( + operation_id=receipt.operation_id, + txid=receipt.txid, + status=TransferStatus.CONFIRMED, + confirmed_round=receipt.confirmed_round, + ) + + confirmed_round = self.gateway.lookup_confirmed_round(intent.txid) + if confirmed_round is not None: + confirmed = self.repository.mark_confirmed( + intent.id, + intent.txid, + confirmed_round, + ) + return TransferReconciliation( + operation_id=confirmed.id, + txid=confirmed.txid, + status=TransferStatus.CONFIRMED, + confirmed_round=confirmed.confirmed_round, + ) + + current_round = self.gateway.current_round() + intent = self._reload_reconciliation_intent(intent) + if intent.status == TransferStatus.CONFIRMED: + receipt = self._confirmed_receipt(intent, already_confirmed=True) + return TransferReconciliation( + operation_id=receipt.operation_id, + txid=receipt.txid, + status=TransferStatus.CONFIRMED, + confirmed_round=receipt.confirmed_round, + ) + + status = "expired_unconfirmed" if current_round > intent.last_valid_round else intent.status + return TransferReconciliation( + operation_id=intent.id, + txid=intent.txid, + status=status, + confirmed_round=None, + ) + + def _reload_reconciliation_intent( + self, + expected: AssetTransferIntent, + ) -> AssetTransferIntent: + current = self.repository.get(expected.id) + if current is None: + raise AssetTransferError(f"transfer {expected.id!r} disappeared during reconciliation") + if current.txid != expected.txid: + raise AssetTransferConflictError( + f"transfer {expected.id!r} changed transaction from " + f"{expected.txid!r} to {current.txid!r} during reconciliation" + ) + return current + + @staticmethod + def _assert_same_transfer( + intent: AssetTransferIntent, + request: AssetTransferRequest, + ) -> None: + persisted = ( + intent.receiver, + intent.asset_id_int, + intent.amount_micros_int, + intent.note, + ) + requested = ( + request.receiver, + request.asset_id, + request.amount_micros, + request.note, + ) + if persisted != requested: + raise AssetTransferConflictError( + f"idempotency key {request.operation_id!r} already belongs to a different transfer" + ) + + @staticmethod + def _confirmed_receipt( + intent: AssetTransferIntent, + *, + already_confirmed: bool, + ) -> AssetTransferReceipt: + if intent.confirmed_round is None: + raise AssetTransferError("confirmed transfer is missing confirmed_round") + return AssetTransferReceipt( + operation_id=intent.id, + txid=intent.txid, + confirmed_round=intent.confirmed_round, + already_confirmed=already_confirmed, + ) diff --git a/flex/application/price_refresh.py b/flex/application/price_refresh.py index 67ac245a..190b5b2c 100644 --- a/flex/application/price_refresh.py +++ b/flex/application/price_refresh.py @@ -6,7 +6,13 @@ import httpx -from flex.domain.pricing import DecimalInput, PriceQuote, PriceSource, PricingError +from flex.domain.pricing import ( + DecimalInput, + PriceQuote, + PriceSource, + PricingError, + validate_observation_timestamp, +) from flex.meta_error import MetaError from flex.providers import vestige @@ -76,6 +82,7 @@ def validate_provider_quote( stale_after=fresh_for, observed_round=observed_round, ) + validate_observation_timestamp(quote.observed_at) quote.to_legacy_floats() return quote except PriceRefreshError: diff --git a/flex/application/transfer_runtime.py b/flex/application/transfer_runtime.py new file mode 100644 index 00000000..184b9a55 --- /dev/null +++ b/flex/application/transfer_runtime.py @@ -0,0 +1,27 @@ +"""Composition root for outbound asset transfer infrastructure.""" + +from functools import lru_cache + +from env import settings +from flex import db +from flex.application.asset_transfers import AssetTransferService +from flex.blockchain.asset_transfers import AlgorandAssetTransferGateway +from flex.blockchain.base import algod_client, cometa_private_key, cometa_public_key, indexer_client +from flex.db.asset_transfer_intents import MongoAssetTransferIntentRepository + + +@lru_cache(maxsize=1) +def get_asset_transfer_service() -> AssetTransferService: + repository = MongoAssetTransferIntentRepository( + db.asset_transfer_intents.mongodb_collection, + ) + repository.ensure_indexes() + gateway = AlgorandAssetTransferGateway( + algod=algod_client, + indexer=indexer_client, + sender=cometa_public_key, + private_key=cometa_private_key, + network=settings.algo_network, + max_fee_microalgos=settings.outbound_asset_transfer_max_fee_microalgos, + ) + return AssetTransferService(repository=repository, gateway=gateway) diff --git a/flex/blockchain/asset_transfers.py b/flex/blockchain/asset_transfers.py new file mode 100644 index 00000000..8e3f914f --- /dev/null +++ b/flex/blockchain/asset_transfers.py @@ -0,0 +1,279 @@ +"""Algorand adapter for preparing and submitting persisted asset transfers.""" + +import base64 +import hmac +from dataclasses import dataclass +from hashlib import sha256 +from typing import Any + +from algosdk import account, constants, encoding, transaction +from algosdk.error import AlgodHTTPError +from algosdk.v2client.algod import AlgodClient +from algosdk.v2client.indexer import IndexerClient + +from flex.application.asset_transfers import ( + AssetTransferRequest, + ConfirmedAssetTransfer, + InvalidAssetTransferError, + PreparedAssetTransfer, +) +from flex.domain.algorand import require_algorand_uint64 + +TX_WAIT_ROUNDS = 4 +LEASE_DOMAIN = b"cometa-asset-transfer:v1:" +NETWORK_GENESIS = { + "mainnet": ( + "mainnet-v1.0", + "wGHE2Pwdvd7S12BL5FaOP20EGYesN73ktiC1qzkkit8=", + ), + "testnet": ( + "testnet-v1.0", + "SGO1GKSzyE7IEPItTxCByw9x8FmnrCDexi9/cOUJOiI=", + ), + "betanet": ( + "betanet-v1.0", + "mFgazF+2uRS1tMiL9dsj01hJGySEmPN28B/TjjvpVW0=", + ), +} + + +@dataclass(slots=True) +class AlgorandAssetTransferGateway: + algod: AlgodClient + indexer: IndexerClient + sender: str + private_key: str + network: str + max_fee_microalgos: int + wait_rounds: int = TX_WAIT_ROUNDS + + def prepare(self, request: AssetTransferRequest) -> PreparedAssetTransfer: + if not encoding.is_valid_address(request.receiver): + raise InvalidAssetTransferError("receiver is not a valid Algorand address") + + params = self.algod.suggested_params() + self._apply_fee_and_network_policy(params) + unsigned_transaction = transaction.AssetTransferTxn( + sender=self.sender, + sp=params, + receiver=request.receiver, + amt=request.amount_micros, + index=request.asset_id, + note=request.note.encode() if request.note is not None else None, + lease=sha256(LEASE_DOMAIN + request.operation_id.encode()).digest(), + ) + signed_transaction = unsigned_transaction.sign(self.private_key) + return PreparedAssetTransfer( + signed_transaction=encoding.msgpack_encode(signed_transaction), + txid=unsigned_transaction.get_txid(), + first_valid_round=unsigned_transaction.first_valid_round, + last_valid_round=unsigned_transaction.last_valid_round, + ) + + def broadcast( + self, + prepared: PreparedAssetTransfer, + request: AssetTransferRequest, + ) -> str: + decoded = encoding.msgpack_decode(prepared.signed_transaction) + if not isinstance(decoded, transaction.SignedTransaction): + raise InvalidAssetTransferError("persisted payload is not a signed transaction") + self._validate_persisted_transaction(decoded, prepared, request) + return self.algod.send_transaction(decoded) + + def _validate_persisted_transaction( + self, + signed: transaction.SignedTransaction, + prepared: PreparedAssetTransfer, + request: AssetTransferRequest, + ) -> None: + txn = signed.transaction + expected_note = request.note.encode() if request.note is not None else None + expected_lease = sha256(LEASE_DOMAIN + request.operation_id.encode()).digest() + expected_authorizer = account.address_from_private_key(self.private_key) + expected_signer = None if expected_authorizer == self.sender else expected_authorizer + expected_genesis_id, expected_genesis_hash = self._network_genesis() + + expected_fields = ( + self.sender, + request.receiver, + request.asset_id, + request.amount_micros, + expected_note, + expected_lease, + prepared.first_valid_round, + prepared.last_valid_round, + None, + None, + None, + None, + expected_genesis_id, + expected_genesis_hash, + ) + observed_fields = ( + txn.sender, + getattr(txn, "receiver", None), + getattr(txn, "index", None), + getattr(txn, "amount", None), + txn.note, + txn.lease, + txn.first_valid_round, + txn.last_valid_round, + getattr(txn, "close_assets_to", None), + getattr(txn, "revocation_target", None), + txn.group, + txn.rekey_to, + txn.genesis_id, + txn.genesis_hash, + ) + if not isinstance(txn, transaction.AssetTransferTxn) or observed_fields != expected_fields: + raise InvalidAssetTransferError("persisted transaction fields do not match the transfer intent") + self._validate_fee(txn.fee) + if txn.get_txid() != prepared.txid: + raise InvalidAssetTransferError("persisted transaction ID does not match the transfer intent") + if signed.authorizing_address != expected_signer: + raise InvalidAssetTransferError("persisted transaction has an unexpected authorizing address") + + expected_signature = base64.b64encode(txn.raw_sign(self.private_key)).decode() + if not isinstance(signed.signature, str) or not hmac.compare_digest( + signed.signature, + expected_signature, + ): + raise InvalidAssetTransferError("persisted transaction signature is invalid") + + def _network_genesis(self) -> tuple[str, str]: + try: + return NETWORK_GENESIS[self.network] + except KeyError as exc: + raise InvalidAssetTransferError( + f"unsupported outbound transfer network {self.network!r}", + ) from exc + + def _apply_fee_and_network_policy( + self, + params: transaction.SuggestedParams, + ) -> None: + expected_genesis_id, expected_genesis_hash = self._network_genesis() + if params.gen != expected_genesis_id or params.gh != expected_genesis_hash: + raise InvalidAssetTransferError( + "Algod suggested parameters do not match the configured network", + ) + min_fee = params.min_fee + if isinstance(min_fee, bool) or not isinstance(min_fee, int) or min_fee <= 0: + raise InvalidAssetTransferError("Algod returned an invalid minimum fee") + self._validate_fee(min_fee) + # Payouts are single outer transfers. Paying exactly the protocol + # minimum is deterministic and prevents a node-provided per-byte fee + # from silently draining the signer. Congestion therefore fails closed. + params.flat_fee = True + params.fee = min_fee + + def _validate_fee(self, fee: object) -> None: + if isinstance(fee, bool) or not isinstance(fee, int): + raise InvalidAssetTransferError("outbound transaction fee is not an integer") + if fee < constants.min_txn_fee: + raise InvalidAssetTransferError( + "outbound transaction fee is below the Algorand protocol minimum", + ) + if fee > self.max_fee_microalgos: + raise InvalidAssetTransferError( + "outbound transaction fee exceeds the configured payout policy", + ) + + def wait_for_confirmation(self, txid: str) -> int: + response = transaction.wait_for_confirmation( + self.algod, + txid, + self.wait_rounds, + ) + return self._confirmed_round(response) + + def lookup_confirmed_round(self, txid: str) -> int | None: + try: + pending = self.algod.pending_transaction_info(txid) + except AlgodHTTPError as exc: + if exc.code != 404: + raise + else: + confirmed_round = self._optional_confirmed_round(pending) + if confirmed_round is not None: + return confirmed_round + + # The indexer is the durable lookup path after algod's pending window. + observed = self.lookup_confirmed_transfer(txid) + return observed.confirmed_round if observed is not None else None + + def lookup_confirmed_transfer(self, txid: str) -> ConfirmedAssetTransfer | None: + response = self.indexer.transaction(txid) + transaction_info = response.get("transaction") + if not isinstance(transaction_info, dict): + raise RuntimeError("Algorand indexer response has no transaction") + confirmed_round = self._optional_confirmed_round(transaction_info) + if confirmed_round is None: + return None + + sender = transaction_info.get("sender") + asset_transfer = transaction_info.get("asset-transfer-transaction") + if not isinstance(sender, str) or not encoding.is_valid_address(sender): + raise RuntimeError("Algorand indexer response has an invalid sender") + if not isinstance(asset_transfer, dict): + raise RuntimeError("Algorand transaction is not an asset transfer") + + receiver = asset_transfer.get("receiver") + asset_id = asset_transfer.get("asset-id") + amount_micros = asset_transfer.get("amount") + if not isinstance(receiver, str) or not encoding.is_valid_address(receiver): + raise RuntimeError("Algorand indexer response has an invalid receiver") + try: + asset_id = require_algorand_uint64( + asset_id, + "Algorand indexer asset ID", + positive=True, + ) + amount_micros = require_algorand_uint64( + amount_micros, + "Algorand indexer transfer amount", + positive=True, + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + + return ConfirmedAssetTransfer( + txid=txid, + sender=sender, + receiver=receiver, + asset_id=asset_id, + amount_micros=amount_micros, + confirmed_round=confirmed_round, + ) + + def current_round(self) -> int: + status = self.algod.status() + current_round = status.get("last-round") + try: + return require_algorand_uint64( + current_round, + "Algorand node current round", + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + + @staticmethod + def _confirmed_round(response: dict[str, Any]) -> int: + confirmed_round = AlgorandAssetTransferGateway._optional_confirmed_round(response) + if confirmed_round is None: + raise RuntimeError("Algorand confirmation response has no confirmed round") + return confirmed_round + + @staticmethod + def _optional_confirmed_round(response: dict[str, Any]) -> int | None: + confirmed_round = response.get("confirmed-round") + if confirmed_round is None or confirmed_round == 0: + return None + try: + return require_algorand_uint64( + confirmed_round, + "Algorand confirmed round", + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc diff --git a/flex/blockchain/info.py b/flex/blockchain/info.py index d196c323..6917b76e 100644 --- a/flex/blockchain/info.py +++ b/flex/blockchain/info.py @@ -1,22 +1,33 @@ import asyncio import logging +from dataclasses import dataclass from aiocache import cached from blockchain.node import get_current_round as _sync_get_current_round -from flex.blockchain.base import indexer_client, algod_client -from flex.db.model.blockchain import Asset +from flex.blockchain.base import algod_client, indexer_client +from flex.db.model.blockchain import TOTAL_SUPPLY_SOURCE_INDEXER, Asset +from flex.domain.algorand import require_algorand_uint64 logger = logging.getLogger(__name__) + +@dataclass(frozen=True, slots=True) +class AssetBalanceSnapshot: + balances: dict[int, int] + observed_round: int + + ALGO_ASSET = Asset( id=0, decimals=6, - name='Algorand', - unit_name='ALGO', - creator='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ', - reserve='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ', - total_supply=10_000_000_000 + name="Algorand", + unit_name="ALGO", + creator="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ", + reserve="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ", + total_supply=10_000_000_000, + total_supply_micros=10_000_000_000_000_000, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, ) @@ -31,31 +42,85 @@ async def fetch_asset(asset_id: int) -> Asset: return ALGO_ASSET data = await _run_sync(indexer_client.asset_info, asset_id) - params = data['asset']['params'] + params = data["asset"]["params"] return Asset( id=asset_id, - decimals=params['decimals'], - name=params.get('name', ''), - unit_name=params.get('unit-name', ''), - creator=params.get('creator', ''), - reserve=params.get('reserve', ''), - total_supply=params['total'] / (10 ** params['decimals']) + decimals=params["decimals"], + name=params.get("name", ""), + unit_name=params.get("unit-name", ""), + creator=params.get("creator", ""), + reserve=params.get("reserve", ""), + total_supply=0, + total_supply_micros=params["total"], + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, ) async def get_address_assets(address: str) -> dict: - data = await _run_sync(lambda: indexer_client.lookup_account_assets(address=address)) - return {asset['asset-id']: asset['amount'] for asset in data['assets']} + return (await get_address_asset_snapshot(address, include_algo=False)).balances async def get_address_assets_with_algo(address: str) -> dict: + return (await get_address_asset_snapshot(address, include_algo=True)).balances + + +async def get_address_asset_snapshot( + address: str, + *, + include_algo: bool, +) -> AssetBalanceSnapshot: + """Read balances and their authoritative Indexer round in one response.""" + data = await _run_sync(indexer_client.account_info, address) - asset_balances = {asset['asset-id']: asset['amount'] for asset in data['account']['assets']} - asset_balances[0] = data['account']['amount'] - return asset_balances + account_data = data.get("account") + if not isinstance(account_data, dict): + raise RuntimeError("Indexer account snapshot has no account object") + raw_assets = account_data.get("assets") + if not isinstance(raw_assets, list): + raise RuntimeError("Indexer account snapshot has no asset holdings") + + asset_balances: dict[int, int] = {} + for raw_holding in raw_assets: + if not isinstance(raw_holding, dict): + raise RuntimeError("Indexer account snapshot contains a malformed asset holding") + try: + asset_id = require_algorand_uint64( + raw_holding.get("asset-id"), + "Indexer asset-id", + positive=True, + ) + amount = require_algorand_uint64( + raw_holding.get("amount"), + "Indexer asset amount", + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + if asset_id in asset_balances: + raise RuntimeError(f"Indexer account snapshot repeats asset {asset_id}") + asset_balances[asset_id] = amount + + if include_algo: + try: + asset_balances[0] = require_algorand_uint64( + account_data.get("amount"), + "Indexer ALGO amount", + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + try: + observed_round = require_algorand_uint64( + data.get("current-round"), + "Indexer current-round", + ) + except ValueError as exc: + raise RuntimeError(str(exc)) from exc + return AssetBalanceSnapshot( + balances=asset_balances, + observed_round=observed_round, + ) -@cached(ttl=10, namespace='node', key='current_round') +@cached(ttl=10, namespace="node", key="current_round") async def get_current_round(): """Async wrapper around the sync get_current_round (single implementation, shared cache).""" return await _run_sync(_sync_get_current_round) @@ -63,22 +128,22 @@ async def get_current_round(): async def get_app_address(app_id: int) -> str: data = await _run_sync(lambda: indexer_client.application_logs(application_id=app_id, limit=10)) - log_data = data['log-data'] + log_data = data["log-data"] - txid = log_data[0]['txid'] + txid = log_data[0]["txid"] data = await _run_sync(lambda: indexer_client.transaction(txid=txid)) - return data['transaction']['inner-txns'][0]['sender'] + return data["transaction"]["inner-txns"][0]["sender"] async def get_address_app_ids(address: str) -> list[int]: data = await _run_sync(lambda: indexer_client.account_info(address=address)) - return [app_state['id'] for app_state in data['account']['apps-local-state']] + return [app_state["id"] for app_state in data["account"]["apps-local-state"]] def is_opted_in(address: str, asa_id: int) -> bool: account_info = algod_client.account_info(address) - for account in account_info.get('assets', []): - if account['asset-id'] == asa_id: + for account in account_info.get("assets", []): + if account["asset-id"] == asa_id: return True return False diff --git a/flex/data/asset_prices.py b/flex/data/asset_prices.py index cf30c1ba..497975a0 100644 --- a/flex/data/asset_prices.py +++ b/flex/data/asset_prices.py @@ -17,11 +17,13 @@ from flex.data.assets import get_asset_details from flex.db.model.priced import AssetPrice, AssetPriceInfo from flex.domain.pricing import ( + MAX_OBSERVATION_CLOCK_SKEW, PriceQuote, PriceSource, PriceUnavailableError, PricingError, is_observation_stale, + validate_observation_timestamp, ) from flex.util import build_key_str @@ -70,6 +72,15 @@ def is_asset_price_stale( ) +def is_unverified_legacy_lp_price(asset_price: object) -> bool: + """Return whether a price came from the retired raw-reserve projection.""" + + return ( + getattr(asset_price, "tinyman_algo_pool_id", None) is not None + or getattr(asset_price, "source", None) == PriceSource.DERIVED_LP.value + ) + + def validated_stored_asset_price( asset_price: AssetPrice, *, @@ -78,6 +89,13 @@ def validated_stored_asset_price( ) -> AssetPrice | None: """Return a stored price only when its values and age satisfy policy.""" + if is_unverified_legacy_lp_price(asset_price): + logger.warning( + "Ignoring retired raw-reserve LP projection for asset %s", + getattr(asset_price, "id", "unknown"), + ) + return None + current_time = now or datetime.now(UTC) if is_asset_price_stale( asset_price, @@ -126,22 +144,43 @@ def _skip_asset_price_cache(asset_price: AssetPrice) -> bool: def _upsert_asset_price(asset_price: AssetPrice) -> bool: """Persist a quote only if it is not older than the stored observation.""" + if is_unverified_legacy_lp_price(asset_price): + raise PricingError( + "raw reserve LP projections are retired and cannot be persisted", + ) if not isinstance(asset_price.observed_at, datetime): raise PricingError("persisted asset prices require an observation timestamp") - asset_price.updated = datetime.now(UTC) + current_time = datetime.now(UTC) + observed_at = validate_observation_timestamp( + asset_price.observed_at, + now=current_time, + ) + asset_price.observed_at = observed_at + asset_price.updated = current_time doc = asset_price.to_dict() doc.pop("_id", None) created = doc.pop("created", asset_price.created) collection = db.asset_prices.mongodb_collection - selector = { - "id": asset_price.id, - "$or": [ - {"observed_at": {"$exists": False}}, - {"observed_at": None}, - {"observed_at": {"$lte": asset_price.observed_at}}, - ], - } + selector = AssetPrice.encode_query( + { + "id": asset_price.id, + "$or": [ + {"observed_at": {"$exists": False}}, + {"observed_at": None}, + { + "observed_at": { + "$lte": observed_at, + } + }, + { + "observed_at": { + "$gt": current_time + MAX_OBSERVATION_CLOCK_SKEW, + } + }, + ], + } + ) result = collection.update_one( selector, {"$set": doc, "$setOnInsert": {"created": created}}, @@ -150,7 +189,13 @@ def _upsert_asset_price(asset_price: AssetPrice) -> bool: if result.matched_count: return True - if collection.find_one({"id": asset_price.id}, {"_id": 1}) is not None: + if ( + collection.find_one( + AssetPrice.encode_query({"id": asset_price.id}), + {"_id": 1}, + ) + is not None + ): logger.info( "Discarding older price observation for asset %s at %s", asset_price.id, @@ -161,8 +206,21 @@ def _upsert_asset_price(asset_price: AssetPrice) -> bool: try: collection.insert_one({**doc, "created": created}) except DuplicateKeyError: + # The concurrent insert may be older than this observation. Re-run the + # same conditional write so arrival order cannot defeat timestamp order. + retry = collection.update_one( + selector, + {"$set": doc}, + upsert=False, + ) + if retry.matched_count: + logger.info( + "Replaced a concurrently inserted older price for asset %s", + asset_price.id, + ) + return True logger.info( - "A concurrent price writer won the insert for asset %s", + "A concurrent price writer kept a newer observation for asset %s", asset_price.id, ) return False @@ -171,9 +229,9 @@ def _upsert_asset_price(asset_price: AssetPrice) -> bool: def _current_price_after_rejected_write(asset_id: int) -> AssetPrice: current = db.asset_prices.get_one(id=asset_id) - if current is None: + if current is None or is_unverified_legacy_lp_price(current): raise PyMongoError( - f"price write for asset {asset_id} lost a race but no winner exists", + f"price write for asset {asset_id} lost a race but no safe winner exists", ) return current @@ -185,8 +243,8 @@ async def create_asset_prices_batch( ) -> list[AssetPrice]: """Create prices for multiple assets using Vestige batch API. - LP token prices are handled by the background worker (lp_prices.py), - not by this function. + LP tokens are excluded by the background registry until a verified + economic-reserve adapter exists. """ from flex.providers.vestige import vestige_batch_prices @@ -282,6 +340,7 @@ async def update_asset_price( price_algo=price_algo, price_usd=price_usd, last_update_round=quote.observed_round if quote.observed_round is not None else current_round, + tinyman_algo_pool_id=None, source=quote.source.value, observed_at=quote.observed_at, ) diff --git a/flex/data/assets.py b/flex/data/assets.py index a9d0e4c5..026bc2c5 100644 --- a/flex/data/assets.py +++ b/flex/data/assets.py @@ -1,10 +1,20 @@ +import asyncio import logging +from datetime import UTC, datetime +from typing import Any from aiocache import cached +from pymongo import ReturnDocument from flex import db -from flex.blockchain.info import fetch_asset, ALGO_ASSET -from flex.db.model.blockchain import Asset, AssetInfo, AssetDetails +from flex.blockchain.info import ALGO_ASSET, fetch_asset +from flex.db.bson import decode_bson_uint64 +from flex.db.model.blockchain import ( + TOTAL_SUPPLY_SOURCE_INDEXER, + Asset, + AssetDetails, + AssetInfo, +) from flex.providers.tinyman import get_asset_logo_url from flex.util import build_key_str @@ -15,29 +25,103 @@ async def create_asset(asset_id: int) -> Asset: asset = await fetch_asset(asset_id) asset.logo_url = get_asset_logo_url(asset_id) db.assets.create(asset) - logger.info(f'New Asset: id={asset.id}, name={asset.name}') + logger.info(f"New Asset: id={asset.id}, name={asset.name}") return asset -@cached(ttl=300, namespace='full_asset', key_builder=build_key_str) +@cached(ttl=300, namespace="full_asset", key_builder=build_key_str) async def get_full_asset(asset_id: int) -> Asset: + document = await asyncio.to_thread( + db.assets.mongodb_collection.find_one, + Asset.encode_query({"id": asset_id}), + projection={"total_supply_micros": 1, "total_supply_source": 1}, + ) + if document is not None and _canonical_supply(document) is None: + await _backfill_canonical_supply(asset_id, document) + asset = db.assets.get_by_primary_key(asset_id, throw_ex=False) if asset is None: asset = await create_asset(asset_id) + if not asset.total_supply_is_authoritative: + raise RuntimeError(f"asset {asset_id} canonical supply migration could not be verified") return asset -@cached(namespace='asset_total_supply', key_builder=build_key_str) +def _canonical_supply(document: dict[str, Any] | None) -> int | None: + if document is None or document.get("total_supply_source") != TOTAL_SUPPLY_SOURCE_INDEXER: + return None + value = document.get("total_supply_micros") + return None if value is None else decode_bson_uint64(value) + + +async def _backfill_canonical_supply( + asset_id: int, + document: dict[str, Any], +) -> int: + """Replace a lossy legacy display supply with authoritative Indexer units.""" + + document_id = document.get("_id") + if document_id is None: + raise RuntimeError(f"asset {asset_id} is missing its MongoDB identity") + + authoritative = await fetch_asset(asset_id) + total_supply_micros = authoritative.total_supply_micros + updated = await asyncio.to_thread( + db.assets.mongodb_collection.find_one_and_update, + { + "_id": document_id, + "$or": [ + {"total_supply_micros": {"$exists": False}}, + {"total_supply_micros": None}, + {"total_supply_source": {"$ne": TOTAL_SUPPLY_SOURCE_INDEXER}}, + ], + }, + { + "$set": { + "total_supply_micros": str(total_supply_micros), + "total_supply_source": TOTAL_SUPPLY_SOURCE_INDEXER, + "total_supply": authoritative.total_supply, + "updated": datetime.now(UTC), + } + }, + return_document=ReturnDocument.AFTER, + ) + if updated is None: + updated = await asyncio.to_thread( + db.assets.mongodb_collection.find_one, + {"_id": document_id}, + projection={"total_supply_micros": 1, "total_supply_source": 1}, + ) + persisted_supply = _canonical_supply(updated) + if persisted_supply is None: + raise RuntimeError(f"asset {asset_id} canonical supply migration was not persisted") + if persisted_supply != total_supply_micros: + raise RuntimeError(f"asset {asset_id} canonical supply changed during migration") + return persisted_supply + + +@cached(namespace="asset_total_supply", key_builder=build_key_str) async def get_asset_total_supply(asset_id: int) -> int: + query = Asset.encode_query({"id": asset_id}) + document = await asyncio.to_thread( + db.assets.mongodb_collection.find_one, + query, + projection={"total_supply_micros": 1, "total_supply_source": 1}, + ) + canonical_supply = _canonical_supply(document) + if canonical_supply is not None: + return canonical_supply + if document is not None: + return await _backfill_canonical_supply(asset_id, document) return (await get_full_asset(asset_id)).total_supply_micros -@cached(namespace='asset_info', key_builder=build_key_str) +@cached(namespace="asset_info", key_builder=build_key_str) async def get_asset_info(asset_id: int) -> AssetInfo: return (await get_full_asset(asset_id)).to_info() -@cached(namespace='asset_details', key_builder=build_key_str) +@cached(namespace="asset_details", key_builder=build_key_str) async def get_asset_details(asset_id: int) -> AssetDetails: return (await get_full_asset(asset_id)).to_details() @@ -46,7 +130,7 @@ async def get_asset_details_by_query(query_dict: dict) -> list[AssetDetails]: return [asset.to_details() for asset in db.assets.get_many_by_query(query_dict)] -@cached(ttl=20, namespace='all_asset_details', key='420') +@cached(ttl=20, namespace="all_asset_details", key="420") async def get_all_asset_details() -> list[AssetDetails]: all_assets = db.assets.get_all() return [asset.to_details() for asset in all_assets] @@ -61,7 +145,7 @@ async def amount_to_micros(asset_id: int, amount: float) -> int: async def load_all_assets_data() -> list[Asset]: - logger.info('Loading all assets data.') + logger.info("Loading all assets data.") asset_ids = {ALGO_ASSET.id} @@ -80,5 +164,5 @@ async def load_all_assets_data() -> list[Asset]: asset = await get_full_asset(asset_id) assets.append(asset) - logger.info(f'{len(asset_ids)} assets data loaded.') + logger.info(f"{len(asset_ids)} assets data loaded.") return assets diff --git a/flex/data/lp_prices.py b/flex/data/lp_prices.py deleted file mode 100644 index 86daf88c..00000000 --- a/flex/data/lp_prices.py +++ /dev/null @@ -1,345 +0,0 @@ -import asyncio -import logging -from datetime import timedelta -from decimal import Decimal - -from aiocache import cached -from algosdk.error import AlgodHTTPError - -from blockchain.node import get_current_round -from core.db.contracts import get_contracts_by_type -from core.util import parse_bignum -from env import settings -from flex import db -from flex.blockchain.base import algod_client -from flex.blockchain.info import _run_sync -from flex.data.asset_prices import _upsert_asset_price -from flex.data.assets import get_asset_details -from flex.db.model.priced import AssetPrice -from flex.domain.pricing import ( - PriceQuote, - PriceSource, - PriceUnavailableError, - PricingError, -) -from flex.domain.pricing import ( - calculate_lp_token_price_algo as calculate_lp_token_price_algo_exact, -) -from flex.providers import price_router - -logger = logging.getLogger(__name__) - -LP_CONCURRENCY = 2 -LP_REQUEST_DELAY = 1.0 # seconds between algod calls per slot - - -class LpTokenRegistryError(RuntimeError): - """The LP registry cannot safely distinguish derived from external prices.""" - - -def _extract_stake_token_id(contract) -> int | None: - """Extract stake token ID from contract metadata or cache.""" - meta = contract.metadata or {} - stid = meta.get("stake_token_id") - if stid is not None: - return int(stid) - - cache = meta.get("cache", {}) - initial = cache.get("initial", {}) - raw = initial.get("stakeToken") or initial.get("token") - if raw is None: - return None - if isinstance(raw, dict) and raw.get("type") == "BigNumber" and "hex" in raw: - return int(raw["hex"], 16) - try: - return int(raw) - except (ValueError, TypeError): - return None - - -def _get_active_stake_token_ids() -> set[int]: - """Return stake token IDs for farm contracts where rewards are still active. - - Only includes contracts where endBlock >= current_round (reward period not expired). - Ended farms have static LP reserves — their prices don't need frequent updates. - """ - contracts = get_contracts_by_type("farm") - current_round = get_current_round() - active_ids = set() - total = 0 - - for c in contracts: - stid = _extract_stake_token_id(c) - if not stid: - continue - total += 1 - - meta = c.metadata or {} - cache = meta.get("cache") - if not cache: - continue # no cache = can't determine endBlock, skip - - try: - end_block = parse_bignum(cache.get("initial", {}).get("endBlock")) - if end_block >= current_round: - active_ids.add(stid) - except (ValueError, TypeError): - continue - - logger.info(f"Active LP tokens: {len(active_ids)}/{total} farm contracts") - return active_ids - - -@cached(ttl=300, namespace="lp_token_defs") -async def get_lp_token_definitions() -> list[dict]: - """Build LP token registry from DB sources (no algod calls). - - Sources (in priority order): - 1. lp_tokens collection (has verified asset1_id, asset2_id, dex_provider) - 2. farming_pools collection (has first_token, second_token, dex_name) - 3. Contract metadata (asset1_id, dex fields) - - Returns list of dicts with lp_token_id, asset1_id, asset2_id, dex. - """ - contracts = get_contracts_by_type("farm") - - # Collect all stake token IDs from farm contracts - stake_token_ids = set() - expected_lp_token_ids = set() - for c in contracts: - stid = _extract_stake_token_id(c) - if stid: - stake_token_ids.add(stid) - meta = c.metadata or {} - if "asset1_id" in meta or "asset_1_id" in meta: - expected_lp_token_ids.add(stid) - - if not stake_token_ids: - logger.warning("No stake tokens found in farm contracts") - return [] - - lp_defs_by_id: dict[int, dict] = {} - from_lp_tokens = 0 - from_farming_pools = 0 - from_metadata = 0 - - # Source 1: lp_tokens collection (best quality — has verified pool data) - try: - lp_tokens_list = db.lp_tokens.get_many_by_query({"id": {"$in": list(stake_token_ids)}}) - for lt in lp_tokens_list: - lp_defs_by_id[lt.id] = { - "lp_token_id": lt.id, - "asset1_id": lt.asset1_id, - "asset2_id": lt.asset2_id, - "dex": lt.dex_provider, - } - from_lp_tokens += 1 - except Exception as exc: - raise LpTokenRegistryError("failed to query lp_tokens") from exc - - # Source 2: farming_pools collection - try: - farming_pools_list = db.farming_pools.get_all() - for fp in farming_pools_list: - stid = fp.stake_token.id - if stid not in stake_token_ids or stid in lp_defs_by_id: - continue - asset1_id = fp.first_token.id - asset2_id = fp.second_token.id - # Normalize: non-ALGO token as asset1 (matches lp_tokens convention) - if asset1_id == 0: - asset1_id, asset2_id = asset2_id, asset1_id - lp_defs_by_id[stid] = { - "lp_token_id": stid, - "asset1_id": asset1_id, - "asset2_id": asset2_id, - "dex": fp.dex_name, - } - from_farming_pools += 1 - except Exception as exc: - raise LpTokenRegistryError("failed to query farming_pools") from exc - - # Source 3: contract metadata - for c in contracts: - meta = c.metadata or {} - stid = _extract_stake_token_id(c) - if not stid or stid in lp_defs_by_id: - continue - asset1_id = meta.get("asset1_id", meta.get("asset_1_id")) - asset2_id = meta.get("asset2_id", meta.get("asset_2_id", 0)) - dex = meta.get("dex") or meta.get("dex_provider") - if asset1_id is not None and dex: - lp_defs_by_id[stid] = { - "lp_token_id": stid, - "asset1_id": int(asset1_id), - "asset2_id": int(asset2_id), - "dex": dex, - } - from_metadata += 1 - - unresolved_lp_ids = expected_lp_token_ids - lp_defs_by_id.keys() - if unresolved_lp_ids: - raise LpTokenRegistryError( - f"LP registry is incomplete for token ids {sorted(unresolved_lp_ids)}", - ) - - result = list(lp_defs_by_id.values()) - logger.info( - f"LP token definitions: {len(result)}/{len(stake_token_ids)} " - f"(lp_tokens={from_lp_tokens}, farming_pools={from_farming_pools}, " - f"metadata={from_metadata}, " - f"unresolved={len(stake_token_ids) - len(result)})" - ) - return result - - -def _get_pool_address(asset_info: dict) -> str: - """Get the pool address for an LP token. - - Tinyman V2 stores the pool address in 'reserve' (creator is the factory). - Other DEXes use 'creator' as the pool address directly. - """ - params = asset_info["params"] - reserve = params.get("reserve") - creator = params["creator"] - if reserve and reserve != creator: - return reserve - return creator - - -async def calculate_lp_token_price_algo(lp_def: dict) -> Decimal: - """Calculate LP token price from on-chain reserves and asset1 price. - - Formula: asset1_price_algo * asset1_reserve * 2 / circulating_lp_supply - The entire calculation stays in Decimal until the persistence boundary. - """ - price_algo, _ = await _calculate_lp_token_price_algo_with_quote(lp_def) - return price_algo - - -async def _calculate_lp_token_price_algo_with_quote( - lp_def: dict, -) -> tuple[Decimal, PriceQuote]: - """Calculate an LP price and retain the source asset observation.""" - - lp_token_id = lp_def["lp_token_id"] - asset1_id = lp_def["asset1_id"] - - # 1. Get LP token info (pool address + total supply) - asset_info = await _run_sync(algod_client.asset_info, lp_token_id) - pool_address = _get_pool_address(asset_info) - total_supply_micros = asset_info["params"]["total"] - - # 2. Get pool account balances - account_info = await _run_sync(algod_client.account_info, pool_address) - - asset_balances = {} - for asset in account_info.get("assets", []): - asset_balances[asset["asset-id"]] = asset["amount"] - # ALGO balance (needed when asset1 or asset2 is ALGO) - asset_balances[0] = account_info.get("amount", 0) - - asset1_reserve_micros = asset_balances.get(asset1_id, 0) - pool_lp_balance_micros = asset_balances.get(lp_token_id, 0) - - asset1_details = await get_asset_details(asset1_id) - asset1_quote = await price_router.get_asset_price_quote(asset1_id) - - return ( - calculate_lp_token_price_algo_exact( - asset1_price_algo=asset1_quote.algo, - asset1_reserve_micros=asset1_reserve_micros, - asset1_decimals=asset1_details.decimals, - total_lp_supply_micros=total_supply_micros, - pool_lp_balance_micros=pool_lp_balance_micros, - lp_token_decimals=asset_info["params"]["decimals"], - ), - asset1_quote, - ) - - -async def _update_single_lp( - lp_def: dict, - algo_quote: PriceQuote, - current_round: int, -) -> bool: - """Calculate and persist a single LP token price. Returns True on success.""" - try: - price_algo, asset1_quote = await _calculate_lp_token_price_algo_with_quote(lp_def) - quote = PriceQuote.from_raw( - asset_id=lp_def["lp_token_id"], - algo=price_algo, - usd=price_algo * algo_quote.usd, - source=PriceSource.DERIVED_LP, - stale_after=timedelta(seconds=settings.lp_prices_update_interval), - observed_round=current_round, - observed_at=min(asset1_quote.observed_at, algo_quote.observed_at), - ) - legacy_algo, legacy_usd = quote.to_legacy_floats() - asset_details = await get_asset_details(quote.asset_id) - asset_price = AssetPrice( - id=quote.asset_id, - name=asset_details.name, - price_algo=legacy_algo, - price_usd=legacy_usd, - last_update_round=current_round, - source=quote.source.value, - observed_at=quote.observed_at, - ) - _upsert_asset_price(asset_price) - return True - except (PriceUnavailableError, PricingError) as exc: - logger.warning("Failed LP price for token %s: %s", lp_def["lp_token_id"], exc) - return False - except (AlgodHTTPError, KeyError, TypeError, ValueError) as exc: - logger.warning("Failed LP price for token %s: %s", lp_def["lp_token_id"], exc) - return False - - -_last_lp_update_round: int = 0 - - -async def update_lp_token_prices(current_round: int) -> None: - """Calculate and persist LP token prices for active farm contracts only. - - Skips if fewer than lp_prices_update_interval seconds have elapsed. - Only prices LP tokens from farms that are still active or have stake, - dramatically reducing algod calls (from ~160 to ~10-20 tokens). - """ - global _last_lp_update_round - min_round_gap = int(settings.lp_prices_update_interval / settings.block_time) - if _last_lp_update_round and (current_round - _last_lp_update_round) < min_round_gap: - return - - lp_defs = await get_lp_token_definitions() - if not lp_defs: - return - - # Only price LP tokens from active farms (endBlock not passed or still has stake) - active_ids = _get_active_stake_token_ids() - active_defs = [d for d in lp_defs if d["lp_token_id"] in active_ids] - logger.info(f"LP pricing: {len(active_defs)} active out of {len(lp_defs)} total definitions") - - if not active_defs: - _last_lp_update_round = current_round - return - - try: - algo_quote = await price_router.get_algo_price_quote() - except (PriceUnavailableError, PricingError) as exc: - logger.error("All ALGO price providers failed, skipping LP update: %s", exc) - return - - semaphore = asyncio.Semaphore(LP_CONCURRENCY) - - async def _bounded(lp_def: dict) -> bool: - async with semaphore: - result = await _update_single_lp(lp_def, algo_quote, current_round) - await asyncio.sleep(LP_REQUEST_DELAY) - return result - - results = await asyncio.gather(*[_bounded(d) for d in active_defs]) - lp_updated = sum(1 for r in results if r) - _last_lp_update_round = current_round - - logger.info(f"LP token prices: {lp_updated}/{len(active_defs)} updated") diff --git a/flex/data/lp_registry.py b/flex/data/lp_registry.py new file mode 100644 index 00000000..8097a111 --- /dev/null +++ b/flex/data/lp_registry.py @@ -0,0 +1,131 @@ +"""Read-only LP token classification used to keep price routing fail closed.""" + +import logging +from typing import Any, TypedDict + +from aiocache import cached + +from core.db.contracts import get_contracts_by_type +from flex import db + +logger = logging.getLogger(__name__) + + +class LpTokenRegistryError(RuntimeError): + """The LP registry cannot safely distinguish LP tokens from regular assets.""" + + +class LpTokenDefinition(TypedDict): + lp_token_id: int + asset1_id: int + asset2_id: int + dex: str + + +def _extract_stake_token_id(contract: Any) -> int | None: + """Extract a stake token ID from contract metadata or its cached state.""" + + metadata = contract.metadata or {} + stake_token_id = metadata.get("stake_token_id") + if stake_token_id is not None: + return int(stake_token_id) + + initial = metadata.get("cache", {}).get("initial", {}) + raw = initial.get("stakeToken") or initial.get("token") + if raw is None: + return None + if isinstance(raw, dict) and raw.get("type") == "BigNumber" and "hex" in raw: + return int(raw["hex"], 16) + try: + return int(raw) + except (TypeError, ValueError): + return None + + +@cached(ttl=300, namespace="lp_token_defs") +async def get_lp_token_definitions() -> list[LpTokenDefinition]: + """Build a read-only LP registry without deriving prices from balances.""" + + contracts = get_contracts_by_type("farm") + stake_token_ids: set[int] = set() + for contract in contracts: + stake_token_id = _extract_stake_token_id(contract) + if stake_token_id: + stake_token_ids.add(stake_token_id) + + if not stake_token_ids: + logger.warning("No stake tokens found in farm contracts") + return [] + + definitions: dict[int, LpTokenDefinition] = {} + from_lp_tokens = 0 + from_farming_pools = 0 + from_metadata = 0 + + try: + for token in db.lp_tokens.get_many_by_query( + {"id": {"$in": list(stake_token_ids)}}, + ): + definitions[token.id] = LpTokenDefinition( + lp_token_id=token.id, + asset1_id=token.asset1_id, + asset2_id=token.asset2_id, + dex=token.dex_provider, + ) + from_lp_tokens += 1 + except Exception as exc: + raise LpTokenRegistryError("failed to query lp_tokens") from exc + + try: + for pool in db.farming_pools.get_all(): + stake_token_id = pool.stake_token.id + if stake_token_id not in stake_token_ids or stake_token_id in definitions: + continue + asset1_id = pool.first_token.id + asset2_id = pool.second_token.id + if asset1_id == 0: + asset1_id, asset2_id = asset2_id, asset1_id + definitions[stake_token_id] = LpTokenDefinition( + lp_token_id=stake_token_id, + asset1_id=asset1_id, + asset2_id=asset2_id, + dex=pool.dex_name, + ) + from_farming_pools += 1 + except Exception as exc: + raise LpTokenRegistryError("failed to query farming_pools") from exc + + for contract in contracts: + metadata = contract.metadata or {} + stake_token_id = _extract_stake_token_id(contract) + if not stake_token_id or stake_token_id in definitions: + continue + asset1_id = metadata.get("asset1_id", metadata.get("asset_1_id")) + asset2_id = metadata.get("asset2_id", metadata.get("asset_2_id", 0)) + dex = metadata.get("dex") or metadata.get("dex_provider") + if asset1_id is not None and dex: + definitions[stake_token_id] = LpTokenDefinition( + lp_token_id=stake_token_id, + asset1_id=int(asset1_id), + asset2_id=int(asset2_id), + dex=str(dex), + ) + from_metadata += 1 + + unresolved_stake_token_ids = stake_token_ids - definitions.keys() + if unresolved_stake_token_ids: + raise LpTokenRegistryError( + f"LP classification is incomplete for farm stake token ids {sorted(unresolved_stake_token_ids)}", + ) + + result = list(definitions.values()) + logger.info( + "LP token definitions: %s/%s (lp_tokens=%s, farming_pools=%s, metadata=%s, unresolved=%s)", + len(result), + len(stake_token_ids), + from_lp_tokens, + from_farming_pools, + from_metadata, + len(unresolved_stake_token_ids), + ) + return result diff --git a/flex/data/lp_states.py b/flex/data/lp_states.py index 60fdd731..908825a2 100644 --- a/flex/data/lp_states.py +++ b/flex/data/lp_states.py @@ -1,69 +1,51 @@ +import asyncio import logging +from dataclasses import replace from aiocache import cached from env import settings from flex import db -from flex.blockchain.info import get_address_assets, get_address_assets_with_algo, get_current_round -from flex.data.assets import get_asset_total_supply, micros_to_amount -from flex.data.lp_tokens import get_lp_token_by_id, lp_token_from_tinyman_pool +from flex.blockchain.info import get_address_asset_snapshot +from flex.data.assets import get_asset_total_supply +from flex.data.lp_tokens import ( + get_lp_token_by_id, + lp_token_from_tinyman_pool, + persist_lp_token, +) +from flex.db.lp_projection import ( + LpProjectionPersistenceError, + MongoLpProjectionRepository, +) from flex.db.model.blockchain import LpToken from flex.db.model.liquidity_pools import LpState, LpTransaction -from flex.domain.transactions import event_id_aliases +from flex.domain.algorand import require_algorand_uint64 +from flex.domain.lp_projection import lp_round_end_order from flex.meta_error import MetaError from flex.providers.tinyman import fetch_algo_tinyman_pool_by_asset_id -from flex.providers.vestige import vestige_full_asset_price from flex.util import build_key_str logger = logging.getLogger(__name__) -async def get_price_algo(asset_id) -> float: - # TODO: fix DB caching - # asset_db_price = db.asset_prices.get_one(id=asset_id) - # if asset_db_price is not None: - # price_algo = asset_db_price.price_algo - # else: - # price_algo = (await vestige_full_asset_price(asset_id)).algo - asset_price = await vestige_full_asset_price(asset_id) - return asset_price.algo - - -async def recalculate_lp_state_price_algo_with_micros(lp_state: LpState) -> LpState: - if lp_state.total_tokens_micros == 0: - lp_state.total_tokens = 0 - lp_state.token_price_algo = 0 - return lp_state - - lp_state.asset1_reserve = await micros_to_amount(lp_state.asset1_id, lp_state.asset1_reserve_micros) - lp_state.asset2_reserve = await micros_to_amount(lp_state.asset2_id, lp_state.asset2_reserve_micros) - lp_state.total_tokens = await micros_to_amount(lp_state.token_id, lp_state.total_tokens_micros) - - asset1_price_algo = await get_price_algo(lp_state.asset1_id) - lp_state.token_price_algo = asset1_price_algo * lp_state.asset1_reserve * 2 / lp_state.total_tokens - return lp_state - - -async def create_lp_state_by_lp_token_id(lp_token_id: int, current_round: int | None = None) -> LpState: +async def create_lp_state_by_lp_token_id(lp_token_id: int) -> LpState: lp_token = await get_lp_token_by_id(lp_token_id) if lp_token is None: raise MetaError(f"LP token not found for ID {lp_token_id}") - return await create_lp_state_by_lp_token(lp_token, current_round) - + return await create_lp_state_by_lp_token(lp_token) -async def create_lp_state_by_lp_token(lp_token: LpToken, current_round: int | None = None) -> LpState: - if db.lp_states.exists(token_id=lp_token.id): - raise MetaError(f"LP state already exists for LP token ID {lp_token.id}") +async def create_lp_state_by_lp_token(lp_token: LpToken) -> LpState: # TODO: remove after test if lp_token.asset1_id == 0: raise MetaError(f"ALGO LP token asset1 ID = 0, not id2: {lp_token}") - if lp_token.asset2_id == 0: - balances = await get_address_assets_with_algo(lp_token.address) - else: - balances = await get_address_assets(lp_token.address) + snapshot = await get_address_asset_snapshot( + lp_token.address, + include_algo=True, + ) + balances = snapshot.balances asset1_reserve_micros = balances[lp_token.asset1_id] asset2_reserve_micros = balances[lp_token.asset2_id] @@ -71,15 +53,23 @@ async def create_lp_state_by_lp_token(lp_token: LpToken, current_round: int | No # TODO: add field total_supply_micros to LP token lp_token_total_supply_micros = await get_asset_total_supply(lp_token.id) - issued_lp_tokens_micros = lp_token_total_supply_micros - lp_token_reserve_micros - - asset1_price_full = await vestige_full_asset_price(lp_token.asset1_id) - asset1_reserve = await micros_to_amount(lp_token.asset1_id, asset1_reserve_micros) - asset2_reserve = await micros_to_amount(lp_token.asset2_id, asset2_reserve_micros) - issued_tokens = await micros_to_amount(lp_token.id, issued_lp_tokens_micros) - lp_token_price_algo = asset1_price_full.algo * asset1_reserve * 2 / issued_tokens - - current_round = current_round or (await get_current_round()) + try: + total_supply_micros = require_algorand_uint64( + lp_token_total_supply_micros, + "LP token total supply", + ) + reserve_micros = require_algorand_uint64( + lp_token_reserve_micros, + "pool-owned LP token reserve", + ) + issued_lp_tokens_micros = require_algorand_uint64( + total_supply_micros - reserve_micros, + "issued LP token supply", + ) + except ValueError as exc: + raise LpProjectionPersistenceError(str(exc)) from exc + + current_round = snapshot.observed_round lp_state = LpState( id=lp_token.pool_id, token_id=lp_token.id, @@ -90,44 +80,105 @@ async def create_lp_state_by_lp_token(lp_token: LpToken, current_round: int | No asset1_reserve_micros=asset1_reserve_micros, asset2_reserve_micros=asset2_reserve_micros, total_tokens_micros=issued_lp_tokens_micros, - asset1_reserve=asset1_reserve, - asset2_reserve=asset2_reserve, - total_tokens=issued_tokens, - token_price_algo=lp_token_price_algo, + asset1_reserve=0, + asset2_reserve=0, + total_tokens=0, + token_price_algo=0, + operational_algo_balance_micros=(0 if 0 in {lp_token.asset1_id, lp_token.asset2_id} else balances[0]), last_updated_round=current_round, + last_event_order=lp_round_end_order(current_round), is_algo_pool=lp_token.asset2_id == 0, ) if lp_state.is_algo_pool: logger.info(f"Created new LP state for ALGO pool, asa_id={lp_token.asset1_id}:\n{lp_state.pretty_str()}") - db.lp_states.create(lp_state) - return lp_state + persisted = db.lp_states.get_or_create(lp_state) + immutable_fields = ( + persisted.id, + persisted.token_id, + persisted.asset1_id, + persisted.asset2_id, + persisted.dex_provider, + persisted.address, + ) + requested_fields = ( + lp_state.id, + lp_state.token_id, + lp_state.asset1_id, + lp_state.asset2_id, + lp_state.dex_provider, + lp_state.address, + ) + if immutable_fields != requested_fields: + raise MetaError(f"LP token ID {lp_token.id} is already mapped to different pool metadata") + return persisted async def update_lp_state(lp_state: LpState) -> LpState: - if lp_state.asset1_id == 0 or lp_state.asset2_id == 0: - # TODO: take values from DB sync - balances = await get_address_assets_with_algo(lp_state.address) - else: - balances = await get_address_assets(lp_state.address) - - lp_state.asset1_reserve_micros = balances[lp_state.asset1_id] - lp_state.asset2_reserve_micros = balances[lp_state.asset2_id] + snapshot = await get_address_asset_snapshot( + lp_state.address, + include_algo=True, + ) + balances = snapshot.balances lp_token_reserve_micros = balances[lp_state.token_id] lp_token_total_supply_micros = await get_asset_total_supply(lp_state.token_id) - lp_state.total_tokens_micros = lp_token_total_supply_micros - lp_token_reserve_micros - - lp_state = await recalculate_lp_state_price_algo_with_micros(lp_state) - lp_state.last_updated_round = await get_current_round() - db.lp_states.update(lp_state) - - return lp_state + current_round = snapshot.observed_round + try: + total_supply_micros = require_algorand_uint64( + lp_token_total_supply_micros, + "LP token total supply", + ) + reserve_micros = require_algorand_uint64( + lp_token_reserve_micros, + "pool-owned LP token reserve", + ) + candidate = replace( + lp_state, + asset1_reserve_micros=require_algorand_uint64( + balances[lp_state.asset1_id], + "asset1 reserve", + ), + asset2_reserve_micros=require_algorand_uint64( + balances[lp_state.asset2_id], + "asset2 reserve", + ), + total_tokens_micros=require_algorand_uint64( + total_supply_micros - reserve_micros, + "issued LP token supply", + ), + operational_algo_balance_micros=( + 0 + if 0 in {lp_state.asset1_id, lp_state.asset2_id} + else require_algorand_uint64( + balances[0], + "operational ALGO balance", + ) + ), + last_updated_round=require_algorand_uint64( + current_round, + "LP snapshot round", + ), + ) + except (KeyError, ValueError) as exc: + raise LpProjectionPersistenceError( + f"invalid authoritative LP snapshot for token {lp_state.token_id}: {exc}", + ) from exc + repository = MongoLpProjectionRepository( + states=db.lp_states.mongodb_collection, + events=db.lp_transactions.mongodb_collection, + ) + return await asyncio.to_thread( + repository.replace_snapshot, + candidate, + observed_round=current_round, + ) async def create_lp_states_from_all_pools() -> list[LpState]: farming_pools = db.farming_pools.get_all() new_lp_states = [] + failures: list[Exception] = [] for farming_pool in farming_pools: try: if db.lp_states.exists(token_id=farming_pool.stake_token.id): @@ -136,58 +187,98 @@ async def create_lp_states_from_all_pools() -> list[LpState]: lp_state = await create_lp_state_by_lp_token_id(farming_pool.stake_token.id) new_lp_states.append(lp_state) except Exception as e: + failures.append(e) logger.error(f"Failed to create LP state: {e}\n{farming_pool.pretty_str()}", exc_info=True) + if failures: + raise LpProjectionPersistenceError( + f"failed to create {len(failures)} LP state(s) during cutover" + ) from failures[0] return new_lp_states -async def update_lp_states_with_transactions(transactions: list[LpTransaction]) -> list[LpState]: - if len(transactions) == 0: - return [] - - updated_lp_states = {} - applied_transactions = [] - seen_event_ids: set[str] = set() - for tx in transactions: - if tx.id in seen_event_ids or any(db.lp_transactions.exists(id=alias) for alias in event_id_aliases(tx.id)): - logger.debug(f"Transaction {tx.id} already recorded in DB") - continue - - lp_state = updated_lp_states.get(tx.pool_address) - if lp_state is None: - # TODO: cache - lp_state = db.lp_states.get_one(address=tx.pool_address) - if lp_state is None: - logger.error(f"LP state not found for address {tx.pool_address}") - continue - updated_lp_states[tx.pool_address] = lp_state - - if tx.asa_id == lp_state.token_id: - lp_state.total_tokens_micros += -tx.delta_amount_micros - elif tx.asa_id == lp_state.asset1_id: - lp_state.asset1_reserve_micros += tx.delta_amount_micros - elif tx.asa_id == lp_state.asset2_id: - lp_state.asset2_reserve_micros += tx.delta_amount_micros - else: - logger.error(f"Invalid tx {tx.id} ASA ID {tx.asa_id} for LP state {lp_state.id}") +def _preflight_lp_transactions( + transactions: list[LpTransaction], + *, + expected_round: int, +) -> list[LpTransaction]: + """Validate and canonicalize a complete block batch before any writes.""" + + canonical_by_id: dict[str, LpTransaction] = {} + for transaction in transactions: + if transaction.confirmed_round != expected_round: + raise LpProjectionPersistenceError( + f"LP event {transaction.id!r} belongs to round {transaction.confirmed_round}, expected {expected_round}" + ) + if transaction.event_order is None: + raise LpProjectionPersistenceError(f"LP event {transaction.id!r} has no deterministic order") + + existing = canonical_by_id.get(transaction.id) + if existing is None: + canonical_by_id[transaction.id] = transaction continue + if ( + existing.pool_address, + existing.user_address, + existing.asa_id, + existing.delta_amount_micros, + existing.confirmed_round, + existing.event_position, + existing.event_order, + ) != ( + transaction.pool_address, + transaction.user_address, + transaction.asa_id, + transaction.delta_amount_micros, + transaction.confirmed_round, + transaction.event_position, + transaction.event_order, + ): + raise LpProjectionPersistenceError(f"LP event ID {transaction.id!r} has conflicting data in one block") + + return sorted( + canonical_by_id.values(), + key=lambda item: item.event_order or "", + ) - lp_state.last_updated_round = tx.confirmed_round - applied_transactions.append(tx) - seen_event_ids.add(tx.id) - - lp_states = [] - for lp_state in updated_lp_states.values(): - # TODO: optimize calculate only changed fields - lp_state = await recalculate_lp_state_price_algo_with_micros(lp_state) - db.lp_states.update(lp_state) - lp_states.append(lp_state) - if applied_transactions: - db.lp_transactions.create_many(applied_transactions) +async def update_lp_states_with_transactions( + transactions: list[LpTransaction], + *, + expected_round: int, +) -> list[LpState]: + if not transactions: + return [] - logger.info(f"Updated {len(lp_states)} LP states with {len(applied_transactions)} transactions") - return lp_states + transactions = _preflight_lp_transactions( + transactions, + expected_round=expected_round, + ) + repository = MongoLpProjectionRepository( + states=db.lp_states.mongodb_collection, + events=db.lp_transactions.mongodb_collection, + ) + changed_addresses: set[str] = set() + applied_event_count = 0 + for transaction in transactions: + outcome = await asyncio.to_thread( + repository.project, + transaction, + ) + if outcome.changed_balances: + changed_addresses.add(transaction.pool_address) + applied_event_count += 1 + + updated_states = [ + await asyncio.to_thread(repository.get_state, pool_address) for pool_address in sorted(changed_addresses) + ] + + logger.info( + "Updated %s LP states with %s new transaction(s)", + len(updated_states), + applied_event_count, + ) + return updated_states async def update_all_lp_states_linear() -> list[LpState]: @@ -195,13 +286,19 @@ async def update_all_lp_states_linear() -> list[LpState]: logger.debug(f"Updating {len(lp_states)} LP states...") updated_lp_states = [] + failures: list[Exception] = [] for lp_state in lp_states: try: updated_lp_state = await update_lp_state(lp_state) updated_lp_states.append(updated_lp_state) except Exception as e: + failures.append(e) logger.error(f"Error updating state of LP {lp_state.id}: {e}", exc_info=True) + if failures: + raise LpProjectionPersistenceError( + f"failed to snapshot {len(failures)} LP state(s); sync checkpoint was not advanced" + ) from failures[0] logger.debug(f"Updated {len(updated_lp_states)} LP states") return updated_lp_states @@ -236,6 +333,6 @@ async def get_tinyman_pool_lp_state_by_asset_id(asset_id: int) -> LpState | None lp_token = db.lp_tokens.get_by_primary_key(tinyman_pool.lp_token_id, throw_ex=False) if lp_token is None: lp_token = await lp_token_from_tinyman_pool(tinyman_pool) - db.lp_tokens.create(lp_token) + lp_token = persist_lp_token(lp_token) return await create_lp_state_by_lp_token(lp_token) diff --git a/flex/data/lp_tokens.py b/flex/data/lp_tokens.py index 3953ebe2..a042fa77 100644 --- a/flex/data/lp_tokens.py +++ b/flex/data/lp_tokens.py @@ -4,16 +4,41 @@ from flex import db from flex.blockchain.info import get_address_app_ids +from flex.db.model.blockchain import LpToken from flex.providers import vestige from flex.providers.pact import get_pact_pool_info -from flex.providers.tinyman import get_tinyman_pool_info, TinymanPoolInfo +from flex.providers.tinyman import TinymanPoolInfo, get_tinyman_pool_info from flex.providers.vestige import DexProvider, fetch_lp_token -from flex.db.model.blockchain import LpToken from flex.util import build_key_str logger = logging.getLogger(__name__) +class LpTokenIdentityConflictError(RuntimeError): + """Raised when an ASA ID is already bound to different pool metadata.""" + + +def persist_lp_token(candidate: LpToken) -> LpToken: + """Atomically register an LP token and verify its immutable identity.""" + + persisted = db.lp_tokens.get_or_create(candidate) + identity_fields = ( + "id", + "pool_id", + "asset1_id", + "asset2_id", + "address", + "dex_provider", + ) + if tuple(getattr(persisted, field) for field in identity_fields) != tuple( + getattr(candidate, field) for field in identity_fields + ): + raise LpTokenIdentityConflictError( + f"LP token {candidate.id} is already registered with different pool metadata" + ) + return persisted + + async def lp_token_from_tinyman_pool(tinyman_pool: TinymanPoolInfo) -> LpToken: app_ids = await get_address_app_ids(tinyman_pool.address) return LpToken( @@ -22,7 +47,7 @@ async def lp_token_from_tinyman_pool(tinyman_pool: TinymanPoolInfo) -> LpToken: asset1_id=tinyman_pool.asset1_id, asset2_id=tinyman_pool.asset2_id, address=tinyman_pool.address, - dex_provider=DexProvider.TINYMAN_V2 + dex_provider=DexProvider.TINYMAN_V2, ) @@ -37,7 +62,7 @@ async def fetch_lp_token_strong(lp_token_id: int, asset1_id: int, asset2_id: int asset1_id=asset1_id, asset2_id=asset2_id, address=pact_pool.address, - dex_provider=dex_provider + dex_provider=dex_provider, ) else: return await vestige.fetch_lp_token(lp_token_id, asset1_id, asset2_id, dex_provider) @@ -47,37 +72,40 @@ async def fetch_lp_token_strong(lp_token_id: int, asset1_id: int, asset2_id: int tinyman_pool = await get_tinyman_pool_info(asset1_id, asset2_id) return await lp_token_from_tinyman_pool(tinyman_pool) except Exception as e: - logger.error(f'Tinyman pool for assets {asset1_id} and {asset2_id} not found: {e}') + logger.error(f"Tinyman pool for assets {asset1_id} and {asset2_id} not found: {e}") return await fetch_lp_token(lp_token_id, asset1_id, asset2_id, dex_provider) async def fetch_lp_token_by_id(lp_token_id: int) -> LpToken | None: - farming_pool = db.farming_pools.get_one(**{'stake_token.id': lp_token_id}) + farming_pool = db.farming_pools.get_one(**{"stake_token.id": lp_token_id}) if farming_pool is None: return None if farming_pool.first_token.id < farming_pool.second_token.id: - farming_pool.first_token.id, farming_pool.second_token.id = farming_pool.second_token.id, farming_pool.first_token.id + farming_pool.first_token.id, farming_pool.second_token.id = ( + farming_pool.second_token.id, + farming_pool.first_token.id, + ) return await fetch_lp_token_strong( lp_token_id=lp_token_id, asset1_id=farming_pool.first_token.id, asset2_id=farming_pool.second_token.id, - dex_provider=farming_pool.dex_name + dex_provider=farming_pool.dex_name, ) -@cached(namespace='lp_token_get_all', key_builder=build_key_str) +@cached(namespace="lp_token_get_all", key_builder=build_key_str) async def get_all_lp_tokens() -> list[LpToken]: return db.lp_tokens.get_all() -@cached(namespace='lp_token_by_id', key_builder=build_key_str) +@cached(namespace="lp_token_by_id", key_builder=build_key_str) async def get_lp_token_by_id(lp_token_id: int) -> LpToken | None: lp_token = db.lp_tokens.get_by_primary_key(lp_token_id, throw_ex=False) if lp_token is None: lp_token = await fetch_lp_token_by_id(lp_token_id) if lp_token is not None: - db.lp_tokens.create(lp_token) + lp_token = persist_lp_token(lp_token) return lp_token diff --git a/flex/data/stats.py b/flex/data/stats.py index e3df8524..eca934bc 100644 --- a/flex/data/stats.py +++ b/flex/data/stats.py @@ -1,7 +1,9 @@ import logging +from datetime import timedelta +from env import settings from flex import db -from flex.data.asset_prices import get_asset_price +from flex.data.asset_prices import get_asset_price, validated_stored_asset_price from flex.db.model.pools import PoolType logger = logging.getLogger(__name__) @@ -10,7 +12,12 @@ async def calculate_total_tvl_usd_for_type(type: PoolType) -> float: pools = db.pool_states.get_many(type=type) all_asset_prices = db.asset_prices.get_all() - price_usd_by_id = {price.id: price.price_usd for price in all_asset_prices} + max_age = timedelta(seconds=settings.asset_prices_max_stale) + price_usd_by_id = { + price.id: price.price_usd + for price in all_asset_prices + if validated_stored_asset_price(price, max_age=max_age) is not None + } total_usd = 0 for pool in pools: pool_token_price_usd = price_usd_by_id.get(pool.stake_token.id) diff --git a/flex/data/tinyman_lps.py b/flex/data/tinyman_lps.py deleted file mode 100644 index 5e962c49..00000000 --- a/flex/data/tinyman_lps.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Validated Tinyman ALGO-pool price projection.""" - -import logging -from datetime import UTC, datetime, timedelta -from decimal import Decimal, localcontext - -from env import settings -from flex import db -from flex.blockchain.info import ALGO_ASSET -from flex.data.asset_prices import _upsert_asset_price -from flex.data.assets import get_asset_details -from flex.db.model.liquidity_pools import LpState -from flex.db.model.priced import AssetPrice -from flex.domain.pricing import ( - PERSISTED_PRICE_PRECISION, - InvalidLiquidityPoolError, - InvalidPriceError, - PriceQuote, - PriceSource, -) - -logger = logging.getLogger(__name__) - - -def _pool_observed_at(lp_state: LpState) -> datetime: - observed_at = getattr(lp_state, "updated", None) - if not isinstance(observed_at, datetime): - raise InvalidLiquidityPoolError("Tinyman pool state has no observation timestamp") - if observed_at.tzinfo is None: - return observed_at.replace(tzinfo=UTC) - return observed_at.astimezone(UTC) - - -async def _tinyman_pool_quotes( - lp_state: LpState, - *, - algo_quote: PriceQuote, -) -> tuple[PriceQuote, PriceQuote, str]: - if not lp_state.is_algo_pool or lp_state.asset2_id != ALGO_ASSET.id: - raise InvalidLiquidityPoolError("Tinyman price source must be an asset/ALGO pool") - if lp_state.asset1_reserve_micros <= 0 or lp_state.asset2_reserve_micros <= 0 or lp_state.total_tokens_micros <= 0: - raise InvalidLiquidityPoolError("Tinyman reserves and issued LP supply must be positive") - if algo_quote.asset_id != ALGO_ASSET.id: - raise InvalidPriceError("Tinyman projection requires an ALGO/USD quote") - - asset_details = await get_asset_details(lp_state.asset1_id) - lp_details = await get_asset_details(lp_state.token_id) - with localcontext() as context: - context.prec = PERSISTED_PRICE_PRECISION - asset_reserve = Decimal(lp_state.asset1_reserve_micros) / Decimal( - 10**asset_details.decimals, - ) - algo_reserve = Decimal(lp_state.asset2_reserve_micros) / Decimal( - 10**ALGO_ASSET.decimals, - ) - issued_lp_tokens = Decimal(lp_state.total_tokens_micros) / Decimal( - 10**lp_details.decimals, - ) - asset_price_algo = +(algo_reserve / asset_reserve) - lp_price_algo = +(algo_reserve * Decimal(2) / issued_lp_tokens) - algo_usd = algo_quote.usd - - stale_after = timedelta(seconds=settings.asset_prices_ttl) - observed_at = min(_pool_observed_at(lp_state), algo_quote.observed_at) - asset_quote = PriceQuote.from_raw( - asset_id=lp_state.asset1_id, - algo=asset_price_algo, - usd=asset_price_algo * algo_usd, - source=PriceSource.TINYMAN, - stale_after=stale_after, - observed_round=lp_state.last_updated_round, - observed_at=observed_at, - ) - lp_quote = PriceQuote.from_raw( - asset_id=lp_state.token_id, - algo=lp_price_algo, - usd=lp_price_algo * algo_usd, - source=PriceSource.DERIVED_LP, - stale_after=stale_after, - observed_round=lp_state.last_updated_round, - observed_at=observed_at, - ) - return asset_quote, lp_quote, asset_details.name - - -async def update_tinyman_algo_lp_state_and_prices( - lp_state: LpState, - algo_quote: PriceQuote, -) -> AssetPrice: - """Project one validated Tinyman observation into LP and asset read models.""" - - asset_quote, lp_quote, asset_name = await _tinyman_pool_quotes( - lp_state, - algo_quote=algo_quote, - ) - lp_state.token_price_algo = lp_quote.to_legacy_floats()[0] - db.lp_states.update(lp_state) - - price_algo, price_usd = asset_quote.to_legacy_floats() - asset_price = AssetPrice( - id=asset_quote.asset_id, - price_algo=price_algo, - price_usd=price_usd, - last_update_round=asset_quote.observed_round or 0, - tinyman_algo_pool_id=lp_state.id, - name=asset_name, - source=asset_quote.source.value, - observed_at=asset_quote.observed_at, - ) - _upsert_asset_price(asset_price) - logger.debug( - "Updated asset %s from Tinyman ALGO pool %s: algo=%s, usd=%s", - asset_price.id, - lp_state.id, - asset_price.price_algo, - asset_price.price_usd, - ) - return asset_price diff --git a/flex/db/asset_transfer_intents.py b/flex/db/asset_transfer_intents.py new file mode 100644 index 00000000..dbd3ebc4 --- /dev/null +++ b/flex/db/asset_transfer_intents.py @@ -0,0 +1,180 @@ +"""MongoDB repository for persisted outbound transfer intents.""" + +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any + +from pymongo import ReturnDocument +from pymongo.collection import Collection +from pymongo.errors import DuplicateKeyError + +from flex.db.bson import encode_bson_integer +from flex.db.model.transfers import AssetTransferIntent +from flex.domain.algorand import require_algorand_uint64 + + +class TransferIntentPersistenceError(RuntimeError): + """Raised when an expected transfer intent cannot be persisted.""" + + +@dataclass(slots=True) +class MongoAssetTransferIntentRepository: + collection: Collection[dict[str, Any]] + + def ensure_indexes(self) -> None: + self.collection.create_index("id", unique=True, name="id_unique") + self.collection.create_index( + [("status", 1), ("updated", 1)], + name="status_updated_idx", + ) + + def get(self, operation_id: str) -> AssetTransferIntent | None: + document = self.collection.find_one({"id": operation_id}) + return self._from_document(document) + + def reserve(self, intent: AssetTransferIntent) -> AssetTransferIntent: + try: + document = self.collection.find_one_and_update( + {"id": intent.id}, + {"$setOnInsert": intent.to_dict()}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + except DuplicateKeyError: + document = self.collection.find_one({"id": intent.id}) + reserved = self._from_document(document) + if reserved is None: + raise TransferIntentPersistenceError(f"failed to reserve transfer {intent.id!r}") + return reserved + + def record_attempt(self, operation_id: str, txid: str) -> AssetTransferIntent: + return self._update_non_terminal( + operation_id, + txid, + { + "$inc": {"attempt_count": 1}, + "$set": {"updated": datetime.now(UTC)}, + }, + ) + + def mark_submitted(self, operation_id: str, txid: str) -> AssetTransferIntent: + now = datetime.now(UTC) + return self._update_non_terminal( + operation_id, + txid, + { + "$set": { + "status": "submitted", + "submitted_at": now, + "last_error": None, + "updated": now, + } + }, + ) + + def mark_confirmed( + self, + operation_id: str, + txid: str, + confirmed_round: int, + ) -> AssetTransferIntent: + try: + confirmed_round = require_algorand_uint64( + confirmed_round, + "confirmed_round", + ) + except ValueError as exc: + raise TransferIntentPersistenceError(str(exc)) from exc + now = datetime.now(UTC) + document = self.collection.find_one_and_update( + { + "id": operation_id, + "txid": txid, + "status": {"$ne": "confirmed"}, + }, + { + "$set": { + "status": "confirmed", + "confirmed_round": encode_bson_integer(confirmed_round), + "confirmed_at": now, + "last_error": None, + "updated": now, + } + }, + return_document=ReturnDocument.AFTER, + ) + intent = self._from_document(document) + if intent is not None: + return intent + + current = self.get(operation_id) + if ( + current is not None + and current.txid == txid + and current.status == "confirmed" + and current.confirmed_round == confirmed_round + ): + return current + if current is None: + reason = "no longer exists" + elif current.txid != txid: + reason = f"now belongs to transaction {current.txid!r}" + elif current.status == "confirmed": + reason = f"is already confirmed in round {current.confirmed_round!r}" + else: + reason = f"remains in status {current.status!r}" + raise TransferIntentPersistenceError( + f"transfer {operation_id!r} could not confirm transaction {txid!r} " + f"in round {confirmed_round}: intent {reason}" + ) + + def record_error( + self, + operation_id: str, + txid: str, + error: str, + ) -> AssetTransferIntent: + return self._update_non_terminal( + operation_id, + txid, + { + "$set": { + "last_error": error[:500], + "updated": datetime.now(UTC), + } + }, + ) + + def _update_non_terminal( + self, + operation_id: str, + txid: str, + update: dict[str, Any], + ) -> AssetTransferIntent: + document = self.collection.find_one_and_update( + { + "id": operation_id, + "txid": txid, + "status": {"$ne": "confirmed"}, + }, + update, + return_document=ReturnDocument.AFTER, + ) + intent = self._from_document(document) + if intent is not None: + return intent + + current = self.get(operation_id) + if current is not None and current.txid == txid and current.status == "confirmed": + return current + raise TransferIntentPersistenceError(f"transfer {operation_id!r} changed while updating transaction {txid}") + + @staticmethod + def _from_document( + document: dict[str, Any] | None, + ) -> AssetTransferIntent | None: + if document is None: + return None + payload = dict(document) + payload.pop("_id", None) + return AssetTransferIntent.from_dict(payload) diff --git a/flex/db/bson.py b/flex/db/bson.py new file mode 100644 index 00000000..4e5403ce --- /dev/null +++ b/flex/db/bson.py @@ -0,0 +1,51 @@ +"""Lossless BSON codecs for Algorand integer domains.""" + +from decimal import Decimal +from typing import Any + +from bson import Decimal128 + +ALGORAND_UINT64_MAX = 2**64 - 1 + + +def _decode_integral_decimal(value: object, *, signed: bool) -> int: + decimal_value = value.to_decimal() if isinstance(value, Decimal128) else Decimal(str(value)) + if not decimal_value.is_finite() or decimal_value != decimal_value.to_integral_value(): + raise ValueError("on-chain amount must be a finite integer") + parsed = int(decimal_value) + minimum = -ALGORAND_UINT64_MAX if signed else 0 + if not minimum <= parsed <= ALGORAND_UINT64_MAX: + raise ValueError("on-chain amount is outside the Algorand uint64 range") + return parsed + + +def decode_bson_uint64(value: object) -> int: + return _decode_integral_decimal(value, signed=False) + + +def decode_bson_delta(value: object) -> int: + return _decode_integral_decimal(value, signed=True) + + +def decode_optional_bson_uint64(value: object | None) -> int | None: + return None if value is None else decode_bson_uint64(value) + + +def encode_bson_integer(value: int) -> Decimal128: + return Decimal128(Decimal(value)) + + +def encode_optional_bson_uint64( + value: int | None, +) -> Decimal128 | None: + return None if value is None else encode_bson_integer(value) + + +def encode_uint64_query_value(value: Any) -> Any: + if isinstance(value, int) and not isinstance(value, bool): + return encode_bson_integer(value) + if isinstance(value, list): + return [encode_uint64_query_value(item) for item in value] + if isinstance(value, dict): + return {operator: encode_uint64_query_value(operand) for operator, operand in value.items()} + return value diff --git a/flex/db/classes/base_entity.py b/flex/db/classes/base_entity.py index 8d25fc1e..0f875a90 100644 --- a/flex/db/classes/base_entity.py +++ b/flex/db/classes/base_entity.py @@ -1,11 +1,10 @@ import json from datetime import datetime - -from typing import Any, TypeVar, Generic +from typing import Any, Generic, TypeVar from flex.db.util import string_to_snake_case -EntityT = TypeVar('EntityT') +EntityT = TypeVar("EntityT") # IMPLEMENTATIONS MUST be dataclass and dataclass_json @@ -16,7 +15,20 @@ class BaseEntity(Generic[EntityT]): @classmethod def primary_key_name(cls) -> str: - return 'id' + return "id" + + @classmethod + def encode_query(cls, query: dict[str, Any]) -> dict[str, Any]: + """Encode entity-specific storage types used in Mongo selectors.""" + return dict(query) + + @classmethod + def encode_storage_fields( + cls, + values: dict[str, Any], + ) -> dict[str, Any]: + """Encode a partial Mongo update using the entity storage schema.""" + return dict(values) @property def primary_key(self) -> Any: diff --git a/flex/db/classes/bson_uint64.py b/flex/db/classes/bson_uint64.py new file mode 100644 index 00000000..fea3dd77 --- /dev/null +++ b/flex/db/classes/bson_uint64.py @@ -0,0 +1,33 @@ +"""Entity mixin for lossless uint64 selectors and partial updates.""" + +from typing import Any, ClassVar + +from flex.db.bson import encode_uint64_query_value + + +class BsonUint64StorageMixin: + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset() + + @classmethod + def encode_query( + cls, + query: dict[str, Any], + ) -> dict[str, Any]: + encoded: dict[str, Any] = {} + for field_name, value in query.items(): + if field_name in cls.BSON_UINT64_FIELDS: + encoded[field_name] = encode_uint64_query_value(value) + elif field_name in {"$and", "$nor", "$or"} and isinstance(value, list): + encoded[field_name] = [ + cls.encode_query(branch) if isinstance(branch, dict) else branch for branch in value + ] + else: + encoded[field_name] = value + return encoded + + @classmethod + def encode_storage_fields( + cls, + values: dict[str, Any], + ) -> dict[str, Any]: + return cls.encode_query(values) diff --git a/flex/db/classes/collection_manager.py b/flex/db/classes/collection_manager.py index f6cc7b69..50ff028d 100644 --- a/flex/db/classes/collection_manager.py +++ b/flex/db/classes/collection_manager.py @@ -17,7 +17,7 @@ class DbError(Exception): message: str -EntityT = TypeVar('EntityT', bound='BaseEntity') +EntityT = TypeVar("EntityT", bound="BaseEntity") @dataclass @@ -40,7 +40,7 @@ def create_with(self, **kwargs) -> EntityT: return self.create(item) def get_one(self, **kwargs) -> EntityT | None: - res = self.mongodb_collection.find_one(kwargs) + res = self.mongodb_collection.find_one(self.elem_type.encode_query(kwargs)) if res is None: return None return self.item_from_dict(dict(res)) @@ -48,16 +48,16 @@ def get_one(self, **kwargs) -> EntityT | None: def get_by_primary_key(self, val: Any, throw_ex: bool = True) -> EntityT | None: res = self.get_one(**{self.primary_key_name: val}) if res is None and throw_ex: - raise DbError(code=404, message=f'No {self.name} found with {self.primary_key_name}={val}') + raise DbError(code=404, message=f"No {self.name} found with {self.primary_key_name}={val}") return res def get_or_create(self, item: EntityT) -> EntityT: """Use one upsert operation; concurrent uniqueness requires a primary-key index.""" - query = {self.primary_key_name: item.primary_key} + query = self.elem_type.encode_query({self.primary_key_name: item.primary_key}) try: document = self.mongodb_collection.find_one_and_update( query, - {'$setOnInsert': item.to_dict()}, + {"$setOnInsert": item.to_dict()}, upsert=True, return_document=ReturnDocument.AFTER, ) @@ -66,34 +66,30 @@ def get_or_create(self, item: EntityT) -> EntityT: if document is None: raise if document is None: - raise DbError(code=500, message=f'Failed to read {self.name} after upsert') + raise DbError(code=500, message=f"Failed to read {self.name} after upsert") return self.item_from_dict(document) def get_or_create_with(self, **kwargs) -> EntityT: return self.get_or_create(self.elem_type(**kwargs)) def get_many(self, **kwargs) -> list[EntityT]: - items = self.mongodb_collection.find(kwargs) + items = self.mongodb_collection.find(self.elem_type.encode_query(kwargs)) return [self.item_from_dict(i) for i in items] def get_many_by_query( - self, - query_dict: dict, - sort_by: str | None = None, - reversed: bool = False, - limit: int| None = None + self, query_dict: dict, sort_by: str | None = None, reversed: bool = False, limit: int | None = None ) -> list[EntityT]: - items = self.mongodb_collection.find(query_dict) + items = self.mongodb_collection.find(self.elem_type.encode_query(query_dict)) if sort_by is not None: items = items.sort(sort_by, DESCENDING if reversed else ASCENDING) elif reversed: - items = items.sort('_id', DESCENDING) + items = items.sort("_id", DESCENDING) if limit is not None: items = items.limit(limit) return [self.item_from_dict(i) for i in items] def get_by_array(self, field_name: str, values: list[Any]) -> list[EntityT]: - return self.get_many(**{field_name: {'$in': values}}) + return self.get_many(**{field_name: {"$in": values}}) def get_all(self) -> list[EntityT]: return self.get_many() @@ -102,42 +98,52 @@ def update(self, item: EntityT) -> EntityT: item.updated = datetime.now() item_dict = item.to_dict() self.mongodb_collection.update_one( - {self.primary_key_name: item.primary_key}, {'$set': item_dict} + self.elem_type.encode_query({self.primary_key_name: item.primary_key}), + {"$set": item_dict}, ) return item def update_with(self, item: EntityT, **kwargs) -> EntityT: - kwargs['updated'] = datetime.now() + kwargs["updated"] = datetime.now() + encoded_fields = self.elem_type.encode_storage_fields(kwargs) self.mongodb_collection.update_one( - {self.primary_key_name: item.primary_key}, {'$set': kwargs} + self.elem_type.encode_query({self.primary_key_name: item.primary_key}), + {"$set": encoded_fields}, ) item_dict = item.to_dict() - item_dict.update(kwargs) + item_dict.update(encoded_fields) return self.item_from_dict(item_dict) def update_many_with(self, filter: dict, **kwargs) -> int: - kwargs['updated'] = datetime.now() - res = self.mongodb_collection.update_many(filter, {'$set': kwargs}) + kwargs["updated"] = datetime.now() + res = self.mongodb_collection.update_many( + self.elem_type.encode_query(filter), + {"$set": self.elem_type.encode_storage_fields(kwargs)}, + ) return res.modified_count def remove(self, item: EntityT) -> bool: - res = self.mongodb_collection.delete_one( - {self.primary_key_name: item.primary_key} - ) + res = self.mongodb_collection.delete_one(self.elem_type.encode_query({self.primary_key_name: item.primary_key})) return res.deleted_count > 0 def remove_by(self, **kwargs) -> int: - res = self.mongodb_collection.delete_many(kwargs) + res = self.mongodb_collection.delete_many(self.elem_type.encode_query(kwargs)) return res.deleted_count def remove_all(self) -> int: return self.remove_by() def count(self, **kwargs) -> int: - return self.mongodb_collection.count_documents(kwargs) + return self.mongodb_collection.count_documents(self.elem_type.encode_query(kwargs)) def exists(self, **kwargs) -> bool: - return self.mongodb_collection.find_one(kwargs, projection={'_id': 1}) is not None + return ( + self.mongodb_collection.find_one( + self.elem_type.encode_query(kwargs), + projection={"_id": 1}, + ) + is not None + ) def item_from_dict(self, item_dict: dict) -> EntityT: return self.elem_type.from_dict(item_dict) @@ -147,7 +153,7 @@ def primary_key_name(self) -> str: return self.elem_type.primary_key_name() @classmethod - def create_for_type(cls, elem_type: Type[EntityT], mongodb_database: MongoDatabase) -> 'CollectionManager[EntityT]': - name = f'{elem_type.type_name_snake_case()}s' + def create_for_type(cls, elem_type: Type[EntityT], mongodb_database: MongoDatabase) -> "CollectionManager[EntityT]": + name = f"{elem_type.type_name_snake_case()}s" collection = mongodb_database[name] return cls(name, elem_type, collection) diff --git a/flex/db/cometa_database.py b/flex/db/cometa_database.py index a0b238ea..a8299a81 100644 --- a/flex/db/cometa_database.py +++ b/flex/db/cometa_database.py @@ -1,11 +1,13 @@ from pymongo.database import Database as MongoDatabase from flex.db.classes.database import EntitiesDatabase +from flex.db.model.airdrop import AirdropManifest from flex.db.model.blockchain import LpToken, Asset, PoolTransaction, SyncState, SyncBlock from flex.db.model.liquidity_pools import LpState, LpTransaction from flex.db.model.pool_states import UserState, PoolState from flex.db.model.pools import StakingPool, FarmingPool from flex.db.model.priced import AirdropReward, AssetPrice +from flex.db.model.transfers import AssetTransferIntent class CometaDatabase(EntitiesDatabase): @@ -31,3 +33,5 @@ def __init__(self, mongodb_database: MongoDatabase): self.sync_blocks = self.create_collection_manager_for_type(SyncBlock) self.airdrop_rewards = self.create_collection_manager_for_type(AirdropReward) + self.airdrop_manifests = self.create_collection_manager_for_type(AirdropManifest) + self.asset_transfer_intents = self.create_collection_manager_for_type(AssetTransferIntent) diff --git a/flex/db/indexes.py b/flex/db/indexes.py index 40a5b36b..7286e690 100644 --- a/flex/db/indexes.py +++ b/flex/db/indexes.py @@ -5,29 +5,58 @@ from pymongo.collection import Collection from flex.db.cometa_database import CometaDatabase +from flex.domain.pricing import PriceSource logger = logging.getLogger(__name__) -_UNIQUE_ID_COLLECTIONS = ( - "asset_prices", - "pool_transactions", - "lp_transactions", +_UNIQUE_ID_POLICIES = ( + ("assets", False), + ("asset_prices", True), + ("asset_transfer_intents", False), + ("pool_transactions", False), + ("lp_transactions", False), + ("lp_tokens", False), ) _HOT_INDEXES = ( - ("lp_states", "token_id", "token_id_idx"), + ("lp_states", "token_id", "token_id_unique"), + ("lp_states", "address", "address_unique"), ("pool_states", "pool_id", "pool_id_idx"), ("user_states", "address", "address_idx"), - ("lp_tokens", "id", "lp_token_id_idx"), ) -def _duplicate_id_pipeline() -> list[dict[str, Any]]: - """Group duplicate IDs after sorting the newest record first.""" +def delete_unverified_legacy_lp_prices(database: CometaDatabase) -> int: + """Remove reconstructable prices produced from unauthenticated pool balances.""" + + result = database.asset_prices.mongodb_collection.delete_many( + { + "$or": [ + { + "tinyman_algo_pool_id": { + "$exists": True, + "$ne": None, + } + }, + {"source": PriceSource.DERIVED_LP.value}, + ] + } + ) + if result.deleted_count: + logger.warning( + "Removed %d retired raw-reserve LP price projection(s)", + result.deleted_count, + ) + return result.deleted_count + + +def _duplicate_field_pipeline(field_name: str) -> list[dict[str, Any]]: + """Group duplicate business keys after sorting the newest record first.""" + return [ { "$sort": { - "id": 1, + field_name: 1, "observed_at": -1, "updated": -1, "_id": -1, @@ -35,7 +64,7 @@ def _duplicate_id_pipeline() -> list[dict[str, Any]]: }, { "$group": { - "_id": "$id", + "_id": f"${field_name}", "count": {"$sum": 1}, "keep_id": {"$first": "$_id"}, "all_ids": {"$push": "$_id"}, @@ -52,7 +81,12 @@ def deduplicate_and_create_unique_id_index( ) -> int: """Keep the newest document per ID, then enforce uniqueness.""" removed = 0 - duplicate_groups: Sequence[dict[str, Any]] = list(collection.aggregate(_duplicate_id_pipeline(), allowDiskUse=True)) + duplicate_groups: Sequence[dict[str, Any]] = list( + collection.aggregate( + _duplicate_field_pipeline("id"), + allowDiskUse=True, + ) + ) for group in duplicate_groups: duplicate_ids = [document_id for document_id in group["all_ids"] if document_id != group["keep_id"]] @@ -69,20 +103,142 @@ def deduplicate_and_create_unique_id_index( return removed +def create_unique_id_index_fail_closed( + collection: Collection[dict[str, Any]], + *, + collection_name: str, +) -> None: + """Preserve conflicting immutable records for explicit reconciliation.""" + + duplicate_groups: Sequence[dict[str, Any]] = list( + collection.aggregate( + _duplicate_field_pipeline("id"), + allowDiskUse=True, + ) + ) + if duplicate_groups: + raise RuntimeError( + f"{collection_name} contains {len(duplicate_groups)} duplicate immutable ID group(s); " + "reconcile or quarantine them before startup" + ) + collection.create_index("id", unique=True, name="id_unique") + + +def create_unique_field_index_fail_closed( + collection: Collection[dict[str, Any]], + *, + collection_name: str, + field_name: str, + index_name: str, +) -> None: + """Enforce aggregate identity without destroying conflicting state.""" + + duplicate_groups: Sequence[dict[str, Any]] = list( + collection.aggregate( + _duplicate_field_pipeline(field_name), + allowDiskUse=True, + ) + ) + if duplicate_groups: + raise RuntimeError( + f"{collection_name} contains {len(duplicate_groups)} duplicate {field_name!r} group(s); " + "reconcile them before startup" + ) + collection.create_index( + field_name, + unique=True, + name=index_name, + ) + + +def ensure_airdrop_indexes(database: CometaDatabase) -> None: + """Install the standalone airdrop invariants before any payout work.""" + + create_unique_id_index_fail_closed( + database.airdrop_manifests.mongodb_collection, + collection_name="airdrop_manifests", + ) + reward_collection = database.airdrop_rewards.mongodb_collection + duplicate_operation_ids: Sequence[dict[str, Any]] = list( + reward_collection.aggregate( + [ + {"$match": {"operation_id": {"$type": "string"}}}, + *_duplicate_field_pipeline("operation_id"), + ], + allowDiskUse=True, + ) + ) + if duplicate_operation_ids: + raise RuntimeError( + "airdrop_rewards contains " + f"{len(duplicate_operation_ids)} duplicate 'operation_id' group(s); " + "reconcile them before running an airdrop" + ) + reward_collection.create_index( + "operation_id", + unique=True, + name="operation_id_unique", + partialFilterExpression={"operation_id": {"$type": "string"}}, + ) + + +def ensure_sync_state_singleton(database: CometaDatabase) -> None: + """Migrate one legacy random-ID cursor and reject competing checkpoints.""" + + collection = database.sync_states.mongodb_collection + documents = list( + collection.find( + {}, + projection={"_id": 1, "id": 1, "last_round": 1}, + ) + ) + if len(documents) > 1: + raise RuntimeError( + f"sync_states contains {len(documents)} competing checkpoints; reconcile them before startup" + ) + if documents and documents[0].get("id") != "main": + collection.update_one( + {"_id": documents[0]["_id"]}, + {"$set": {"id": "main"}}, + ) + collection.create_index("id", unique=True, name="id_unique") + + def ensure_database_indexes(database: CometaDatabase) -> dict[str, int]: """Install correctness-critical unique indexes and hot query indexes.""" - removed_by_collection: dict[str, int] = {} + # Delete the retired derived cache before deduplication so a newer unsafe + # row cannot displace a safe provider-backed observation with the same ID. + delete_unverified_legacy_lp_prices(database) + ensure_airdrop_indexes(database) + removed_by_collection: dict[str, int] = {"airdrop_manifests": 0} - for collection_name in _UNIQUE_ID_COLLECTIONS: + for collection_name, can_deduplicate in _UNIQUE_ID_POLICIES: manager = getattr(database, collection_name) - removed_by_collection[collection_name] = deduplicate_and_create_unique_id_index( - manager.mongodb_collection, - collection_name=collection_name, - ) + if can_deduplicate: + removed_by_collection[collection_name] = deduplicate_and_create_unique_id_index( + manager.mongodb_collection, + collection_name=collection_name, + ) + else: + create_unique_id_index_fail_closed( + manager.mongodb_collection, + collection_name=collection_name, + ) + removed_by_collection[collection_name] = 0 + + ensure_sync_state_singleton(database) for manager_name, field_name, index_name in _HOT_INDEXES: manager = getattr(database, manager_name) - manager.mongodb_collection.create_index(field_name, name=index_name) + if manager_name == "lp_states": + create_unique_field_index_fail_closed( + manager.mongodb_collection, + collection_name=manager_name, + field_name=field_name, + index_name=index_name, + ) + else: + manager.mongodb_collection.create_index(field_name, name=index_name) logger.info("Ensured hot query indexes for Flex collections") return removed_by_collection diff --git a/flex/db/lp_projection.py b/flex/db/lp_projection.py new file mode 100644 index 00000000..dca47378 --- /dev/null +++ b/flex/db/lp_projection.py @@ -0,0 +1,343 @@ +"""Atomic MongoDB adapter for liquidity-pool event projection.""" + +from dataclasses import dataclass +from datetime import UTC, datetime +from enum import StrEnum +from typing import Any, cast + +from pymongo import ReturnDocument +from pymongo.collection import Collection +from pymongo.errors import DuplicateKeyError + +from flex.db.bson import encode_bson_integer +from flex.db.model.liquidity_pools import LpState, LpTransaction +from flex.domain.algorand import ( + MAX_ALGORAND_UINT, + require_algorand_uint64, +) +from flex.domain.lp_projection import ( + LpBalanceDelta, + lp_balance_delta, + lp_round_end_order, + snapshot_covers_event, +) +from flex.domain.transactions import event_id_aliases + + +class LpProjectionPersistenceError(RuntimeError): + """Raised when replay cannot prove a safe LP projection outcome.""" + + +class LpProjectionResult(StrEnum): + APPLIED = "applied" + ALREADY_APPLIED = "already_applied" + SNAPSHOT_COVERED = "snapshot_covered" + + +@dataclass(frozen=True, slots=True) +class LpProjectionOutcome: + state: LpState + result: LpProjectionResult + + @property + def changed_balances(self) -> bool: + return self.result is LpProjectionResult.APPLIED + + +@dataclass(slots=True) +class MongoLpProjectionRepository: + states: Collection[dict[str, Any]] + events: Collection[dict[str, Any]] + + def project(self, transaction: LpTransaction) -> LpProjectionOutcome: + """Apply one event atomically or prove that replay is already covered.""" + + while True: + state = self._get_state(transaction.pool_address) + order = transaction.event_order + if order is None: + raise LpProjectionPersistenceError("LP transaction has no deterministic event order") + + canonical_event = self.events.find_one( + {"id": transaction.id}, + ) + if canonical_event is not None: + self._assert_event_document( + canonical_event, + expected=transaction, + ) + legacy_alias = self._legacy_alias(transaction) + if legacy_alias is not None: + raise LpProjectionPersistenceError( + f"legacy LP marker {legacy_alias!r} is ambiguous for {transaction.id!r}; " + "reconcile from an authoritative pool snapshot" + ) + + cursor = state.last_event_order + if cursor is None: + raise LpProjectionPersistenceError( + f"LP state {state.token_id} has no cutover cursor; take an authoritative snapshot before replay" + ) + + if canonical_event is not None: + if cursor < order: + # Another worker may have advanced the state and recorded + # the marker after this worker read its initial snapshot. + # Re-read before classifying marker/state divergence as + # persistent corruption. + state = self._get_state(transaction.pool_address) + cursor = state.last_event_order + if cursor is None or cursor < order: + raise LpProjectionPersistenceError( + f"LP event {transaction.id!r} is recorded but state " + f"{state.token_id} is behind it; reconcile from an " + "authoritative pool snapshot" + ) + return LpProjectionOutcome( + state=state, + result=LpProjectionResult.ALREADY_APPLIED, + ) + + if cursor == order: + self._record_event(transaction) + return LpProjectionOutcome( + state=state, + result=LpProjectionResult.ALREADY_APPLIED, + ) + + if cursor > order: + if snapshot_covers_event( + cursor, + confirmed_round=transaction.confirmed_round, + ): + return LpProjectionOutcome( + state=state, + result=LpProjectionResult.SNAPSHOT_COVERED, + ) + raise LpProjectionPersistenceError( + f"LP state {state.token_id} advanced past unrecorded event {transaction.id!r}" + ) + + delta = lp_balance_delta( + token_id=state.token_id, + asset1_id=state.asset1_id, + asset2_id=state.asset2_id, + event_asset_id=transaction.asa_id, + event_pool_delta_micros=transaction.delta_amount_micros, + ) + updated = self._apply_delta( + state=state, + order=order, + confirmed_round=transaction.confirmed_round, + delta=delta, + ) + if updated is None: + latest = self._get_state(transaction.pool_address) + if latest.last_event_order != cursor: + continue + raise LpProjectionPersistenceError( + f"LP event {transaction.id!r} would underflow or overflow {delta.field}" + ) + + # Marker-last is intentional: if this write fails, replay observes + # cursor == order and heals the marker without repeating the delta. + self._record_event(transaction) + return LpProjectionOutcome( + state=updated, + result=LpProjectionResult.APPLIED, + ) + + def replace_snapshot( + self, + state: LpState, + *, + observed_round: int, + ) -> LpState: + """Replace balances only when the authoritative snapshot is newer.""" + + try: + for field_name in ( + "id", + "token_id", + "asset1_id", + "asset2_id", + "asset1_reserve_micros", + "asset2_reserve_micros", + "total_tokens_micros", + "operational_algo_balance_micros", + ): + require_algorand_uint64( + getattr(state, field_name), + f"LP snapshot {field_name}", + ) + require_algorand_uint64( + observed_round, + "LP snapshot observed_round", + ) + except ValueError as exc: + raise LpProjectionPersistenceError(str(exc)) from exc + snapshot_order = lp_round_end_order(observed_round) + document = self.states.find_one_and_update( + { + "token_id": encode_bson_integer(state.token_id), + "$or": [ + {"last_event_order": {"$exists": False}}, + {"last_event_order": None}, + {"last_event_order": {"$lt": snapshot_order}}, + ], + }, + { + "$set": { + "asset1_reserve_micros": encode_bson_integer( + state.asset1_reserve_micros, + ), + "asset2_reserve_micros": encode_bson_integer( + state.asset2_reserve_micros, + ), + "total_tokens_micros": encode_bson_integer( + state.total_tokens_micros, + ), + "operational_algo_balance_micros": encode_bson_integer( + state.operational_algo_balance_micros, + ), + "last_updated_round": encode_bson_integer( + observed_round, + ), + "last_event_order": snapshot_order, + "updated": datetime.now(UTC), + } + }, + return_document=ReturnDocument.AFTER, + ) + replaced = self._from_state_document(document) + if replaced is not None: + return replaced + + current = self._get_state(state.address) + if current.last_event_order == snapshot_order and self._balances(current) != self._balances(state): + raise LpProjectionPersistenceError( + f"LP snapshot {snapshot_order} conflicts with persisted balances for token {state.token_id}" + ) + return current + + def get_state(self, pool_address: str) -> LpState: + return self._get_state(pool_address) + + def _get_state(self, pool_address: str) -> LpState: + document = self.states.find_one({"address": pool_address}) + state = self._from_state_document(document) + if state is None: + raise LpProjectionPersistenceError(f"LP state not found for address {pool_address}") + return state + + def _apply_delta( + self, + *, + state: LpState, + order: str, + confirmed_round: int, + delta: LpBalanceDelta, + ) -> LpState | None: + bounds: dict[str, Any] = { + "$gte": encode_bson_integer(max(0, -delta.amount)), + } + if delta.amount > 0: + bounds["$lte"] = encode_bson_integer( + MAX_ALGORAND_UINT - delta.amount, + ) + document = self.states.find_one_and_update( + { + "token_id": encode_bson_integer(state.token_id), + "last_event_order": state.last_event_order, + delta.field: bounds, + }, + { + "$inc": { + delta.field: encode_bson_integer(delta.amount), + }, + "$set": { + "last_event_order": order, + "last_updated_round": encode_bson_integer( + confirmed_round, + ), + "updated": datetime.now(UTC), + }, + }, + return_document=ReturnDocument.AFTER, + ) + return self._from_state_document(document) + + def _record_event(self, transaction: LpTransaction) -> None: + payload = transaction.to_dict() + try: + document = self.events.find_one_and_update( + {"id": transaction.id}, + {"$setOnInsert": payload}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + except DuplicateKeyError: + document = self.events.find_one({"id": transaction.id}) + if document is None: + raise LpProjectionPersistenceError(f"failed to record LP event {transaction.id!r}") + self._assert_event_document( + document, + expected=transaction, + ) + + @staticmethod + def _assert_event_document( + document: dict[str, Any], + *, + expected: LpTransaction, + ) -> None: + persisted = dict(document) + persisted.pop("_id", None) + persisted_event = LpTransaction.from_dict(persisted) + if ( + persisted_event.pool_address, + persisted_event.user_address, + persisted_event.asa_id, + persisted_event.delta_amount_micros, + persisted_event.confirmed_round, + persisted_event.event_position, + persisted_event.event_order, + ) != ( + expected.pool_address, + expected.user_address, + expected.asa_id, + expected.delta_amount_micros, + expected.confirmed_round, + expected.event_position, + expected.event_order, + ): + raise LpProjectionPersistenceError(f"LP event ID {expected.id!r} belongs to different immutable data") + + def _legacy_alias(self, transaction: LpTransaction) -> str | None: + aliases = event_id_aliases(transaction.id) + for alias in aliases: + if alias == transaction.id: + continue + if self.events.find_one({"id": alias}, projection={"_id": 1}) is not None: + return alias + return None + + @staticmethod + def _balances(state: LpState) -> tuple[int, int, int, int]: + return ( + state.asset1_reserve_micros, + state.asset2_reserve_micros, + state.total_tokens_micros, + state.operational_algo_balance_micros, + ) + + @staticmethod + def _from_state_document(document: dict[str, Any] | None) -> LpState | None: + if document is None: + return None + payload = dict(document) + payload.pop("_id", None) + return cast( + LpState, + LpState.from_dict(payload), + ) diff --git a/flex/db/model/airdrop.py b/flex/db/model/airdrop.py new file mode 100644 index 00000000..332025ad --- /dev/null +++ b/flex/db/model/airdrop.py @@ -0,0 +1,34 @@ +"""Persistence model for immutable airdrop batch manifests.""" + +from dataclasses import dataclass, field +from datetime import UTC, datetime + +from dataclasses_json import dataclass_json + +from flex.db.classes.base_entity import BaseEntity + + +def _utc_now() -> datetime: + return datetime.now(UTC) + + +@dataclass_json +@dataclass +class AirdropManifest(BaseEntity["AirdropManifest"]): + id: str + asset_id: str + total_amount_micros: str + recipient_count: int + manifest_hash: str + status: str = "prepared" + + created: datetime = field(default_factory=_utc_now) + updated: datetime = field(default_factory=_utc_now) + + def __post_init__(self) -> None: + self.asset_id = str(self.asset_id) + self.total_amount_micros = str(self.total_amount_micros) + + @property + def asset_id_int(self) -> int: + return int(self.asset_id) diff --git a/flex/db/model/blockchain.py b/flex/db/model/blockchain.py index 640ed984..671309d6 100644 --- a/flex/db/model/blockchain.py +++ b/flex/db/model/blockchain.py @@ -1,12 +1,48 @@ from abc import ABC from dataclasses import dataclass, field from datetime import datetime +from decimal import Decimal from functools import cached_property +from typing import ClassVar -from dataclasses_json import dataclass_json +from dataclasses_json import config, dataclass_json +from flex.db.bson import ( + decode_bson_uint64, + decode_optional_bson_uint64, + encode_bson_integer, + encode_optional_bson_uint64, +) from flex.db.classes.base_entity import BaseEntity -from flex.db.util import get_uuid +from flex.db.classes.bson_uint64 import BsonUint64StorageMixin + +UINT64_MAX = (1 << 64) - 1 +TOTAL_SUPPLY_SOURCE_INDEXER = "indexer" + + +class _MissingTotalSupply(int): + """Typed sentinel that cannot be confused with an explicit integer.""" + + +_MISSING_TOTAL_SUPPLY = _MissingTotalSupply(-1) + + +class _UnverifiedTotalSupply(int): + """Compatibility value that must never be serialized as canonical.""" + + +def _decode_total_supply(value: object) -> int: + if isinstance(value, _MissingTotalSupply): + return value + return int(value) + + +def _validate_uint64(value: int, *, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{field_name} must be an integer") + if not 0 <= value <= UINT64_MAX: + raise ValueError(f"{field_name} must be between 0 and {UINT64_MAX}") + return value @dataclass_json @@ -18,7 +54,7 @@ class TxInfo: @dataclass_json @dataclass -class PoolTransaction(BaseEntity['PoolTransaction']): +class PoolTransaction(BaseEntity["PoolTransaction"]): id: str pool_id: int pool_address: str @@ -47,13 +83,45 @@ class LpTokenInfo: @dataclass_json @dataclass -class LpToken(BaseEntity['LpToken']): - id: int - asset1_id: int - asset2_id: int # asset1_id > asset2_id +class LpToken( + BsonUint64StorageMixin, + BaseEntity["LpToken"], +): + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "id", + "asset1_id", + "asset2_id", + "pool_id", + } + ) + + id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + asset1_id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + asset2_id: int = field( # asset1_id > asset2_id + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) dex_provider: str address: str - pool_id: int + pool_id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) @@ -65,7 +133,7 @@ def to_info(self) -> LpTokenInfo: asset2_id=self.asset2_id, dex_provider=self.dex_provider, address=self.address, - pool_id=self.pool_id + pool_id=self.pool_id, ) @@ -77,13 +145,19 @@ class AssetBase(ABC): @cached_property def amount_multiplier(self) -> int: - return 10 ** self.decimals + return 10**self.decimals + + def amount_to_micros(self, amount: Decimal | float | int | str) -> int: + """Convert a display amount without introducing binary-float error.""" + return int(Decimal(str(amount)) * self.amount_multiplier) - def amount_to_micros(self, amount: float) -> int: - return int(amount * self.amount_multiplier) + def micros_to_decimal(self, micros: int) -> Decimal: + """Return an exact display-unit value for an on-chain base-unit amount.""" + return Decimal(micros) / self.amount_multiplier def micros_to_amount(self, micros: int) -> float: - return micros / self.amount_multiplier + """Compatibility conversion for API models that still expose floats.""" + return float(self.micros_to_decimal(micros)) @dataclass_json @@ -110,8 +184,19 @@ class AssetDetails(AssetBase): @dataclass_json @dataclass -class Asset(BaseEntity['Asset'], AssetBase): - id: int +class Asset( + BsonUint64StorageMixin, + BaseEntity["Asset"], + AssetBase, +): + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset({"id"}) + + id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) name: str decimals: int unit_name: str @@ -120,21 +205,54 @@ class Asset(BaseEntity['Asset'], AssetBase): total_supply: float logo_url: str | None = None + # ASA amounts occupy the full uint64 range, which does not fit BSON int64. + # Keep an int in memory and serialize it as a decimal string in MongoDB. + total_supply_micros: int = field( + default=_MISSING_TOTAL_SUPPLY, + metadata=config( + encoder=str, + decoder=_decode_total_supply, + exclude=lambda value: isinstance(value, _UnverifiedTotalSupply), + ), + ) + total_supply_source: str | None = None + # Kept out of storage: old documents may be decoded for display, but code + # that makes financial decisions must first migrate from the Indexer. + total_supply_is_authoritative: bool = field( + init=False, + default=False, + repr=False, + metadata=config(exclude=lambda _: True), + ) created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) - @cached_property - def total_supply_micros(self) -> int: - return self.amount_to_micros(self.total_supply) + def __post_init__(self) -> None: + if isinstance(self.total_supply_micros, _MissingTotalSupply): + # Retain a display-compatible value for old read models, but mark + # it as non-authoritative so financial paths cannot trust it. + self.total_supply_source = None + self.total_supply_micros = self.amount_to_micros(self.total_supply) - def to_info(self) -> AssetInfo: - return AssetInfo( - name=self.name, - decimals=self.decimals, - unit_name=self.unit_name, - id=self.id + self.total_supply_micros = _validate_uint64( + self.total_supply_micros, + field_name="total_supply_micros", ) + self.total_supply_is_authoritative = self.total_supply_source == TOTAL_SUPPLY_SOURCE_INDEXER + if not self.total_supply_is_authoritative: + self.total_supply_micros = _UnverifiedTotalSupply(self.total_supply_micros) + # The persisted base-unit amount is authoritative. Keep the old float + # field as a presentation-only compatibility value. + self.total_supply = self.micros_to_amount(self.total_supply_micros) + + @property + def total_supply_base_units(self) -> int: + """Canonical on-chain supply; alias clarifies the historical name.""" + return self.total_supply_micros + + def to_info(self) -> AssetInfo: + return AssetInfo(name=self.name, decimals=self.decimals, unit_name=self.unit_name, id=self.id) def to_details(self) -> AssetDetails: return AssetDetails( @@ -145,29 +263,69 @@ def to_details(self) -> AssetDetails: creator=self.creator, reserve=self.reserve, logo_url=self.logo_url, - total_supply=self.total_supply + total_supply=self.total_supply, ) @dataclass_json @dataclass -class SyncState(BaseEntity['SyncState']): - id: str = field(default_factory=get_uuid) - last_round: int | None = None +class SyncState( + BsonUint64StorageMixin, + BaseEntity["SyncState"], +): + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "last_round", + "claimed_round", + } + ) + + id: str = "main" + last_round: int | None = field( + default=None, + metadata=config( + encoder=encode_optional_bson_uint64, + decoder=decode_optional_bson_uint64, + ), + ) + claimed_round: int | None = field( + default=None, + metadata=config( + encoder=encode_optional_bson_uint64, + decoder=decode_optional_bson_uint64, + ), + ) + lease_owner: str | None = None + lease_until: datetime | None = None + last_error: str | None = None created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) def rounds_since_updated(self, current_round: int) -> int | None: - return current_round - self.last_round if self.last_round else None + return current_round - self.last_round if self.last_round is not None else None @dataclass_json @dataclass -class SyncBlock(BaseEntity['SyncBlock']): - round: int +class SyncBlock( + BsonUint64StorageMixin, + BaseEntity["SyncBlock"], +): + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset({"round"}) + + round: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) timestamp: int - id: str = field(default_factory=get_uuid) + id: str | None = None created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) + + def __post_init__(self) -> None: + if self.id is None: + self.id = f"round:{self.round}" diff --git a/flex/db/model/liquidity_pools.py b/flex/db/model/liquidity_pools.py index 49cf81a0..29ebde95 100644 --- a/flex/db/model/liquidity_pools.py +++ b/flex/db/model/liquidity_pools.py @@ -1,9 +1,18 @@ from dataclasses import dataclass, field from datetime import datetime +from typing import ClassVar -from dataclasses_json import dataclass_json +from dataclasses_json import config, dataclass_json +from flex.db.bson import ( + decode_bson_delta, + decode_bson_uint64, + encode_bson_integer, +) from flex.db.classes.base_entity import BaseEntity +from flex.db.classes.bson_uint64 import BsonUint64StorageMixin +from flex.domain.algorand import MAX_ALGORAND_UINT, require_algorand_uint64 +from flex.domain.lp_projection import InvalidLpProjectionError, lp_event_order @dataclass_json @@ -34,31 +43,109 @@ class LpStateInfo: @dataclass_json @dataclass -class LpState(BaseEntity["LpState"]): - id: int - token_id: int - asset1_id: int # asset1_id > asset2_id - asset2_id: int +class LpState( + BsonUint64StorageMixin, + BaseEntity["LpState"], +): + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "id", + "token_id", + "asset1_id", + "asset2_id", + "last_updated_round", + "asset1_reserve_micros", + "asset2_reserve_micros", + "total_tokens_micros", + "operational_algo_balance_micros", + } + ) + + id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + token_id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + asset1_id: int = field( # asset1_id > asset2_id + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + asset2_id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) dex_provider: str address: str - last_updated_round: int + last_updated_round: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) # TODO: could not fit into MongoDB ??? (64 bits) - asset1_reserve_micros: int - asset2_reserve_micros: int - total_tokens_micros: int # TODO: change name to issued_tokens_micros + asset1_reserve_micros: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + asset2_reserve_micros: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + total_tokens_micros: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) asset1_reserve: float asset2_reserve: float total_tokens: float token_price_algo: float + # Token-token pools still hold ALGO to fund network fees. Keep this + # operational balance separate from economic reserves and public pricing. + operational_algo_balance_micros: int = field( + default=0, + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ), + ) is_algo_pool: bool = False + last_event_order: str | None = None + derived_observed_at: datetime | None = None swap_fee_apr: float | None = None updated: datetime = field(default_factory=datetime.now) created: datetime = field(default_factory=datetime.now) + def __post_init__(self) -> None: + try: + for field_name in self.BSON_UINT64_FIELDS: + require_algorand_uint64( + getattr(self, field_name), + field_name, + ) + except ValueError as exc: + raise InvalidLpProjectionError(str(exc)) from exc + @classmethod def primary_key_name(cls) -> str: return "token_id" @@ -92,9 +179,57 @@ class LpTransaction(BaseEntity["LpTransaction"]): id: str pool_address: str user_address: str - asa_id: int - delta_amount_micros: int - confirmed_round: int - + asa_id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + delta_amount_micros: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_delta, + ) + ) + confirmed_round: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + + event_position: int = field( + default=0, + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ), + ) + event_order: str | None = None created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) + + def __post_init__(self) -> None: + try: + require_algorand_uint64(self.asa_id, "asa_id") + require_algorand_uint64(self.confirmed_round, "confirmed_round") + require_algorand_uint64(self.event_position, "event_position") + except ValueError as exc: + raise InvalidLpProjectionError(str(exc)) from exc + if ( + isinstance(self.delta_amount_micros, bool) + or not isinstance(self.delta_amount_micros, int) + or not -MAX_ALGORAND_UINT <= self.delta_amount_micros <= MAX_ALGORAND_UINT + ): + raise InvalidLpProjectionError( + "delta_amount_micros must fit the signed Algorand uint64 domain", + ) + expected_order = lp_event_order( + self.confirmed_round, + self.id, + self.event_position, + ) + if self.event_order is None: + self.event_order = expected_order + elif self.event_order != expected_order: + raise InvalidLpProjectionError("event_order does not match the immutable event fields") diff --git a/flex/db/model/pools.py b/flex/db/model/pools.py index 17895a08..8d0fdfaf 100644 --- a/flex/db/model/pools.py +++ b/flex/db/model/pools.py @@ -5,14 +5,14 @@ from dataclasses_json import dataclass_json from flex.blockchain.info import ALGO_ASSET -from flex.db.model.blockchain import AssetInfo from flex.db.classes.base_entity import BaseEntity +from flex.db.model.blockchain import AssetInfo class PoolType(str, Enum): - STAKING = 'staking' - FARMING = 'farming' - ANY = 'any' + STAKING = "staking" + FARMING = "farming" + ANY = "any" @dataclass_json @@ -46,7 +46,7 @@ def length_blocks(self) -> int: @dataclass_json @dataclass -class StakingPool(BaseEntity['StakingPool']): +class StakingPool(BaseEntity["StakingPool"]): description: str address: str @@ -80,7 +80,9 @@ def to_info(self) -> PoolInfo: stake_token=self.stake_token, reward_token=self.reward_token, reward_amount_micros=self.reward_amount_micros, - reward_amount=self.stake_token.micros_to_amount(self.reward_amount_micros), + reward_amount=self.reward_token.micros_to_amount( + self.reward_amount_micros, + ), algo_reward_amount_micros=self.algo_reward_amount_micros, algo_reward_amount=ALGO_ASSET.micros_to_amount(self.algo_reward_amount_micros), begin_block=self.begin_block, @@ -88,13 +90,13 @@ def to_info(self) -> PoolInfo: lock_length_blocks=self.lock_length_blocks, deploy_date=self.deploy_date, begin_date=self.begin_date, - end_date=self.end_date + end_date=self.end_date, ) @dataclass_json @dataclass -class FarmingPool(BaseEntity['FarmingPool']): +class FarmingPool(BaseEntity["FarmingPool"]): description: str address: str @@ -141,7 +143,7 @@ def to_info(self) -> PoolInfo: lock_length_blocks=self.lock_length_blocks, deploy_date=self.deploy_date, begin_date=self.begin_date, - end_date=self.end_date + end_date=self.end_date, ) diff --git a/flex/db/model/priced.py b/flex/db/model/priced.py index 34d1f14a..27d6c4c9 100644 --- a/flex/db/model/priced.py +++ b/flex/db/model/priced.py @@ -1,9 +1,17 @@ from dataclasses import dataclass, field from datetime import UTC, datetime +from typing import ClassVar -from dataclasses_json import dataclass_json +from dataclasses_json import config, dataclass_json +from flex.db.bson import ( + decode_bson_uint64, + decode_optional_bson_uint64, + encode_bson_integer, + encode_optional_bson_uint64, +) from flex.db.classes.base_entity import BaseEntity +from flex.db.classes.bson_uint64 import BsonUint64StorageMixin from flex.db.model.pool_states import PoolStateInfo from flex.db.model.pools import PoolInfo from flex.db.util import get_uuid @@ -47,14 +55,34 @@ class UserCost: class AirdropReward(BaseEntity["AirdropReward"]): airdrop_id: str address: str - asa_id: int - amount_micros: int + asa_id: str + amount_micros: str txid: str + operation_id: str | None = None + confirmed_round: int | None = field( + default=None, + metadata=config( + encoder=encode_optional_bson_uint64, + decoder=decode_optional_bson_uint64, + ), + ) id: str = field(default_factory=get_uuid) created: datetime = field(default_factory=datetime.now) updated: datetime = field(default_factory=datetime.now) + def __post_init__(self) -> None: + self.asa_id = str(self.asa_id) + self.amount_micros = str(self.amount_micros) + + @property + def asa_id_int(self) -> int: + return int(self.asa_id) + + @property + def amount_micros_int(self) -> int: + return int(self.amount_micros) + @dataclass_json @dataclass @@ -69,14 +97,41 @@ class AssetPriceInfo: @dataclass_json @dataclass -class AssetPrice(BaseEntity["AssetPrice"]): - id: int +class AssetPrice( + BsonUint64StorageMixin, + BaseEntity["AssetPrice"], +): + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "id", + "last_update_round", + "tinyman_algo_pool_id", + } + ) + + id: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) price_usd: float price_algo: float - last_update_round: int + last_update_round: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) name: str - tinyman_algo_pool_id: int | None = None + tinyman_algo_pool_id: int | None = field( + default=None, + metadata=config( + encoder=encode_optional_bson_uint64, + decoder=decode_optional_bson_uint64, + ), + ) source: str | None = None observed_at: datetime | None = None diff --git a/flex/db/model/transfers.py b/flex/db/model/transfers.py new file mode 100644 index 00000000..2e67c2da --- /dev/null +++ b/flex/db/model/transfers.py @@ -0,0 +1,102 @@ +"""Persistence model for crash-safe outbound Algorand transfers.""" + +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import ClassVar + +from dataclasses_json import config, dataclass_json + +from flex.db.bson import ( + decode_bson_uint64, + decode_optional_bson_uint64, + encode_bson_integer, + encode_optional_bson_uint64, +) +from flex.db.classes.base_entity import BaseEntity +from flex.db.classes.bson_uint64 import BsonUint64StorageMixin +from flex.domain.algorand import require_algorand_uint64 + + +def _utc_now() -> datetime: + return datetime.now(UTC) + + +@dataclass_json +@dataclass +class AssetTransferIntent( + BsonUint64StorageMixin, + BaseEntity["AssetTransferIntent"], +): + """A signed transaction persisted before its first broadcast attempt.""" + + BSON_UINT64_FIELDS: ClassVar[frozenset[str]] = frozenset( + { + "first_valid_round", + "last_valid_round", + "confirmed_round", + } + ) + + id: str + receiver: str + asset_id: str + amount_micros: str + note: str | None + signed_transaction: str + txid: str + first_valid_round: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + last_valid_round: int = field( + metadata=config( + encoder=encode_bson_integer, + decoder=decode_bson_uint64, + ) + ) + status: str + + attempt_count: int = 0 + confirmed_round: int | None = field( + default=None, + metadata=config( + encoder=encode_optional_bson_uint64, + decoder=decode_optional_bson_uint64, + ), + ) + last_error: str | None = None + submitted_at: datetime | None = None + confirmed_at: datetime | None = None + created: datetime = field(default_factory=_utc_now) + updated: datetime = field(default_factory=_utc_now) + + def __post_init__(self) -> None: + # Legacy BSON documents may contain int64 values; new writes stay + # string-backed so the full Algorand uint64 domain is lossless. + self.asset_id = str(self.asset_id) + self.amount_micros = str(self.amount_micros) + self.first_valid_round = require_algorand_uint64( + self.first_valid_round, + "first_valid_round", + ) + self.last_valid_round = require_algorand_uint64( + self.last_valid_round, + "last_valid_round", + ) + if self.first_valid_round > self.last_valid_round: + raise ValueError("first_valid_round cannot exceed last_valid_round") + if self.confirmed_round is not None: + self.confirmed_round = require_algorand_uint64( + self.confirmed_round, + "confirmed_round", + ) + + @property + def amount_micros_int(self) -> int: + return int(self.amount_micros) + + @property + def asset_id_int(self) -> int: + return int(self.asset_id) diff --git a/flex/db/sync_coordinator.py b/flex/db/sync_coordinator.py new file mode 100644 index 00000000..44cb9ec0 --- /dev/null +++ b/flex/db/sync_coordinator.py @@ -0,0 +1,194 @@ +"""Mongo-backed single-writer ordering gate for the block projector.""" + +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any, cast + +from pymongo import ReturnDocument +from pymongo.collection import Collection + +from flex.db.bson import encode_bson_integer +from flex.db.model.blockchain import SyncState + +SYNC_STATE_ID = "main" + + +class SyncCoordinatorError(RuntimeError): + """Raised when a worker loses ownership of a claimed round.""" + + +@dataclass(slots=True) +class MongoSyncCoordinator: + collection: Collection[dict[str, Any]] + lease_duration: timedelta = timedelta(minutes=2) + + def claim_round( + self, + *, + owner: str, + expected_last_round: int | None, + round_number: int, + now: datetime | None = None, + ) -> SyncState | None: + self._validate_next_round( + expected_last_round=expected_last_round, + round_number=round_number, + ) + current_time = now or datetime.now(UTC) + encoded_expected_round = None if expected_last_round is None else encode_bson_integer(expected_last_round) + document = self.collection.find_one_and_update( + { + "id": SYNC_STATE_ID, + "last_round": encoded_expected_round, + "$or": [ + {"lease_until": {"$exists": False}}, + {"lease_until": None}, + {"lease_until": {"$lte": current_time}}, + {"lease_owner": owner}, + ], + }, + { + "$set": { + "claimed_round": encode_bson_integer(round_number), + "lease_owner": owner, + "lease_until": current_time + self.lease_duration, + "last_error": None, + "updated": current_time, + } + }, + return_document=ReturnDocument.AFTER, + ) + return self._from_document(document) + + def complete_round( + self, + *, + owner: str, + expected_last_round: int | None, + round_number: int, + now: datetime | None = None, + ) -> SyncState: + self._validate_next_round( + expected_last_round=expected_last_round, + round_number=round_number, + ) + current_time = now or datetime.now(UTC) + encoded_expected_round = None if expected_last_round is None else encode_bson_integer(expected_last_round) + document = self.collection.find_one_and_update( + { + "id": SYNC_STATE_ID, + "last_round": encoded_expected_round, + "claimed_round": encode_bson_integer(round_number), + "lease_owner": owner, + "lease_until": {"$gt": current_time}, + }, + { + "$set": { + "last_round": encode_bson_integer(round_number), + "claimed_round": None, + "lease_owner": None, + "lease_until": None, + "last_error": None, + "updated": current_time, + } + }, + return_document=ReturnDocument.AFTER, + ) + state = self._from_document(document) + if state is None: + raise SyncCoordinatorError(f"worker {owner!r} lost its claim for round {round_number}") + return state + + def advance_snapshot( + self, + *, + expected_last_round: int | None, + round_number: int, + now: datetime | None = None, + ) -> SyncState: + if ( + isinstance(round_number, bool) + or not isinstance(round_number, int) + or round_number < 0 + or (expected_last_round is not None and round_number < expected_last_round) + ): + raise SyncCoordinatorError("snapshot checkpoint cannot regress") + current_time = now or datetime.now(UTC) + encoded_expected_round = None if expected_last_round is None else encode_bson_integer(expected_last_round) + document = self.collection.find_one_and_update( + { + "id": SYNC_STATE_ID, + "last_round": encoded_expected_round, + "$or": [ + {"lease_until": {"$exists": False}}, + {"lease_until": None}, + {"lease_until": {"$lte": current_time}}, + ], + }, + { + "$set": { + "last_round": encode_bson_integer(round_number), + "claimed_round": None, + "lease_owner": None, + "lease_until": None, + "last_error": None, + "updated": current_time, + } + }, + return_document=ReturnDocument.AFTER, + ) + state = self._from_document(document) + if state is None: + raise SyncCoordinatorError("sync checkpoint changed during authoritative snapshot") + return state + + def release_after_error( + self, + *, + owner: str, + round_number: int, + error: str, + now: datetime | None = None, + ) -> None: + current_time = now or datetime.now(UTC) + self.collection.update_one( + { + "id": SYNC_STATE_ID, + "claimed_round": encode_bson_integer(round_number), + "lease_owner": owner, + }, + { + "$set": { + "claimed_round": None, + "lease_owner": None, + "lease_until": None, + "last_error": error[:500], + "updated": current_time, + } + }, + ) + + @staticmethod + def _from_document(document: dict[str, Any] | None) -> SyncState | None: + if document is None: + return None + payload = dict(document) + payload.pop("_id", None) + return cast( + SyncState, + SyncState.from_dict(payload), + ) + + @staticmethod + def _validate_next_round( + *, + expected_last_round: int | None, + round_number: int, + ) -> None: + if ( + isinstance(round_number, bool) + or not isinstance(round_number, int) + or round_number < 0 + or (expected_last_round is not None and round_number != expected_last_round + 1) + ): + raise SyncCoordinatorError("projected rounds must advance exactly once") diff --git a/flex/domain/algorand.py b/flex/domain/algorand.py new file mode 100644 index 00000000..70a0f6f7 --- /dev/null +++ b/flex/domain/algorand.py @@ -0,0 +1,17 @@ +"""Algorand protocol value boundaries shared by external adapters.""" + +MAX_ALGORAND_UINT = 2**64 - 1 + + +def require_algorand_uint64( + value: object, + field: str, + *, + positive: bool = False, +) -> int: + """Return an exact protocol integer or reject coercible lookalikes.""" + + if isinstance(value, bool) or not isinstance(value, int) or value < int(positive) or value > MAX_ALGORAND_UINT: + requirement = "a positive Algorand uint64" if positive else "an Algorand uint64" + raise ValueError(f"{field} must be {requirement}") + return value diff --git a/flex/domain/allocation.py b/flex/domain/allocation.py new file mode 100644 index 00000000..d935ed8b --- /dev/null +++ b/flex/domain/allocation.py @@ -0,0 +1,80 @@ +"""Exact, deterministic allocation of integer asset base units.""" + +from collections.abc import Mapping +from decimal import Decimal, InvalidOperation +from fractions import Fraction + +type ShareInput = Decimal | int | float | str + +MAX_SHARE_EXPONENT = 300 +MAX_SHARE_SIGNIFICANT_DIGITS = 50 + + +class AllocationError(ValueError): + """Raised when an allocation cannot preserve its financial invariants.""" + + +def _positive_decimal(value: ShareInput, *, field: str) -> Decimal: + if isinstance(value, bool): + raise AllocationError(f"{field} must be numeric, not bool") + try: + parsed = value if isinstance(value, Decimal) else Decimal(str(value)) + except (InvalidOperation, ValueError) as exc: + raise AllocationError(f"{field} is not a valid decimal") from exc + + if not parsed.is_finite(): + raise AllocationError(f"{field} must be finite") + if parsed <= 0: + raise AllocationError(f"{field} must be positive") + if abs(parsed.adjusted()) > MAX_SHARE_EXPONENT: + raise AllocationError(f"{field} exponent is outside the supported range") + if len(parsed.as_tuple().digits) > MAX_SHARE_SIGNIFICANT_DIGITS: + raise AllocationError(f"{field} has too many significant digits") + return parsed + + +def allocate_proportionally( + total_micros: int, + shares: Mapping[str, ShareInput], +) -> dict[str, int]: + """Allocate an integer budget exactly using the largest-remainder method. + + Fractional remainders are resolved by recipient ID, making the result + reproducible regardless of mapping insertion order. + """ + + if isinstance(total_micros, bool) or not isinstance(total_micros, int): + raise AllocationError("total_micros must be an integer") + if total_micros <= 0: + raise AllocationError("total_micros must be positive") + if not shares: + raise AllocationError("shares must not be empty") + + weights: dict[str, Fraction] = {} + for recipient, share in shares.items(): + if not isinstance(recipient, str) or not recipient.strip(): + raise AllocationError("recipient IDs must be non-empty strings") + parsed = _positive_decimal(share, field=f"shares[{recipient!r}]") + weights[recipient] = Fraction(parsed) + + total_weight = sum(weights.values(), start=Fraction()) + allocations: dict[str, int] = {} + remainders: list[tuple[Fraction, str]] = [] + + for recipient in sorted(weights): + exact_amount = Fraction(total_micros) * weights[recipient] / total_weight + floor_amount = exact_amount.numerator // exact_amount.denominator + allocations[recipient] = floor_amount + remainders.append((exact_amount - floor_amount, recipient)) + + undistributed = total_micros - sum(allocations.values()) + ranked_remainders = sorted( + remainders, + key=lambda item: (-item[0], item[1]), + ) + for _, recipient in ranked_remainders[:undistributed]: + allocations[recipient] += 1 + + if sum(allocations.values()) != total_micros: + raise AssertionError("allocation must preserve the integer budget") + return allocations diff --git a/flex/domain/lp_projection.py b/flex/domain/lp_projection.py new file mode 100644 index 00000000..db1034c9 --- /dev/null +++ b/flex/domain/lp_projection.py @@ -0,0 +1,125 @@ +"""Pure invariants for ordered liquidity-pool projections.""" + +from dataclasses import dataclass +from typing import Literal + +from flex.domain.algorand import ( + MAX_ALGORAND_UINT, + require_algorand_uint64, +) + +EVENT_ROUND_WIDTH = 20 +ROUND_END_SUFFIX = "~" + +type LpBalanceField = Literal[ + "asset1_reserve_micros", + "asset2_reserve_micros", + "total_tokens_micros", + "operational_algo_balance_micros", +] + + +class InvalidLpProjectionError(ValueError): + """Raised when an LP event cannot safely update its target aggregate.""" + + +@dataclass(frozen=True, slots=True) +class LpBalanceDelta: + field: LpBalanceField + amount: int + + +def lp_event_order( + confirmed_round: int, + event_id: str, + event_position: int = 0, +) -> str: + """Build a lexicographically sortable, deterministic event cursor.""" + + try: + require_algorand_uint64(confirmed_round, "confirmed_round") + except ValueError as exc: + raise InvalidLpProjectionError(str(exc)) from exc + if not isinstance(event_id, str) or not event_id or ":" in event_id: + raise InvalidLpProjectionError("event_id must be non-empty and cannot contain ':'") + try: + require_algorand_uint64(event_position, "event_position") + except ValueError as exc: + raise InvalidLpProjectionError(str(exc)) from exc + return f"{confirmed_round:0{EVENT_ROUND_WIDTH}d}:{event_position:0{EVENT_ROUND_WIDTH}d}:{event_id}" + + +def lp_round_end_order(confirmed_round: int) -> str: + """Represent an authoritative snapshot taken after an entire round.""" + + prefix = lp_event_order(confirmed_round, "snapshot").partition(":")[0] + return f"{prefix}:{ROUND_END_SUFFIX}" + + +def snapshot_covers_event(cursor: str, *, confirmed_round: int) -> bool: + """Return whether a round-end snapshot is authoritative for an event.""" + + round_prefix, separator, suffix = cursor.partition(":") + return ( + separator == ":" + and suffix == ROUND_END_SUFFIX + and len(round_prefix) == EVENT_ROUND_WIDTH + and round_prefix.isdigit() + and int(round_prefix) >= confirmed_round + ) + + +def lp_cursor_complete_through(cursor: str | None) -> int: + """Return the last round fully covered by a persisted LP cursor.""" + + if cursor is None: + raise InvalidLpProjectionError("LP cursor is missing") + round_prefix, separator, suffix = cursor.partition(":") + if separator != ":" or len(round_prefix) != EVENT_ROUND_WIDTH or not round_prefix.isdigit() or not suffix: + raise InvalidLpProjectionError("LP cursor has an invalid format") + round_number = int(round_prefix) + if suffix == ROUND_END_SUFFIX: + return round_number + if round_number == 0: + raise InvalidLpProjectionError("an event cursor in round zero covers no complete round") + return round_number - 1 + + +def lp_balance_delta( + *, + token_id: int, + asset1_id: int, + asset2_id: int, + event_asset_id: int, + event_pool_delta_micros: int, +) -> LpBalanceDelta: + """Map one pool-account transfer to its canonical LP balance field.""" + + if isinstance(event_pool_delta_micros, bool) or not isinstance(event_pool_delta_micros, int): + raise InvalidLpProjectionError("event delta must be an integer") + if not -MAX_ALGORAND_UINT <= event_pool_delta_micros <= MAX_ALGORAND_UINT: + raise InvalidLpProjectionError("event delta is outside the Algorand uint64 range") + + if event_asset_id == token_id: + return LpBalanceDelta( + field="total_tokens_micros", + amount=-event_pool_delta_micros, + ) + if event_asset_id == asset1_id: + return LpBalanceDelta( + field="asset1_reserve_micros", + amount=event_pool_delta_micros, + ) + if event_asset_id == asset2_id: + return LpBalanceDelta( + field="asset2_reserve_micros", + amount=event_pool_delta_micros, + ) + if event_asset_id == 0: + return LpBalanceDelta( + field="operational_algo_balance_micros", + amount=event_pool_delta_micros, + ) + raise InvalidLpProjectionError( + f"asset {event_asset_id} does not belong to LP token {token_id}", + ) diff --git a/flex/domain/pricing.py b/flex/domain/pricing.py index e87c20ea..b5cb9975 100644 --- a/flex/domain/pricing.py +++ b/flex/domain/pricing.py @@ -13,6 +13,7 @@ MAX_ASSET_DECIMALS = 19 CALCULATION_PRECISION = 80 PERSISTED_PRICE_PRECISION = 34 +MAX_OBSERVATION_CLOCK_SKEW = timedelta(minutes=1) class PricingError(ValueError): @@ -66,6 +67,20 @@ def _utc(value: datetime) -> datetime: return value.astimezone(UTC) +def validate_observation_timestamp( + observed_at: datetime, + *, + now: datetime | None = None, +) -> datetime: + """Reject timestamps that could pin a quote ahead of wall-clock time.""" + + observed = _utc(observed_at) + current_time = _utc(now or datetime.now(UTC)) + if observed > current_time + MAX_OBSERVATION_CLOCK_SKEW: + raise InvalidPriceError("observed_at is too far in the future") + return observed + + def _legacy_float(value: Decimal, *, field: str) -> float: converted = float(value) if not isfinite(converted) or (value != 0 and converted == 0): @@ -73,6 +88,27 @@ def _legacy_float(value: Decimal, *, field: str) -> float: return converted +def base_units_to_decimal( + amount_micros: int, + *, + decimals: int, + field: str = "amount_micros", +) -> Decimal: + """Convert integer base units without crossing a float boundary.""" + + amount = _non_negative_int(amount_micros, field=field) + precision = _asset_decimals(decimals, field=f"{field}_decimals") + with localcontext() as context: + context.prec = CALCULATION_PRECISION + return Decimal(amount) / Decimal(10**precision) + + +def decimal_to_legacy_float(value: DecimalInput, *, field: str) -> float: + """Convert an exact value only at a legacy float API/storage boundary.""" + + return _legacy_float(_decimal(value, field=field), field=field) + + @dataclass(frozen=True, slots=True) class PriceQuote: """A validated provider observation before legacy float persistence.""" @@ -135,6 +171,8 @@ def from_raw( def is_stale(self, *, now: datetime | None = None) -> bool: current_time = _utc(now or datetime.now(UTC)) + if self.observed_at > current_time + MAX_OBSERVATION_CLOCK_SKEW: + return True return current_time - self.observed_at >= self.stale_after def to_legacy_floats(self) -> tuple[float, float]: @@ -153,7 +191,10 @@ def is_observation_stale( if fresh_for <= timedelta(0): raise ValueError("fresh_for must be positive") current_time = _utc(now or datetime.now(UTC)) - return current_time - _utc(observed_at) >= fresh_for + observed = _utc(observed_at) + if observed > current_time + MAX_OBSERVATION_CLOCK_SKEW: + return True + return current_time - observed >= fresh_for def _non_negative_int(value: int, *, field: str) -> int: @@ -195,13 +236,62 @@ def calculate_lp_token_price_algo( if pool_balance_micros >= total_supply_micros: raise InvalidLiquidityPoolError("circulating LP supply must be positive") + return calculate_lp_token_price_from_issued_supply( + asset1_price_algo=price, + asset1_reserve_micros=reserve_micros, + asset1_decimals=asset_decimals, + issued_lp_supply_micros=total_supply_micros - pool_balance_micros, + lp_token_decimals=lp_decimals, + ) + + +def calculate_lp_token_price_from_issued_supply( + *, + asset1_price_algo: DecimalInput, + asset1_reserve_micros: int, + asset1_decimals: int, + issued_lp_supply_micros: int, + lp_token_decimals: int, +) -> Decimal: + """Calculate LP price from an already-derived issued token supply.""" + + price = _decimal(asset1_price_algo, field="asset1_price_algo") + if price <= 0: + raise InvalidLiquidityPoolError("asset1_price_algo must be positive") + reserve_micros = _non_negative_int( + asset1_reserve_micros, + field="asset1_reserve_micros", + ) + issued_micros = _non_negative_int( + issued_lp_supply_micros, + field="issued_lp_supply_micros", + ) + asset_decimals = _asset_decimals( + asset1_decimals, + field="asset1_decimals", + ) + lp_decimals = _asset_decimals( + lp_token_decimals, + field="lp_token_decimals", + ) + if reserve_micros == 0: + raise InvalidLiquidityPoolError("asset1 reserve must be positive") + if issued_micros == 0: + raise InvalidLiquidityPoolError("issued LP supply must be positive") + with localcontext() as context: context.prec = CALCULATION_PRECISION - reserve = Decimal(reserve_micros) / Decimal(10**asset_decimals) - circulating_supply = Decimal( - total_supply_micros - pool_balance_micros, - ) / Decimal(10**lp_decimals) - calculated = price * reserve * Decimal(2) / circulating_supply + reserve = base_units_to_decimal( + reserve_micros, + decimals=asset_decimals, + field="asset1_reserve_micros", + ) + issued_supply = base_units_to_decimal( + issued_micros, + decimals=lp_decimals, + field="issued_lp_supply_micros", + ) + calculated = price * reserve * Decimal(2) / issued_supply if not calculated.is_finite() or calculated <= 0: raise InvalidLiquidityPoolError("calculated LP price is invalid") with localcontext() as context: diff --git a/flex/migrations/fix_dex_providers.py b/flex/migrations/fix_dex_providers.py index ef3253d6..84520afe 100644 --- a/flex/migrations/fix_dex_providers.py +++ b/flex/migrations/fix_dex_providers.py @@ -1,34 +1,36 @@ import logging from flex import db -from flex.providers.vestige import is_valid_dex_provider, get_dex_tag_by_name - +from flex.providers.vestige import get_dex_tag_by_name, is_valid_dex_provider logger = logging.getLogger(__name__) def fix_dex_names() -> None: - logger.info('Fixing dex names.') + logger.info("Fixing dex names.") lp_tokens = db.lp_tokens.get_all() for lp_token in lp_tokens: if not is_valid_dex_provider(lp_token.dex_provider): - logger.info(f'Invalid DEX in LP token:\n{lp_token.pretty_str()}') + logger.info(f"Invalid DEX in LP token:\n{lp_token.pretty_str()}") lp_token.dex_provider = get_dex_tag_by_name(lp_token.dex_provider) db.lp_tokens.update(lp_token) farming_pools = db.farming_pools.get_all() for farming_pool in farming_pools: if not is_valid_dex_provider(farming_pool.dex_name): - logger.info(f'Invalid DEX in farming pool:\n{farming_pool.pretty_str()}') + logger.info(f"Invalid DEX in farming pool:\n{farming_pool.pretty_str()}") farming_pool.dex_name = get_dex_tag_by_name(farming_pool.dex_name) db.farming_pools.update(farming_pool) lp_states = db.lp_states.get_all() for lp_state in lp_states: if not is_valid_dex_provider(lp_state.dex_provider): - logger.info(f'Invalid DEX in LP state:\n{lp_state.pretty_str()}') + logger.info(f"Invalid DEX in LP state:\n{lp_state.pretty_str()}") lp_state.dex_provider = get_dex_tag_by_name(lp_state.dex_provider) - db.lp_states.update(lp_state) + db.lp_states.update_with( + lp_state, + dex_provider=lp_state.dex_provider, + ) - logger.info('All DEX names are valid now!') + logger.info("All DEX names are valid now!") diff --git a/flex/providers/price_router.py b/flex/providers/price_router.py index 9a453bcf..ae12cb23 100644 --- a/flex/providers/price_router.py +++ b/flex/providers/price_router.py @@ -8,6 +8,7 @@ from env import settings from flex import db +from flex.data.asset_prices import is_unverified_legacy_lp_price from flex.domain.pricing import ( DecimalInput, PriceQuote, @@ -163,7 +164,12 @@ def _stored_quote( now: datetime, max_age: timedelta, ) -> PriceQuote | None: - if record is None: + if record is None or is_unverified_legacy_lp_price(record): + if record is not None: + logger.warning( + "Ignoring retired raw-reserve LP projection for asset %s", + asset_id, + ) return None try: observed_at = _record_observed_at(record) diff --git a/flex/sync_pools.py b/flex/sync_pools.py index 6444310b..1d132073 100644 --- a/flex/sync_pools.py +++ b/flex/sync_pools.py @@ -1,14 +1,14 @@ import asyncio import logging +import random +from uuid import uuid4 from aiocache import cached as cached_async -from cachetools import TTLCache, cached from env import settings from flex import db from flex.blockchain.base import indexer_client from flex.blockchain.info import get_current_round -from flex.data.asset_prices import create_and_update_asset_prices from flex.data.assets import load_all_assets_data from flex.data.lp_states import ( create_lp_states_from_all_pools, @@ -17,16 +17,20 @@ ) from flex.data.pool_state import ( get_or_create_pool_state, - update_all_pool_states_linear, update_pool_state, update_pool_states_with_transactions, ) -from flex.data.tinyman_lps import update_tinyman_algo_lp_state_and_prices from flex.db.model.blockchain import PoolTransaction, SyncBlock, SyncState from flex.db.model.liquidity_pools import LpState, LpTransaction from flex.db.model.pool_states import PoolState, UserState -from flex.db.model.priced import AssetPrice -from flex.domain.pricing import PricingError +from flex.db.sync_coordinator import ( + MongoSyncCoordinator, + SyncCoordinatorError, +) +from flex.domain.algorand import require_algorand_uint64 +from flex.domain.lp_projection import ( + lp_cursor_complete_through, +) from flex.domain.transactions import ( ASSET_TRANSFER_TX, PAYMENT_TX, @@ -34,16 +38,24 @@ flatten_transfer_payments, projection_event_id, ) -from flex.providers.price_router import get_algo_price_quote from flex.sync_state import get_sync_state, is_sync_delayed from flex.util import build_key_str logger = logging.getLogger(__name__) -@cached(cache=TTLCache(maxsize=1, ttl=30)) -def get_all_lp_state_addresses() -> set[str]: - return set(lp_state.address for lp_state in db.lp_states.get_all()) +def get_lp_tracked_assets_by_address() -> dict[str, frozenset[int]]: + return { + lp_state.address: frozenset( + { + 0, + lp_state.asset1_id, + lp_state.asset2_id, + lp_state.token_id, + } + ) + for lp_state in db.lp_states.get_all() + } def find_transfer_payment_transactions(txns: list[dict]) -> list[dict]: @@ -59,27 +71,91 @@ async def get_pool_state_by_address(address: str) -> PoolState: return db.pool_states.get_one(address=address) +def _transaction_fee(tx: dict, *, txid: str) -> int: + try: + return require_algorand_uint64( + tx.get("fee"), + "LP transaction fee", + ) + except ValueError as exc: + raise ValueError(f"{exc}: {txid}") from exc + + async def process_lp_transactions(transactions: list[dict]) -> list[LpTransaction]: - all_lp_addresses = get_all_lp_state_addresses() + tracked_assets_by_address = get_lp_tracked_assets_by_address() + all_lp_addresses = tracked_assets_by_address.keys() lp_transactions = [] - for tx in transactions: + for event_position, tx in enumerate(transactions): txid = tx["id"] sender = tx["sender"] - confirmed_round = tx["confirmed-round"] + confirmed_round = require_algorand_uint64( + tx.get("confirmed-round"), + "LP transaction confirmed-round", + ) if ASSET_TRANSFER_TX in tx: - asa_id = tx[ASSET_TRANSFER_TX]["asset-id"] - receiver = tx[ASSET_TRANSFER_TX]["receiver"] - amount = tx[ASSET_TRANSFER_TX]["amount"] + transfer = tx[ASSET_TRANSFER_TX] + receiver = transfer["receiver"] + asa_id = require_algorand_uint64( + transfer.get("asset-id"), + "LP transaction asset-id", + positive=True, + ) + amount = require_algorand_uint64( + transfer.get("amount"), + "LP transaction amount", + ) + affected_addresses = { + sender, + receiver, + transfer.get("sender"), + transfer.get("close-to"), + } + if not (affected_addresses & all_lp_addresses): + continue + if any(field in transfer for field in ("sender", "close-to", "close-amount")): + tracked_affected_pool = any( + address in tracked_assets_by_address and asa_id in tracked_assets_by_address[address] + for address in affected_addresses + if address is not None + ) + if tracked_affected_pool: + raise ValueError(f"LP projection does not support clawback/close semantics: {txid}") elif PAYMENT_TX in tx: + payment = tx[PAYMENT_TX] + receiver = payment["receiver"] + affected_addresses = { + sender, + receiver, + payment.get("close-remainder-to"), + } + if not (affected_addresses & all_lp_addresses): + continue + if any(field in payment for field in ("close-remainder-to", "close-amount")): + raise ValueError(f"LP projection does not support payment close semantics: {txid}") asa_id = 0 - receiver = tx[PAYMENT_TX]["receiver"] - amount = tx[PAYMENT_TX]["amount"] + amount = require_algorand_uint64( + payment.get("amount"), + "LP transaction amount", + ) else: raise ValueError(f"Invalid transaction type: {tx}") - if sender in all_lp_addresses: + fee = _transaction_fee(tx, txid=txid) + sender_assets = tracked_assets_by_address.get(sender) + receiver_assets = tracked_assets_by_address.get(receiver) + sender_tracks_asset = sender_assets is not None and asa_id in sender_assets + receiver_tracks_asset = receiver_assets is not None and asa_id in receiver_assets + is_pool_self_transfer = sender == receiver and sender_tracks_asset + if is_pool_self_transfer: + # A self-transfer has zero net effect on the pool. Emitting both + # legs would create the same scoped event ID twice and make a + # deduplicating projector persist only one side. Its network fee + # is still a real ALGO debit and is projected separately below. + pass + + elif sender_tracks_asset: lp_tx = LpTransaction( id=projection_event_id(txid, sender), pool_address=sender, @@ -87,10 +163,11 @@ async def process_lp_transactions(transactions: list[dict]) -> list[LpTransactio asa_id=asa_id, delta_amount_micros=-amount, confirmed_round=confirmed_round, + event_position=event_position, ) lp_transactions.append(lp_tx) - if receiver in all_lp_addresses: + if not is_pool_self_transfer and receiver_tracks_asset: lp_tx = LpTransaction( id=projection_event_id(txid, receiver), pool_address=receiver, @@ -98,9 +175,23 @@ async def process_lp_transactions(transactions: list[dict]) -> list[LpTransactio asa_id=asa_id, delta_amount_micros=amount, confirmed_round=confirmed_round, + event_position=event_position, ) lp_transactions.append(lp_tx) + if sender_assets is not None and fee: + lp_transactions.append( + LpTransaction( + id=projection_event_id(f"{txid}#fee", sender), + pool_address=sender, + user_address=sender, + asa_id=0, + delta_amount_micros=-fee, + confirmed_round=confirmed_round, + event_position=event_position, + ) + ) + return lp_transactions @@ -151,35 +242,38 @@ async def update_pools(txns: list[dict]) -> [PoolState]: return await update_pool_states_with_transactions(pool_transactions) -async def update_lp_states(txns: list[dict]) -> list[LpState]: +async def update_lp_states( + txns: list[dict], + *, + expected_round: int, +) -> list[LpState]: lp_transactions = await process_lp_transactions(txns) - return await update_lp_states_with_transactions(lp_transactions) - + return await update_lp_states_with_transactions( + lp_transactions, + expected_round=expected_round, + ) -async def update_asset_prices(updated_lp_states: list[LpState]) -> list[AssetPrice]: - if len(updated_lp_states) == 0: - return [] - algo_quote = await get_algo_price_quote() - updated_asset_prices = [] - for lp_state in updated_lp_states: - if lp_state.is_algo_pool: - try: - updated_asset_price = await update_tinyman_algo_lp_state_and_prices( - lp_state, - algo_quote, - ) - updated_asset_prices.append(updated_asset_price) - except PricingError as exc: - logger.warning( - "Skipping invalid Tinyman LP state %s: %s", - lp_state.id, - exc, - ) +def _snapshot_checkpoint_round( + *, + previous_round: int | None, + node_round: int, + lp_states: list[LpState], +) -> int: + """Choose a checkpoint covered by every persisted LP snapshot.""" - if len(updated_asset_prices) > 0: - logger.debug(f"Updated {len(updated_asset_prices)} asset prices.") - return updated_asset_prices + checkpoint = min( + [ + node_round, + *(lp_cursor_complete_through(state.last_event_order) for state in lp_states), + ], + ) + if previous_round is not None and checkpoint < previous_round: + raise SyncCoordinatorError( + "Indexer snapshots lag behind the persisted sync checkpoint; " + "wait for Indexer catch-up before resuming projection" + ) + return checkpoint async def catch_up_the_sync_manually(sync_state: SyncState, current_round: int) -> SyncState: @@ -189,41 +283,81 @@ async def catch_up_the_sync_manually(sync_state: SyncState, current_round: int) ) _ = await load_all_assets_data() - _ = await update_all_pool_states_linear(reset_pool_states=True) + if settings.sync_staking_pools: + raise RuntimeError( + "legacy staking-pool projection is disabled until full Algorand group validation is implemented" + ) current_round = await get_current_round() logger.info(f"\n\nAnother, shorter loop, starting from round {current_round}\n") - await update_all_pool_states_linear() - - logger.info("\n\nSyncing LP states linearly.\n") - _ = await create_lp_states_from_all_pools() - _ = await update_all_lp_states_linear() - _ = await create_and_update_asset_prices() + if settings.sync_liquidity_pools: + logger.info("\n\nSyncing LP states from authoritative account snapshots.\n") + _ = await create_lp_states_from_all_pools() + snapshotted_states = await update_all_lp_states_linear() + else: + snapshotted_states = [] + + checkpoint_round = _snapshot_checkpoint_round( + previous_round=sync_state.last_round, + node_round=current_round, + lp_states=snapshotted_states, + ) - sync_state.last_round = current_round - db.sync_states.update(sync_state) - logger.info(f"\n\nALL synced up to round {current_round}.\n") + coordinator = MongoSyncCoordinator( + db.sync_states.mongodb_collection, + ) + sync_state = await asyncio.to_thread( + coordinator.advance_snapshot, + expected_last_round=sync_state.last_round, + round_number=checkpoint_round, + ) + logger.info(f"\n\nALL synced up to round {checkpoint_round}.\n") return sync_state async def sync_pools_loop(): + if settings.sync_staking_pools: + raise RuntimeError( + "legacy staking-pool projection is disabled until full Algorand group validation is implemented" + ) + if not settings.sync_liquidity_pools: + logger.warning( + "Financial sync is disabled; set SYNC_LIQUIDITY_POOLS=true only after reviewing the snapshot cutover" + ) + return + current_round = await get_current_round() logger.info(f"\n\nEnter sync loop. Current round = {current_round}") sync_state = await get_sync_state() - if sync_state.last_round is None or sync_state.rounds_since_updated(current_round) > settings.sync_lag_max_rounds: + previous_round = sync_state.last_round + try: sync_state = await catch_up_the_sync_manually(sync_state, current_round) + except SyncCoordinatorError: + # A competing worker may have completed the same authoritative cutover. + refreshed = await get_sync_state() + if refreshed.last_round == previous_round: + raise + sync_state = refreshed logger.info("\n\nMain BLOCKCHAIN sync loop!\n") + coordinator = MongoSyncCoordinator( + db.sync_states.mongodb_collection, + ) + worker_id = uuid4().hex no_block_seconds = 0 + processing_attempts = 0 MAX_BLOCK_DELAY_SECONDS = 10 while True: next_round = sync_state.last_round + 1 try: - block_dict = indexer_client.block_info(round_num=next_round) + block_dict = await asyncio.to_thread( + indexer_client.block_info, + round_num=next_round, + ) except Exception as e: no_block_seconds += 1 if no_block_seconds > MAX_BLOCK_DELAY_SECONDS: @@ -234,21 +368,67 @@ async def sync_pools_loop(): no_block_seconds = 0 + claimed = await asyncio.to_thread( + coordinator.claim_round, + owner=worker_id, + expected_last_round=sync_state.last_round, + round_number=next_round, + ) + if claimed is None: + await asyncio.sleep(0.25) + sync_state = await get_sync_state() + continue + try: raw_transactions = block_dict["transactions"] logger.debug(f"Fetch #{next_round}: sync {len(raw_transactions)} txns") - updated_pool_states = await update_pools(find_transfer_transactions(raw_transactions)) - updated_lp_states = await update_lp_states(find_transfer_payment_transactions(raw_transactions)) - _ = await update_asset_prices(updated_lp_states) + if settings.sync_staking_pools: + raise RuntimeError( + "legacy staking-pool projection is disabled until full Algorand group validation is implemented" + ) + updated_pool_states: list[PoolState] = [] + updated_lp_states = ( + await update_lp_states( + find_transfer_payment_transactions(raw_transactions), + expected_round=next_round, + ) + if settings.sync_liquidity_pools + else [] + ) - sync_state.last_round = next_round - db.sync_states.update(sync_state) - db.sync_blocks.create(SyncBlock(round=next_round, timestamp=block_dict["timestamp"])) + sync_state = await asyncio.to_thread( + coordinator.complete_round, + owner=worker_id, + expected_last_round=sync_state.last_round, + round_number=next_round, + ) + db.sync_blocks.get_or_create( + SyncBlock( + round=next_round, + timestamp=block_dict["timestamp"], + ) + ) + processing_attempts = 0 logger.debug(f"#{next_round} sync OK! Saved {len(updated_pool_states) + len(updated_lp_states)} txns\n") except Exception as e: + processing_attempts += 1 + await asyncio.to_thread( + coordinator.release_after_error, + owner=worker_id, + round_number=next_round, + error=f"{type(e).__name__}: {e}", + ) logger.error(f"Error processing round {next_round}: {e}", exc_info=True) + if processing_attempts >= settings.sync_round_max_attempts: + raise RuntimeError(f"round {next_round} failed {processing_attempts} consecutive times") from e + delay_cap = min( + 2 ** (processing_attempts - 1), + settings.sync_retry_max_seconds, + ) + await asyncio.sleep(random.uniform(0, delay_cap)) + sync_state = await get_sync_state() continue diff --git a/flex/sync_state.py b/flex/sync_state.py index 14ff5ea6..890d345f 100644 --- a/flex/sync_state.py +++ b/flex/sync_state.py @@ -7,10 +7,7 @@ async def get_sync_state() -> SyncState: - sync_state = db.sync_states.get_one() - if sync_state is None: - sync_state = db.sync_states.create(SyncState()) - return sync_state + return db.sync_states.get_or_create(SyncState()) sync_max_delay = timedelta(seconds=settings.sync_behind_seconds_threshold) @@ -20,4 +17,5 @@ async def get_sync_state() -> SyncState: async def is_sync_delayed(current_round: int | None = None) -> bool: current_round = current_round or (await get_current_round()) sync_state = await get_sync_state() - return sync_state.rounds_since_updated(current_round) > sync_max_delay_rounds + rounds_behind = sync_state.rounds_since_updated(current_round) + return rounds_behind is None or rounds_behind > sync_max_delay_rounds diff --git a/flex/tools/airdrop.py b/flex/tools/airdrop.py index cd740b0c..18e9fa5b 100644 --- a/flex/tools/airdrop.py +++ b/flex/tools/airdrop.py @@ -1,91 +1,529 @@ +"""Operational airdrop tool with exact allocation and replay-safe transfers.""" + import json import logging -import random +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from hashlib import sha256 +from pathlib import Path + +from algosdk import encoding +from pymongo import ReturnDocument from flex import db +from flex.application.asset_transfers import ( + MAX_ALGORAND_UINT, + AssetTransferReceipt, + AssetTransferRequest, + AssetTransferService, +) +from flex.application.transfer_runtime import get_asset_transfer_service from flex.blockchain.base import cometa_public_key -from flex.blockchain.info import get_current_round, is_opted_in +from flex.blockchain.info import get_current_round +from flex.db.indexes import ensure_airdrop_indexes +from flex.db.model.airdrop import AirdropManifest from flex.db.model.blockchain import AssetInfo from flex.db.model.priced import AirdropReward -from flex.txns import TxInfo, send_asset_micros_with_wait, send_asset_micros +from flex.domain.allocation import ShareInput, allocate_proportionally +from flex.txns import TxInfo logger = logging.getLogger(__name__) +class AirdropError(RuntimeError): + """Base class for invalid or incomplete airdrop execution.""" + + +class AirdropConflictError(AirdropError): + """Raised when an airdrop ID is reused with different transfer details.""" + + +@dataclass(frozen=True, slots=True) +class AirdropFailure: + address: str + error_type: str + txid: str | None = None + + +class AirdropIncompleteError(AirdropError): + """Raised after all recipients were attempted and at least one is unresolved.""" + + def __init__( + self, + failures: Sequence[AirdropFailure], + confirmed_transactions: Sequence[TxInfo], + ) -> None: + super().__init__(f"airdrop incomplete: {len(failures)} recipient(s) unresolved; rerun with the same airdrop_id") + self.failures = tuple(failures) + self.confirmed_transactions = tuple(confirmed_transactions) + + +def _operation_id(airdrop_id: str, address: str) -> str: + return f"airdrop:{airdrop_id}:{address}" + + +def _select_note(operation_id: str, notes: Sequence[str]) -> str: + digest = sha256(operation_id.encode()).digest() + index = int.from_bytes(digest[:8], "big") % len(notes) + return notes[index] + + +def _validate_airdrop( + *, + airdrop_id: str, + addresses: Sequence[str], + notes: Sequence[str], + amounts: Mapping[str, int], +) -> None: + if not isinstance(airdrop_id, str) or not airdrop_id.strip(): + raise AirdropError("airdrop_id must be a non-empty string") + if isinstance(notes, (str, bytes)) or not notes: + raise AirdropError("notes must not be empty") + if any(not isinstance(note, str) or len(note.encode()) > 1_000 for note in notes): + raise AirdropError("each note must be a string within Algorand's 1000-byte limit") + invalid_addresses = [address for address in addresses if not encoding.is_valid_address(address)] + if invalid_addresses: + raise AirdropError(f"airdrop contains {len(invalid_addresses)} invalid Algorand address(es)") + zero_allocations = [address for address, amount in amounts.items() if amount == 0] + if zero_allocations: + raise AirdropError( + f"airdrop budget is too small: {len(zero_allocations)} recipient(s) would receive zero base units" + ) + + +def _assert_reward_matches( + reward: AirdropReward, + *, + asset_id: int, + amount_micros: int, +) -> None: + if (reward.asa_id_int, reward.amount_micros_int) != (asset_id, amount_micros): + raise AirdropConflictError( + f"existing reward {reward.airdrop_id!r}/{reward.address} has different transfer details" + ) + + +def _get_existing_reward( + *, + airdrop_id: str, + address: str, +) -> AirdropReward | None: + rewards = db.airdrop_rewards.get_many( + address=address, + airdrop_id=airdrop_id, + ) + if len(rewards) > 1: + raise AirdropError( + f"multiple legacy rewards exist for {airdrop_id!r}/{address}; manual reconciliation required" + ) + return rewards[0] if rewards else None + + +def _manifest_hash( + *, + airdrop_id: str, + asset_id: int, + total_amount_micros: int, + requests: Mapping[str, AssetTransferRequest], +) -> str: + canonical = { + "airdrop_id": airdrop_id, + "asset_id": asset_id, + "total_amount_micros": str(total_amount_micros), + "recipients": [ + [ + address, + str(requests[address].amount_micros), + requests[address].note, + ] + for address in sorted(requests) + ], + } + encoded = json.dumps( + canonical, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode() + return sha256(encoded).hexdigest() + + +def _reserve_manifest( + manifest: AirdropManifest, +) -> AirdropManifest: + collection = db.airdrop_manifests.mongodb_collection + document = collection.find_one_and_update( + {"id": manifest.id}, + {"$setOnInsert": manifest.to_dict()}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + if document is None: + raise AirdropError(f"failed to reserve airdrop manifest {manifest.id!r}") + payload = dict(document) + payload.pop("_id", None) + persisted = AirdropManifest.from_dict(payload) + immutable_fields = ( + persisted.asset_id_int, + persisted.total_amount_micros, + persisted.recipient_count, + persisted.manifest_hash, + ) + requested_fields = ( + manifest.asset_id_int, + manifest.total_amount_micros, + manifest.recipient_count, + manifest.manifest_hash, + ) + if immutable_fields != requested_fields: + raise AirdropConflictError(f"airdrop_id {manifest.id!r} is already reserved for a different immutable manifest") + return persisted + + +def _has_manifest(airdrop_id: str) -> bool: + return ( + db.airdrop_manifests.mongodb_collection.find_one( + {"id": airdrop_id}, + projection={"_id": 1}, + ) + is not None + ) + + +def _mark_manifest_status(airdrop_id: str, status: str) -> str: + collection = db.airdrop_manifests.mongodb_collection + query: dict[str, object] = {"id": airdrop_id} + if status != "complete": + query["status"] = {"$ne": "complete"} + result = collection.update_one( + query, + { + "$set": { + "status": status, + "updated": datetime.now(UTC), + } + }, + ) + if result.matched_count: + return status + + persisted = collection.find_one( + {"id": airdrop_id}, + projection={"status": 1}, + ) + if persisted is not None and persisted.get("status") == "complete": + return "complete" + raise AirdropError(f"airdrop manifest {airdrop_id!r} changed while marking status {status!r}") + + +def _reconcile_existing_reward( + reward: AirdropReward, + *, + operation_id: str, + transfer_service: AssetTransferService, +) -> AirdropReward: + if reward.operation_id not in (None, operation_id): + raise AirdropConflictError(f"reward {reward.airdrop_id!r}/{reward.address} belongs to another operation") + + confirmed_round = reward.confirmed_round + if reward.operation_id is None or confirmed_round is None: + observed = transfer_service.lookup_confirmed_transfer(reward.txid) + if observed is None: + raise AirdropError( + f"legacy reward {reward.airdrop_id!r}/{reward.address} is not confirmed; manual reconciliation required" + ) + expected = ( + cometa_public_key, + reward.address, + reward.asa_id_int, + reward.amount_micros_int, + ) + actual = ( + observed.sender, + observed.receiver, + observed.asset_id, + observed.amount_micros, + ) + if actual != expected: + raise AirdropConflictError( + f"legacy transaction {reward.txid} does not match its stored sender/receiver/asset/amount" + ) + confirmed_round = observed.confirmed_round + + collection = db.airdrop_rewards.mongodb_collection + document = collection.find_one_and_update( + { + "airdrop_id": reward.airdrop_id, + "address": reward.address, + "txid": reward.txid, + }, + { + "$set": { + "operation_id": operation_id, + "confirmed_round": confirmed_round, + "updated": datetime.now(UTC), + } + }, + return_document=ReturnDocument.AFTER, + ) + if document is None: + raise AirdropError(f"failed to backfill legacy reward {reward.airdrop_id!r}/{reward.address}") + payload = dict(document) + payload.pop("_id", None) + return AirdropReward.from_dict(payload) + + +def _persist_reward( + reward: AirdropReward, +) -> AirdropReward: + if reward.operation_id is None: + raise AirdropError("operation_id is required for a persisted reward") + collection = db.airdrop_rewards.mongodb_collection + document = collection.find_one_and_update( + {"operation_id": reward.operation_id}, + {"$setOnInsert": reward.to_dict()}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + if document is None: + raise AirdropError(f"failed to persist reward {reward.operation_id!r}") + payload = dict(document) + payload.pop("_id", None) + persisted = AirdropReward.from_dict(payload) + _assert_reward_matches( + persisted, + asset_id=reward.asa_id_int, + amount_micros=reward.amount_micros_int, + ) + if persisted.txid != reward.txid: + raise AirdropConflictError(f"reward {reward.operation_id!r} points to another transaction") + if persisted.confirmed_round != reward.confirmed_round: + raise AirdropConflictError(f"reward {reward.operation_id!r} has a different confirmation round") + return persisted + + +def _to_tx_info( + *, + receipt: AssetTransferReceipt, + asset_id: int, + amount_micros: int, + address: str, + note: str, +) -> TxInfo: + return TxInfo( + id=receipt.txid, + amount=amount_micros, + asa_id=asset_id, + receiver=address, + note=note, + sender=cometa_public_key, + confirmed_round=receipt.confirmed_round, + ) + + +def _write_manifest( + directory: Path, + *, + asset_id: int, + current_round: int, + amounts: Mapping[str, int], + transactions: Sequence[TxInfo], +) -> None: + directory.mkdir(parents=True, exist_ok=True) + amounts_path = directory / f"airdrop_amounts_{asset_id}_{current_round}.json" + transactions_path = directory / f"airdrop_txns_{asset_id}_{current_round}.json" + amounts_path.write_text( + json.dumps(amounts, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + transactions_path.write_text( + json.dumps([transaction_info.to_dict() for transaction_info in transactions], indent=2) + "\n", + encoding="utf-8", + ) + + async def send_airdrop( - asset_info: AssetInfo, - total_amount_micros: int, - address_shares: dict[str, float], - notes: list[str], - airdrop_id: str + asset_info: AssetInfo, + total_amount_micros: int, + address_shares: Mapping[str, ShareInput], + notes: Sequence[str], + airdrop_id: str, + *, + manifest_directory: Path | None = None, + transfer_service: AssetTransferService | None = None, ) -> list[TxInfo]: + """Execute or resume an airdrop without exceeding its integer budget.""" + current_round = await get_current_round() - logger.info(f'Sending airdrop for {asset_info.name}! Round: {current_round}. Cometa address: {cometa_public_key}') - - # logger.info('Checking opted-in addresses...') - # opted_in_shares = {} - # total_shares = 0 - # it_num = 0 - # for address, share in address_shares.items(): - # it_num += 1 - # if is_opted_in(address, asset_info.id): - # opted_in_shares[address] = share - # total_shares += share - # if it_num % 10 == 0: - # logger.info(f'{len(opted_in_shares)}/{it_num}') - - opted_in_shares = address_shares - total_shares = sum(opted_in_shares.values()) - - amount_per_share = asset_info.micros_to_amount(int(total_amount_micros / total_shares + 1)) - logger.info(f'Total shares: {total_shares}. Total amount: {asset_info.micros_to_amount(total_amount_micros)}. Amount per share: {amount_per_share}') - - opted_in_amounts = {} - for address, share in opted_in_shares.items(): - amount_micros = int(share * total_amount_micros / total_shares + 1) - opted_in_amounts[address] = amount_micros - - with open(f'airdrop_amounts_{asset_info.name.lower()}_{current_round}.json', 'w') as f: - json.dump(opted_in_amounts, f, indent=4) - - sent_txns: list[TxInfo] = [] - total_sent_micros = 0 - - it_num = 0 - for address, amount_micros in opted_in_amounts.items(): - try: - it_num += 1 - logger.info(f'{it_num}/{len(opted_in_amounts)}') + if total_amount_micros > MAX_ALGORAND_UINT: + raise AirdropError("total_amount_micros exceeds Algorand's uint64 range") + amounts = allocate_proportionally(total_amount_micros, address_shares) + _validate_airdrop( + airdrop_id=airdrop_id, + addresses=list(address_shares), + notes=notes, + amounts=amounts, + ) + # Operator scripts invoke this function without FastAPI startup. Install + # both uniqueness invariants before any reward read or transaction signing. + ensure_airdrop_indexes(db) + service = transfer_service or get_asset_transfer_service() + requests = { + address: AssetTransferRequest( + operation_id=_operation_id(airdrop_id, address), + receiver=address, + asset_id=asset_info.id, + amount_micros=amount_micros, + note=_select_note(_operation_id(airdrop_id, address), notes), + ) + for address, amount_micros in amounts.items() + } - already_sent = db.airdrop_rewards.get_one(address=address, airdrop_id=airdrop_id) - if already_sent is not None: - logger.info(f'Skipping {address}: already sent {asset_info.micros_to_amount(already_sent.amount_micros)} {asset_info.name} tokens with txid {already_sent.txid}!') - continue + campaign_rewards = db.airdrop_rewards.get_many(airdrop_id=airdrop_id) + if campaign_rewards and not _has_manifest(airdrop_id): + raise AirdropError( + f"legacy airdrop {airdrop_id!r} has rewards but no immutable manifest; reconcile and migrate it explicitly" + ) - note = random.choice(notes) - txid = send_asset_micros(asset_info, address, amount_micros, note) - db.airdrop_rewards.create(AirdropReward( + # Configuration conflicts must abort the entire batch before any new + # transaction is broadcast. + for address, request in requests.items(): + service.validate(request) + existing_reward = _get_existing_reward( + airdrop_id=airdrop_id, + address=address, + ) + if existing_reward is not None: + _assert_reward_matches( + existing_reward, + asset_id=asset_info.id, + amount_micros=amounts[address], + ) + _reconcile_existing_reward( + existing_reward, + operation_id=request.operation_id, + transfer_service=service, + ) + + _reserve_manifest( + AirdropManifest( + id=airdrop_id, + asset_id=str(asset_info.id), + total_amount_micros=str(total_amount_micros), + recipient_count=len(requests), + manifest_hash=_manifest_hash( + airdrop_id=airdrop_id, + asset_id=asset_info.id, + total_amount_micros=total_amount_micros, + requests=requests, + ), + ) + ) + + logger.info( + "Executing airdrop %s for asset %s at round %s: %s base units across %s recipients", + airdrop_id, + asset_info.id, + current_round, + total_amount_micros, + len(amounts), + ) + + confirmed_transactions: list[TxInfo] = [] + failures: list[AirdropFailure] = [] + + for position, (address, amount_micros) in enumerate(amounts.items(), start=1): + request = requests[address] + try: + existing_reward = _get_existing_reward( airdrop_id=airdrop_id, address=address, - asa_id=asset_info.id, - amount_micros=amount_micros, - txid=txid - )) - sent_txns.append(TxInfo( - id=txid, - amount=amount_micros, - asa_id=asset_info.id, - receiver=address, - note=note, - sender=cometa_public_key - )) - total_sent_micros += amount_micros - except Exception as e: - logger.error(f'Error while sending airdrop to {address}: {e}', exc_info=True) - - logger.info(f'\nSent {asset_info.micros_to_amount(total_sent_micros)} {asset_info.name} tokens to {len(sent_txns)} addresses!') - - with open(f'airdrop_txns_{asset_info.name.lower()}_{current_round}.json', 'w') as f: - json.dump([tx.to_dict() for tx in sent_txns], f, indent=4) - - return sent_txns + ) + if existing_reward is not None: + _assert_reward_matches( + existing_reward, + asset_id=asset_info.id, + amount_micros=amount_micros, + ) + existing_reward = _reconcile_existing_reward( + existing_reward, + operation_id=request.operation_id, + transfer_service=service, + ) + logger.info( + "Airdrop %s recipient %s/%s already confirmed as %s", + airdrop_id, + position, + len(amounts), + existing_reward.txid, + ) + continue + + receipt = service.execute( + request, + ) + _persist_reward( + AirdropReward( + airdrop_id=airdrop_id, + address=address, + asa_id=str(asset_info.id), + amount_micros=str(amount_micros), + txid=receipt.txid, + operation_id=request.operation_id, + confirmed_round=receipt.confirmed_round, + ) + ) + if not receipt.already_confirmed: + confirmed_transactions.append( + _to_tx_info( + receipt=receipt, + asset_id=asset_info.id, + amount_micros=amount_micros, + address=address, + note=request.note or "", + ) + ) + except Exception as exc: + failures.append( + AirdropFailure( + address=address, + error_type=type(exc).__name__, + txid=getattr(exc, "txid", None), + ) + ) + logger.exception( + "Airdrop %s recipient %s/%s is unresolved", + airdrop_id, + position, + len(amounts), + ) + + if manifest_directory is not None: + _write_manifest( + manifest_directory, + asset_id=asset_info.id, + current_round=current_round, + amounts=amounts, + transactions=confirmed_transactions, + ) + + if failures: + effective_status = _mark_manifest_status(airdrop_id, "partial") + if effective_status != "complete": + raise AirdropIncompleteError(failures, confirmed_transactions) + logger.info( + "Airdrop %s was completed by another worker while this worker had %s stale failure(s)", + airdrop_id, + len(failures), + ) + return confirmed_transactions + + _mark_manifest_status(airdrop_id, "complete") + logger.info( + "Airdrop %s confirmed %s new transaction(s); allocated total remains exactly %s base units", + airdrop_id, + len(confirmed_transactions), + sum(amounts.values()), + ) + return confirmed_transactions diff --git a/pyproject.toml b/pyproject.toml index 8cab0df7..6a9448cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,9 @@ addopts = "-ra --strict-config --strict-markers" pythonpath = ["."] testpaths = ["tests"] +markers = [ + "integration: requires an external service such as MongoDB", +] [tool.coverage.run] branch = true @@ -27,6 +30,11 @@ quote-style = "double" files = [ "core/circuit_breaker.py", "flex/blockchain/contract_state.py", + "flex/application/asset_transfers.py", + "flex/db/lp_projection.py", + "flex/db/sync_coordinator.py", + "flex/domain/allocation.py", + "flex/domain/lp_projection.py", "flex/domain/pricing.py", "flex/domain/transactions.py", "flex/providers/pact.py", diff --git a/tests/integration/test_legacy_price_cutover_mongo.py b/tests/integration/test_legacy_price_cutover_mongo.py new file mode 100644 index 00000000..721d8265 --- /dev/null +++ b/tests/integration/test_legacy_price_cutover_mongo.py @@ -0,0 +1,72 @@ +import os +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from bson import Decimal128 +from pymongo import MongoClient +from pymongo.database import Database + +from flex.db.indexes import delete_unverified_legacy_lp_prices + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def mongo_database() -> Database: + uri = os.getenv("MONGODB_TEST_URI") + if not uri: + pytest.skip("MONGODB_TEST_URI is not configured") + + client = MongoClient( + uri, + serverSelectionTimeoutMS=2_000, + tz_aware=True, + ) + client.admin.command("ping") + database = client[f"cometa_legacy_price_cutover_{uuid4().hex}"] + try: + yield database + finally: + client.drop_database(database.name) + client.close() + + +def test_cutover_deletes_both_legacy_signatures_and_preserves_safe_prices( + mongo_database: Database, +) -> None: + collection = mongo_database["asset_prices"] + collection.insert_many( + [ + {"id": Decimal128("1"), "source": "vestige"}, + { + "id": Decimal128("2"), + "source": "tinyman", + "tinyman_algo_pool_id": None, + }, + {"id": Decimal128("3")}, + { + "id": Decimal128("4"), + "source": "vestige", + "tinyman_algo_pool_id": Decimal128("99"), + }, + {"id": Decimal128("5"), "source": "derived_lp"}, + { + "id": Decimal128("6"), + "source": "derived_lp", + "tinyman_algo_pool_id": None, + }, + ] + ) + + removed = delete_unverified_legacy_lp_prices( + SimpleNamespace( + asset_prices=SimpleNamespace( + mongodb_collection=collection, + ) + ) + ) + + remaining_ids = {document["id"].to_decimal() for document in collection.find({}, projection={"id": 1})} + assert removed == 3 + assert remaining_ids == {1, 2, 3} diff --git a/tests/integration/test_mongo_asset_transfer_races.py b/tests/integration/test_mongo_asset_transfer_races.py new file mode 100644 index 00000000..5c9a05c6 --- /dev/null +++ b/tests/integration/test_mongo_asset_transfer_races.py @@ -0,0 +1,245 @@ +import os +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime +from threading import Barrier +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from pymongo import MongoClient +from pymongo.database import Database + +from flex.application.asset_transfers import ( + AssetTransferService, + TransferStatus, +) +from flex.db.asset_transfer_intents import ( + MongoAssetTransferIntentRepository, + TransferIntentPersistenceError, +) +from flex.db.model.transfers import AssetTransferIntent +from flex.tools import airdrop + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def mongo_database() -> Database: + uri = os.getenv("MONGODB_TEST_URI") + if not uri: + pytest.skip("MONGODB_TEST_URI is not configured") + + client = MongoClient( + uri, + serverSelectionTimeoutMS=2_000, + tz_aware=True, + ) + client.admin.command("ping") + database_name = f"cometa_transfer_races_{uuid4().hex}" + database = client[database_name] + try: + yield database + finally: + client.drop_database(database_name) + client.close() + + +def _intent() -> AssetTransferIntent: + timestamp = datetime(2026, 1, 1, tzinfo=UTC) + return AssetTransferIntent( + id="airdrop:summer:address", + receiver="address", + asset_id="42", + amount_micros="1000", + note="hello", + signed_transaction="signed", + txid="txid", + first_valid_round=100, + last_valid_round=1_100, + status=TransferStatus.PREPARED, + created=timestamp, + updated=timestamp, + ) + + +def _repository( + database: Database, +) -> MongoAssetTransferIntentRepository: + repository = MongoAssetTransferIntentRepository( + database["asset_transfer_intents"], + ) + repository.ensure_indexes() + repository.reserve(_intent()) + return repository + + +@pytest.mark.parametrize( + "nonterminal_update", + ["record_attempt", "mark_submitted", "record_error"], +) +def test_confirmation_wins_real_mongo_race_against_nonterminal_update( + mongo_database: Database, + nonterminal_update: str, +) -> None: + repository = _repository(mongo_database) + intent = _intent() + barrier = Barrier(2) + + def confirm() -> AssetTransferIntent: + barrier.wait(timeout=5) + return repository.mark_confirmed(intent.id, intent.txid, 777) + + def update_nonterminal() -> AssetTransferIntent: + barrier.wait(timeout=5) + if nonterminal_update == "record_attempt": + return repository.record_attempt(intent.id, intent.txid) + if nonterminal_update == "mark_submitted": + return repository.mark_submitted(intent.id, intent.txid) + return repository.record_error(intent.id, intent.txid, "late failure") + + with ThreadPoolExecutor(max_workers=2) as executor: + confirmation = executor.submit(confirm) + nonterminal = executor.submit(update_nonterminal) + confirmation.result(timeout=5) + nonterminal.result(timeout=5) + + confirmed = repository.get(intent.id) + assert confirmed is not None + assert confirmed.status == TransferStatus.CONFIRMED + assert confirmed.confirmed_round == 777 + assert confirmed.confirmed_at is not None + assert confirmed.last_error is None + + original_confirmed_at = confirmed.confirmed_at + repository.record_error(intent.id, intent.txid, "must not overwrite terminal evidence") + persisted = repository.get(intent.id) + assert persisted is not None + assert persisted.status == TransferStatus.CONFIRMED + assert persisted.confirmed_round == 777 + assert persisted.confirmed_at == original_confirmed_at + assert persisted.last_error is None + + +def test_conflicting_real_mongo_confirmations_preserve_one_terminal_result( + mongo_database: Database, +) -> None: + repository = _repository(mongo_database) + intent = _intent() + barrier = Barrier(2) + + def confirm(round_number: int) -> tuple[str, int]: + barrier.wait(timeout=5) + try: + result = repository.mark_confirmed( + intent.id, + intent.txid, + round_number, + ) + except TransferIntentPersistenceError: + return "conflict", round_number + return "confirmed", result.confirmed_round or 0 + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(confirm, round_number) for round_number in (777, 778)] + results = [future.result(timeout=5) for future in futures] + + assert sorted(status for status, _ in results) == ["confirmed", "conflict"] + persisted = repository.get(intent.id) + assert persisted is not None + assert persisted.status == TransferStatus.CONFIRMED + assert persisted.confirmed_round in {777, 778} + assert persisted.confirmed_at is not None + + confirmed_at = persisted.confirmed_at + repeated = repository.mark_confirmed( + intent.id, + intent.txid, + persisted.confirmed_round, + ) + assert repeated.confirmed_at == confirmed_at + + conflicting_round = 778 if persisted.confirmed_round == 777 else 777 + with pytest.raises(TransferIntentPersistenceError, match="already confirmed"): + repository.mark_confirmed( + intent.id, + intent.txid, + conflicting_round, + ) + unchanged = repository.get(intent.id) + assert unchanged is not None + assert unchanged.confirmed_round == persisted.confirmed_round + assert unchanged.confirmed_at == confirmed_at + + +class _ConfirmDuringLookupGateway: + def __init__( + self, + repository: MongoAssetTransferIntentRepository, + intent: AssetTransferIntent, + ) -> None: + self.repository = repository + self.intent = intent + + def lookup_confirmed_round(self, txid: str) -> int | None: + assert txid == self.intent.txid + with ThreadPoolExecutor(max_workers=1) as executor: + executor.submit( + self.repository.mark_confirmed, + self.intent.id, + txid, + 779, + ).result(timeout=5) + return None + + @staticmethod + def current_round() -> int: + return 200 + + +def test_reconcile_reloads_real_mongo_terminal_state_before_nonterminal_result( + mongo_database: Database, +) -> None: + repository = _repository(mongo_database) + intent = _intent() + service = AssetTransferService( + repository=repository, + gateway=_ConfirmDuringLookupGateway(repository, intent), # type: ignore[arg-type] + ) + + result = service.reconcile(intent.id) + + assert result.status == TransferStatus.CONFIRMED + assert result.txid == intent.txid + assert result.confirmed_round == 779 + + +def test_complete_airdrop_manifest_wins_real_mongo_partial_status_race( + mongo_database: Database, + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = mongo_database["airdrop_manifests"] + collection.insert_one({"id": "summer-2026", "status": "prepared"}) + monkeypatch.setattr( + airdrop, + "db", + SimpleNamespace( + airdrop_manifests=SimpleNamespace( + mongodb_collection=collection, + ), + ), + ) + barrier = Barrier(2) + + def mark(status: str) -> str: + barrier.wait(timeout=5) + return airdrop._mark_manifest_status("summer-2026", status) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(mark, status) for status in ("complete", "partial")] + results = [future.result(timeout=5) for future in futures] + + persisted = collection.find_one({"id": "summer-2026"}) + assert persisted is not None + assert persisted["status"] == "complete" + assert results[0] == "complete" + assert results[1] in {"partial", "complete"} diff --git a/tests/integration/test_mongo_financial_projection.py b/tests/integration/test_mongo_financial_projection.py new file mode 100644 index 00000000..8b714ce6 --- /dev/null +++ b/tests/integration/test_mongo_financial_projection.py @@ -0,0 +1,578 @@ +import asyncio +import os +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime, timedelta +from threading import Barrier, Event +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from bson import Decimal128 +from pymongo import MongoClient +from pymongo.database import Database + +from flex import sync_pools +from flex.data import asset_prices as asset_price_data +from flex.data import assets as asset_data +from flex.db.indexes import ( + create_unique_field_index_fail_closed, + create_unique_id_index_fail_closed, + delete_unverified_legacy_lp_prices, +) +from flex.db.lp_projection import ( + LpProjectionResult, + MongoLpProjectionRepository, +) +from flex.db.model.blockchain import ( + TOTAL_SUPPLY_SOURCE_INDEXER, + UINT64_MAX, + Asset, + SyncState, +) +from flex.db.model.liquidity_pools import LpState, LpTransaction +from flex.db.model.priced import AssetPrice +from flex.db.sync_coordinator import ( + MongoSyncCoordinator, + SyncCoordinatorError, +) +from flex.domain.lp_projection import lp_round_end_order +from flex.domain.pricing import MAX_OBSERVATION_CLOCK_SKEW +from flex.domain.transactions import ASSET_TRANSFER_TX + +pytestmark = pytest.mark.integration + + +class _CoordinatedRepository(MongoLpProjectionRepository): + """Hold stale readers until one worker persists both state and marker.""" + + def __init__( + self, + *, + states, + events, + first_read_barrier: Barrier, + winner_committed: Event, + wait_for_winner: bool, + ) -> None: + super().__init__(states=states, events=events) + self._first_read_barrier = first_read_barrier + self._winner_committed = winner_committed + self._wait_for_winner = wait_for_winner + self._first_read = True + + def _get_state(self, pool_address: str) -> LpState: + state = super()._get_state(pool_address) + if self._first_read: + self._first_read = False + self._first_read_barrier.wait(timeout=5) + if self._wait_for_winner and not self._winner_committed.wait(timeout=5): + raise TimeoutError("winning replay did not persist its marker") + return state + + def _record_event(self, transaction: LpTransaction) -> None: + super()._record_event(transaction) + if not self._wait_for_winner: + self._winner_committed.set() + + +@pytest.fixture +def mongo_database() -> Database: + uri = os.getenv("MONGODB_TEST_URI") + if not uri: + pytest.skip("MONGODB_TEST_URI is not configured") + + client = MongoClient( + uri, + serverSelectionTimeoutMS=2_000, + tz_aware=True, + ) + client.admin.command("ping") + database_name = f"cometa_integration_{uuid4().hex}" + database = client[database_name] + try: + yield database + finally: + client.drop_database(database_name) + client.close() + + +def _state(*, reserve: int = 100) -> LpState: + return LpState( + id=1, + token_id=99, + asset1_id=7, + asset2_id=0, + dex_provider="tinyman", + address="POOL", + last_updated_round=99, + last_event_order=lp_round_end_order(99), + asset1_reserve_micros=reserve, + asset2_reserve_micros=100, + total_tokens_micros=100, + asset1_reserve=0, + asset2_reserve=0, + total_tokens=0, + token_price_algo=0, + ) + + +def _transaction( + *, + amount: int = 10, + asa_id: int = 7, + event_id: str = "TX@POOL", +) -> LpTransaction: + return LpTransaction( + id=event_id, + pool_address="POOL", + user_address="USER", + asa_id=asa_id, + delta_amount_micros=amount, + confirmed_round=100, + event_position=1, + ) + + +def _repository( + database: Database, + state: LpState, +) -> MongoLpProjectionRepository: + states = database["lp_states"] + events = database["lp_transactions"] + states.create_index("token_id", unique=True) + states.create_index("address", unique=True) + events.create_index("id", unique=True) + states.insert_one(state.to_dict()) + return MongoLpProjectionRepository( + states=states, + events=events, + ) + + +def test_concurrent_lp_replay_applies_real_mongo_delta_once( + mongo_database: Database, +) -> None: + repository = _repository(mongo_database, _state()) + transaction = _transaction() + first_read_barrier = Barrier(8) + winner_committed = Event() + workers = [ + _CoordinatedRepository( + states=repository.states, + events=repository.events, + first_read_barrier=first_read_barrier, + winner_committed=winner_committed, + wait_for_winner=index != 0, + ) + for index in range(8) + ] + + with ThreadPoolExecutor(max_workers=8) as executor: + futures = [executor.submit(worker.project, transaction) for worker in workers] + outcomes = [future.result(timeout=10) for future in futures] + + persisted = repository.get_state("POOL") + assert persisted.asset1_reserve_micros == 110 + assert mongo_database["lp_transactions"].count_documents({}) == 1 + assert sum(outcome.result is LpProjectionResult.APPLIED for outcome in outcomes) == 1 + + +def test_real_mongo_repairs_marker_gap_without_repeating_delta( + mongo_database: Database, +) -> None: + state = _state() + repository = _repository(mongo_database, state) + transaction = _transaction() + mongo_database["lp_states"].update_one( + {"address": state.address}, + { + "$inc": { + "asset1_reserve_micros": Decimal128("10"), + }, + "$set": { + "last_event_order": transaction.event_order, + "last_updated_round": Decimal128("100"), + }, + }, + ) + + outcome = repository.project(transaction) + + assert outcome.result is LpProjectionResult.ALREADY_APPLIED + assert outcome.state.asset1_reserve_micros == 110 + assert mongo_database["lp_transactions"].count_documents({}) == 1 + + +def test_real_mongo_decimal128_increment_reaches_uint64_max( + mongo_database: Database, +) -> None: + maximum = 2**64 - 1 + repository = _repository( + mongo_database, + _state(reserve=1), + ) + + outcome = repository.project( + _transaction(amount=maximum - 1), + ) + + raw = mongo_database["lp_states"].find_one({"address": "POOL"}) + assert outcome.state.asset1_reserve_micros == maximum + assert raw is not None + assert raw["asset1_reserve_micros"] == Decimal128(str(maximum)) + + +def test_real_mongo_projects_token_pool_fee_to_operational_algo( + mongo_database: Database, +) -> None: + state = _state() + state.asset2_id = 8 + state.operational_algo_balance_micros = 1_000 + repository = _repository(mongo_database, state) + + outcome = repository.project( + _transaction( + amount=-1_000, + asa_id=0, + event_id="TX#fee@POOL", + ), + ) + + raw = mongo_database["lp_states"].find_one({"address": "POOL"}) + assert outcome.state.asset1_reserve_micros == 100 + assert outcome.state.asset2_reserve_micros == 100 + assert outcome.state.operational_algo_balance_micros == 0 + assert raw is not None + assert raw["operational_algo_balance_micros"] == Decimal128("0") + + +def test_real_mongo_replays_amount_and_fee_from_one_raw_transaction( + mongo_database: Database, + monkeypatch: pytest.MonkeyPatch, +) -> None: + state = _state() + state.asset2_id = 8 + state.operational_algo_balance_micros = 2_000 + repository = _repository(mongo_database, state) + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 8, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 100, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "USER", + }, + } + + transactions = asyncio.run(sync_pools.process_lp_transactions([raw_transaction])) + ordered = sorted(transactions, key=lambda transaction: transaction.event_order) + first_outcomes = [repository.project(transaction) for transaction in ordered] + replay_outcomes = [repository.project(transaction) for transaction in ordered] + + persisted = repository.get_state("POOL") + assert [transaction.id for transaction in transactions] == [ + "TX@POOL", + "TX#fee@POOL", + ] + assert persisted.asset1_reserve_micros == 95 + assert persisted.asset2_reserve_micros == 100 + assert persisted.operational_algo_balance_micros == 1_000 + assert persisted.last_event_order == max(transaction.event_order for transaction in transactions) + assert mongo_database["lp_transactions"].count_documents({}) == 2 + assert all(outcome.result is LpProjectionResult.APPLIED for outcome in first_outcomes) + assert all(outcome.result is LpProjectionResult.ALREADY_APPLIED for outcome in replay_outcomes) + + +def test_real_mongo_promotes_legacy_int64_balance_to_decimal128( + mongo_database: Database, +) -> None: + state = _state() + payload = state.to_dict() + for field_name in ( + "id", + "token_id", + "asset1_id", + "asset2_id", + "last_updated_round", + "asset1_reserve_micros", + "asset2_reserve_micros", + "total_tokens_micros", + "operational_algo_balance_micros", + ): + payload[field_name] = int(payload[field_name].to_decimal()) + + states = mongo_database["lp_states"] + events = mongo_database["lp_transactions"] + states.create_index("token_id", unique=True) + states.create_index("address", unique=True) + events.create_index("id", unique=True) + states.insert_one(payload) + repository = MongoLpProjectionRepository(states=states, events=events) + + outcome = repository.project(_transaction()) + + raw = states.find_one({"address": "POOL"}) + assert outcome.state.asset1_reserve_micros == 110 + assert raw is not None + assert raw["asset1_reserve_micros"] == Decimal128("110") + + +def test_duplicate_financial_business_key_fails_without_deletion( + mongo_database: Database, +) -> None: + collection = mongo_database["lp_states"] + collection.insert_many( + [ + {"id": "a", "token_id": Decimal128("99")}, + {"id": "b", "token_id": Decimal128("99")}, + ] + ) + + with pytest.raises(RuntimeError, match="duplicate"): + create_unique_field_index_fail_closed( + collection, + collection_name="lp_states", + field_name="token_id", + index_name="token_id_unique", + ) + + assert collection.count_documents({}) == 2 + + +def test_duplicate_asset_identity_fails_without_deletion( + mongo_database: Database, +) -> None: + collection = mongo_database["assets"] + collection.insert_many( + [ + {"id": Decimal128("42"), "name": "first"}, + {"id": Decimal128("42"), "name": "second"}, + ] + ) + + with pytest.raises(RuntimeError, match="duplicate immutable ID"): + create_unique_id_index_fail_closed( + collection, + collection_name="assets", + ) + + assert collection.count_documents({}) == 2 + + +def test_real_mongo_deletes_only_retired_lp_price_projections( + mongo_database: Database, +) -> None: + collection = mongo_database["asset_prices"] + collection.insert_many( + [ + {"id": Decimal128("1"), "source": "vestige"}, + {"id": Decimal128("2"), "source": "vestige", "tinyman_algo_pool_id": None}, + { + "id": Decimal128("3"), + "source": "tinyman", + "tinyman_algo_pool_id": Decimal128("99"), + }, + ] + ) + + removed = delete_unverified_legacy_lp_prices( + SimpleNamespace( + asset_prices=SimpleNamespace( + mongodb_collection=collection, + ) + ) + ) + + assert removed == 1 + assert {document["id"].to_decimal() for document in collection.find({}, projection={"id": 1})} == {1, 2} + + +def test_valid_price_replaces_far_future_observation( + mongo_database: Database, + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = mongo_database["asset_prices"] + collection.create_index("id", unique=True) + now = datetime.now(UTC) + collection.insert_one( + { + "id": Decimal128("42"), + "name": "poisoned", + "price_algo": 999.0, + "price_usd": 999.0, + "last_update_round": Decimal128("1"), + "source": "vestige", + "observed_at": now + MAX_OBSERVATION_CLOCK_SKEW + timedelta(days=1), + "created": now, + "updated": now, + } + ) + monkeypatch.setattr( + asset_price_data, + "db", + SimpleNamespace( + asset_prices=SimpleNamespace( + mongodb_collection=collection, + ) + ), + ) + candidate = AssetPrice( + id=42, + name="verified", + price_algo=0.25, + price_usd=1.0, + last_update_round=2, + source="vestige", + observed_at=now, + ) + + assert asset_price_data._upsert_asset_price(candidate) is True + + stored = collection.find_one({"id": Decimal128("42")}) + assert stored is not None + assert stored["name"] == "verified" + assert abs(stored["observed_at"] - now) < timedelta(milliseconds=1) + + +@pytest.mark.asyncio +async def test_concurrent_real_mongo_supply_migration_persists_provenance_once( + mongo_database: Database, + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = mongo_database["assets"] + inserted = collection.insert_one( + { + "id": Decimal128("42"), + "total_supply": float(UINT64_MAX), + "total_supply_micros": str(2**53), + } + ) + legacy_document = collection.find_one({"_id": inserted.inserted_id}) + assert legacy_document is not None + monkeypatch.setattr( + asset_data, + "db", + SimpleNamespace( + assets=SimpleNamespace( + mongodb_collection=collection, + ) + ), + ) + authoritative = Asset( + id=42, + name="Canonical Asset", + decimals=0, + unit_name="CANON", + creator="CREATOR", + reserve="RESERVE", + total_supply=0, + total_supply_micros=UINT64_MAX, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, + ) + + async def fetch_asset(asset_id: int) -> Asset: + assert asset_id == 42 + await asyncio.sleep(0) + return authoritative + + monkeypatch.setattr(asset_data, "fetch_asset", fetch_asset) + + results = await asyncio.gather(*(asset_data._backfill_canonical_supply(42, legacy_document) for _ in range(8))) + + persisted = collection.find_one({"_id": inserted.inserted_id}) + assert results == [UINT64_MAX] * 8 + assert persisted is not None + assert persisted["total_supply_micros"] == str(UINT64_MAX) + assert persisted["total_supply_source"] == TOTAL_SUPPLY_SOURCE_INDEXER + + +def test_expired_real_mongo_sync_lease_fences_stale_owner( + mongo_database: Database, +) -> None: + collection = mongo_database["sync_states"] + collection.create_index("id", unique=True) + collection.insert_one(SyncState(last_round=99).to_dict()) + coordinator = MongoSyncCoordinator( + collection=collection, + lease_duration=timedelta(seconds=30), + ) + now = datetime(2026, 1, 1, tzinfo=UTC) + + assert ( + coordinator.claim_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now, + ) + is not None + ) + assert ( + coordinator.claim_round( + owner="worker-b", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=31), + ) + is not None + ) + + with pytest.raises(SyncCoordinatorError, match="lost its claim"): + coordinator.complete_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=32), + ) + + completed = coordinator.complete_round( + owner="worker-b", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=32), + ) + assert completed.last_round == 100 + + +def test_expired_real_mongo_sync_lease_cannot_commit_without_takeover( + mongo_database: Database, +) -> None: + collection = mongo_database["sync_states"] + collection.create_index("id", unique=True) + collection.insert_one(SyncState(last_round=99).to_dict()) + coordinator = MongoSyncCoordinator( + collection=collection, + lease_duration=timedelta(seconds=30), + ) + now = datetime(2026, 1, 1, tzinfo=UTC) + + assert ( + coordinator.claim_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now, + ) + is not None + ) + + with pytest.raises(SyncCoordinatorError, match="lost its claim"): + coordinator.complete_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=31), + ) + + persisted_document = collection.find_one({"id": "main"}) + assert persisted_document is not None + persisted_document.pop("_id") + persisted = SyncState.from_dict(persisted_document) + assert persisted.last_round == 99 diff --git a/tests/integration/test_mongo_lottery_index_upgrade.py b/tests/integration/test_mongo_lottery_index_upgrade.py new file mode 100644 index 00000000..fe4cbf37 --- /dev/null +++ b/tests/integration/test_mongo_lottery_index_upgrade.py @@ -0,0 +1,311 @@ +import os +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from pymongo import MongoClient +from pymongo.database import Database +from pymongo.errors import DuplicateKeyError + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def mongo_database() -> Database: + uri = os.getenv("MONGODB_TEST_URI") + if not uri: + pytest.skip("MONGODB_TEST_URI is not configured") + + client = MongoClient( + uri, + serverSelectionTimeoutMS=2_000, + tz_aware=True, + ) + client.admin.command("ping") + database_name = f"cometa_lottery_index_{uuid4().hex}" + database = client[database_name] + try: + yield database + finally: + client.drop_database(database_name) + client.close() + + +def test_old_lottery_id_index_is_replaced_without_a_uniqueness_gap( + mongo_database: Database, + monkeypatch, +) -> None: + from api import nft_lottery + + collection = mongo_database["lottery_draws"] + collection.insert_many( + [ + {"marker": "missing-id"}, + {"marker": "empty-id", "id": ""}, + {"marker": "current", "id": "draw-current"}, + ] + ) + collection.create_index( + "id", + unique=True, + name="id_unique", + partialFilterExpression={"id": {"$type": "string"}}, + ) + original_ids = set(collection.distinct("_id")) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + + nft_lottery.ensure_lottery_indexes() + + assert set(collection.distinct("_id")) == original_ids + indexes = collection.index_information() + assert "id_unique" not in indexes + replacement = indexes[nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME] + assert replacement["unique"] is True + assert replacement["partialFilterExpression"] == nft_lottery.LOTTERY_DRAW_ID_FILTER + + # The upgraded filter intentionally leaves legacy placeholders outside the + # index while continuing to reject every duplicate durable draw identity. + collection.insert_many([{"id": ""}, {"id": ""}, {"marker": "another-missing-id"}]) + with pytest.raises(DuplicateKeyError): + collection.insert_one({"id": "draw-current"}) + + +def test_lottery_id_index_upgrade_preserves_conflicting_records( + mongo_database: Database, + monkeypatch, +) -> None: + from api import nft_lottery + + collection = mongo_database["lottery_draws"] + collection.insert_many( + [ + {"marker": "first", "id": "duplicate"}, + {"marker": "second", "id": "duplicate"}, + ] + ) + original_ids = set(collection.distinct("_id")) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + + with pytest.raises(RuntimeError, match="preserved 1 duplicate group"): + nft_lottery.ensure_lottery_indexes() + + assert set(collection.distinct("_id")) == original_ids + assert nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME not in collection.index_information() + + +def test_staking_entitlement_allows_only_one_concurrent_draw( + mongo_database: Database, + monkeypatch, +) -> None: + from api import nft_lottery + + draws = mongo_database["lottery_draws"] + entitlements = mongo_database["lottery_entitlements"] + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=draws), + ) + monkeypatch.setattr( + nft_lottery, + "lottery_entitlements", + entitlements, + ) + monkeypatch.setattr( + nft_lottery, + "draw_id", + lambda lottery: 777, + ) + lottery = nft_lottery.NftLottery( + name="staking-summer", + asset_id=7, + min_amount=1, + probability=1.0, + available_nfts=[777], + type=nft_lottery.LotteryType.STAKING, + ) + now = datetime(2026, 7, 19, 12, tzinfo=UTC) + + def create_draw(_: int): + return nft_lottery._get_or_create_staking_draw( + lottery, + "WALLET", + now=now, + ) + + with ThreadPoolExecutor(max_workers=16) as executor: + results = list(executor.map(create_draw, range(32))) + + materialized = [draw for draw in results if draw is not None] + assert materialized + assert len({draw.id for draw in materialized}) == 1 + assert draws.count_documents({}) == 1 + entitlement = entitlements.find_one({}) + assert entitlement is not None + assert entitlement["generation"] == 1 + assert entitlement["active"]["draw_id"] == materialized[0].id + assert entitlement["active"]["status"] == nft_lottery.StakingEntitlementStatus.MATERIALIZED + + +def test_staking_entitlement_recovers_a_crash_before_draw_insert( + mongo_database: Database, + monkeypatch, +) -> None: + from api import nft_lottery + + draws = mongo_database["lottery_draws"] + entitlements = mongo_database["lottery_entitlements"] + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=draws), + ) + monkeypatch.setattr( + nft_lottery, + "lottery_entitlements", + entitlements, + ) + monkeypatch.setattr( + nft_lottery, + "draw_id", + lambda lottery: 888, + ) + lottery = nft_lottery.NftLottery( + name="staking-recovery", + asset_id=7, + min_amount=1, + probability=1.0, + available_nfts=[888], + type=nft_lottery.LotteryType.STAKING, + ) + now = datetime(2026, 7, 19, 12, tzinfo=UTC) + claimed = nft_lottery._claim_staking_entitlement( + lottery_name=lottery.name, + wallet="WALLET", + now=now, + ) + + assert claimed is not None + assert draws.count_documents({}) == 0 + + recovered = nft_lottery._get_or_create_staking_draw( + lottery, + "WALLET", + now=now, + ) + + assert recovered is not None + assert recovered.id == claimed["active"]["draw_id"] + assert recovered.prize == 888 + assert draws.count_documents({}) == 1 + + +def test_staking_entitlement_repairs_confirmed_draw_before_next_generation( + mongo_database: Database, + monkeypatch, +) -> None: + from api import nft_lottery + + draws = mongo_database["lottery_draws"] + entitlements = mongo_database["lottery_entitlements"] + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=draws), + ) + monkeypatch.setattr( + nft_lottery, + "lottery_entitlements", + entitlements, + ) + monkeypatch.setattr( + nft_lottery, + "draw_id", + lambda lottery: 999, + ) + lottery = nft_lottery.NftLottery( + name="staking-confirmed-repair", + asset_id=7, + min_amount=1, + probability=1.0, + available_nfts=[999], + type=nft_lottery.LotteryType.STAKING, + ) + first_time = datetime(2026, 7, 19, 12, tzinfo=UTC) + first = nft_lottery._get_or_create_staking_draw( + lottery, + "WALLET", + now=first_time, + ) + assert first is not None + draws.update_one( + {"id": first.id}, + { + "$set": { + "claimed": True, + "payout_status": nft_lottery.LotteryPayoutStatus.CONFIRMED, + } + }, + ) + + second = nft_lottery._get_or_create_staking_draw( + lottery, + "WALLET", + now=first_time + timedelta(seconds=nft_lottery.MIN_DRAW_INTERVAL), + ) + + assert second is not None + assert second.id != first.id + entitlement = entitlements.find_one({}) + assert entitlement is not None + assert entitlement["generation"] == 2 + + +def test_confirmed_staking_entitlement_is_terminal_against_stale_error( + mongo_database: Database, + monkeypatch, +) -> None: + from api import nft_lottery + + entitlements = mongo_database["lottery_entitlements"] + monkeypatch.setattr( + nft_lottery, + "lottery_entitlements", + entitlements, + ) + entitlements.insert_one( + { + "_id": "entitlement", + "generation": 1, + "active": { + "draw_id": "draw", + "status": nft_lottery.StakingEntitlementStatus.CONFIRMED, + }, + } + ) + draw = nft_lottery.LotteryDraw( + id="draw", + wallet="WALLET", + prize=999, + entitlement_id="entitlement", + entitlement_generation=1, + ) + + nft_lottery._update_staking_entitlement_status( + draw, + nft_lottery.StakingEntitlementStatus.UNRESOLVED, + ) + + persisted = entitlements.find_one({"_id": "entitlement"}) + assert persisted is not None + assert persisted["active"]["status"] == nft_lottery.StakingEntitlementStatus.CONFIRMED diff --git a/tests/unit/test_airdrop.py b/tests/unit/test_airdrop.py new file mode 100644 index 00000000..b98a9e1e --- /dev/null +++ b/tests/unit/test_airdrop.py @@ -0,0 +1,514 @@ +import asyncio +from types import SimpleNamespace + +import pytest +from algosdk import account + +from flex.application.asset_transfers import ( + AssetTransferPendingError, + AssetTransferReceipt, + AssetTransferRequest, + ConfirmedAssetTransfer, +) +from flex.db.model.airdrop import AirdropManifest +from flex.db.model.blockchain import AssetInfo +from flex.db.model.priced import AirdropReward +from flex.tools import airdrop + + +class FakeRewardCollection: + def __init__(self, rewards: list[AirdropReward] | None = None) -> None: + self.documents = [reward.to_dict() for reward in rewards or []] + self.indexes: list[tuple[tuple, dict]] = [] + + def aggregate(self, *args, **kwargs): + return [] + + def create_index(self, *args, **kwargs) -> None: + self.indexes.append((args, kwargs)) + + def find_one_and_update(self, query, update, **kwargs): + document = next( + ( + document + for document in self.documents + if all(document.get(field) == value for field, value in query.items()) + ), + None, + ) + if document is None and "$setOnInsert" in update: + document = dict(update["$setOnInsert"]) + self.documents.append(document) + if document is not None and "$set" in update: + document.update(update["$set"]) + return dict(document) if document is not None else None + + +class FakeRewardManager: + def __init__(self, rewards: list[AirdropReward] | None = None) -> None: + self.mongodb_collection = FakeRewardCollection(rewards) + + def get_many(self, **query): + matches = [] + for document in self.mongodb_collection.documents: + reward = AirdropReward.from_dict(document) + if all(getattr(reward, field) == value for field, value in query.items()): + matches.append(reward) + return matches + + +class FakeManifestCollection: + def __init__(self) -> None: + self.documents: dict[str, dict] = {} + self.complete_before_partial = False + + def create_index(self, *args, **kwargs) -> None: + return None + + def aggregate(self, *args, **kwargs): + return [] + + def find_one(self, query, projection=None): + document = self.documents.get(query["id"]) + return dict(document) if document is not None else None + + def find_one_and_update(self, query, update, **kwargs): + manifest_id = query["id"] + self.documents.setdefault(manifest_id, dict(update["$setOnInsert"])) + return dict(self.documents[manifest_id]) + + def update_one(self, query, update): + document = self.documents.get(query["id"]) + expected_status = query.get("status") + if ( + document is not None + and self.complete_before_partial + and isinstance(expected_status, dict) + and expected_status.get("$ne") == "complete" + ): + document["status"] = "complete" + if document is None or ( + isinstance(expected_status, dict) + and "$ne" in expected_status + and document.get("status") == expected_status["$ne"] + ): + return SimpleNamespace(matched_count=0) + document.update(update["$set"]) + return SimpleNamespace(matched_count=1) + + +class FakeTransferService: + def __init__( + self, + *, + fail_address: str | None = None, + legacy_transfer: ConfirmedAssetTransfer | None = None, + ) -> None: + self.validated = [] + self.executed = [] + self.fail_address = fail_address + self.legacy_transfer = legacy_transfer + + def validate(self, request) -> None: + self.validated.append(request) + + def execute(self, request) -> AssetTransferReceipt: + self.executed.append(request) + if request.receiver == self.fail_address: + raise AssetTransferPendingError(request.operation_id, f"tx-{request.receiver}") + return AssetTransferReceipt( + operation_id=request.operation_id, + txid=f"tx-{request.receiver}", + confirmed_round=123, + already_confirmed=False, + ) + + def lookup_confirmed_transfer(self, txid: str) -> ConfirmedAssetTransfer | None: + return self.legacy_transfer + + +def _addresses(count: int) -> list[str]: + return [account.generate_account()[1] for _ in range(count)] + + +def _asset() -> AssetInfo: + return AssetInfo( + name="Test Asset", + decimals=6, + unit_name="TEST", + id=42, + ) + + +def _install_fake_db( + monkeypatch, + rewards: list[AirdropReward] | None = None, +) -> tuple[FakeRewardManager, FakeManifestCollection]: + reward_manager = FakeRewardManager(rewards) + manifest_collection = FakeManifestCollection() + monkeypatch.setattr( + airdrop, + "db", + SimpleNamespace( + airdrop_rewards=reward_manager, + airdrop_manifests=SimpleNamespace( + mongodb_collection=manifest_collection, + ), + ), + ) + + async def current_round() -> int: + return 100 + + monkeypatch.setattr(airdrop, "get_current_round", current_round) + return reward_manager, manifest_collection + + +def _seed_manifest( + manifests: FakeManifestCollection, + *, + airdrop_id: str, + amounts: dict[str, int], + note: str = "hello", +) -> None: + requests = { + address: AssetTransferRequest( + operation_id=f"airdrop:{airdrop_id}:{address}", + receiver=address, + asset_id=42, + amount_micros=amount_micros, + note=note, + ) + for address, amount_micros in amounts.items() + } + total_amount_micros = sum(amounts.values()) + manifests.documents[airdrop_id] = AirdropManifest( + id=airdrop_id, + asset_id=42, + total_amount_micros=str(total_amount_micros), + recipient_count=len(requests), + manifest_hash=airdrop._manifest_hash( + airdrop_id=airdrop_id, + asset_id=42, + total_amount_micros=total_amount_micros, + requests=requests, + ), + ).to_dict() + + +def test_airdrop_allocates_the_exact_budget_and_persists_every_receipt(monkeypatch) -> None: + rewards, manifests = _install_fake_db(monkeypatch) + addresses = _addresses(3) + service = FakeTransferService() + + transactions = asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares=dict.fromkeys(reversed(addresses), 1), + notes=["one", "two"], + airdrop_id="summer-2026", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert len(service.validated) == 3 + assert len(service.executed) == 3 + assert sum(request.amount_micros for request in service.executed) == 10 + assert sorted(request.amount_micros for request in service.executed) == [3, 3, 4] + assert len(transactions) == 3 + assert len(rewards.mongodb_collection.documents) == 3 + assert manifests.documents["summer-2026"]["status"] == "complete" + + +@pytest.mark.parametrize("stale_status", ["prepared", "partial"]) +def test_complete_airdrop_manifest_cannot_regress(monkeypatch, stale_status: str) -> None: + _, manifests = _install_fake_db(monkeypatch) + manifests.documents["summer-2026"] = {"id": "summer-2026", "status": "complete"} + + airdrop._mark_manifest_status("summer-2026", stale_status) + + assert manifests.documents["summer-2026"]["status"] == "complete" + + +def test_airdrop_configuration_conflict_aborts_before_any_broadcast(monkeypatch) -> None: + address, other_address = _addresses(2) + existing = AirdropReward( + airdrop_id="summer-2026", + address=other_address, + asa_id=42, + amount_micros=999, + txid="legacy-txid", + ) + _, manifests = _install_fake_db(monkeypatch, [existing]) + _seed_manifest( + manifests, + airdrop_id="summer-2026", + amounts={address: 5, other_address: 5}, + ) + service = FakeTransferService() + + with pytest.raises(airdrop.AirdropConflictError): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares={address: 1, other_address: 1}, + notes=["hello"], + airdrop_id="summer-2026", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert service.executed == [] + assert manifests.documents["summer-2026"]["status"] == "prepared" + + +def test_airdrop_reports_partial_failure_and_continues_safe_recipients(monkeypatch) -> None: + rewards, manifests = _install_fake_db(monkeypatch) + addresses = _addresses(3) + service = FakeTransferService(fail_address=addresses[1]) + + with pytest.raises(airdrop.AirdropIncompleteError) as error: + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=9, + address_shares=dict.fromkeys(addresses, 1), + notes=["hello"], + airdrop_id="summer-2026", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert len(service.executed) == 3 + assert error.value.failures == ( + airdrop.AirdropFailure( + address=addresses[1], + error_type="AssetTransferPendingError", + txid=f"tx-{addresses[1]}", + ), + ) + assert len(error.value.confirmed_transactions) == 2 + assert len(rewards.mongodb_collection.documents) == 2 + assert manifests.documents["summer-2026"]["status"] == "partial" + + +def test_stale_airdrop_failure_observes_concurrent_terminal_completion(monkeypatch) -> None: + rewards, manifests = _install_fake_db(monkeypatch) + addresses = _addresses(2) + service = FakeTransferService(fail_address=addresses[1]) + manifests.complete_before_partial = True + + transactions = asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares=dict.fromkeys(addresses, 1), + notes=["hello"], + airdrop_id="summer-2026", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert len(transactions) == 1 + assert len(rewards.mongodb_collection.documents) == 1 + assert manifests.documents["summer-2026"]["status"] == "complete" + + +def test_airdrop_rejects_zero_allocations_before_execution(monkeypatch) -> None: + _install_fake_db(monkeypatch) + addresses = _addresses(2) + service = FakeTransferService() + + with pytest.raises(airdrop.AirdropError, match="would receive zero"): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=1, + address_shares={addresses[0]: 1, addresses[1]: 1}, + notes=["hello"], + airdrop_id="summer-2026", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert service.executed == [] + + +def test_airdrop_manifest_rejects_recipient_set_changes_after_partial_run(monkeypatch) -> None: + _, manifests = _install_fake_db(monkeypatch) + first, second, added = _addresses(3) + first_service = FakeTransferService(fail_address=second) + + with pytest.raises(airdrop.AirdropIncompleteError): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares={first: 1, second: 1}, + notes=["hello"], + airdrop_id="immutable-batch", + transfer_service=first_service, # type: ignore[arg-type] + ) + ) + + assert manifests.documents["immutable-batch"]["status"] == "partial" + changed_service = FakeTransferService() + with pytest.raises(airdrop.AirdropConflictError, match="immutable manifest"): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=15, + address_shares={first: 1, second: 1, added: 1}, + notes=["hello"], + airdrop_id="immutable-batch", + transfer_service=changed_service, # type: ignore[arg-type] + ) + ) + + assert changed_service.executed == [] + + +def test_legacy_unconfirmed_reward_is_not_treated_as_paid(monkeypatch) -> None: + address = _addresses(1)[0] + legacy_reward = AirdropReward( + airdrop_id="legacy", + address=address, + asa_id=42, + amount_micros=10, + txid="legacy-txid", + ) + _, manifests = _install_fake_db(monkeypatch, [legacy_reward]) + _seed_manifest( + manifests, + airdrop_id="legacy", + amounts={address: 10}, + ) + service = FakeTransferService() + + with pytest.raises(airdrop.AirdropError, match="not confirmed"): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares={address: 1}, + notes=["hello"], + airdrop_id="legacy", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert service.executed == [] + + +def test_legacy_confirmed_reward_is_backfilled_and_skipped(monkeypatch) -> None: + address = _addresses(1)[0] + legacy_reward = AirdropReward( + airdrop_id="legacy", + address=address, + asa_id=42, + amount_micros=10, + txid="legacy-txid", + ) + rewards, manifests = _install_fake_db(monkeypatch, [legacy_reward]) + _seed_manifest( + manifests, + airdrop_id="legacy", + amounts={address: 10}, + ) + service = FakeTransferService( + legacy_transfer=ConfirmedAssetTransfer( + txid="legacy-txid", + sender=airdrop.cometa_public_key, + receiver=address, + asset_id=42, + amount_micros=10, + confirmed_round=456, + ) + ) + + transactions = asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares={address: 1}, + notes=["hello"], + airdrop_id="legacy", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert transactions == [] + assert service.executed == [] + assert rewards.mongodb_collection.documents[0]["confirmed_round"] == 456 + assert rewards.mongodb_collection.documents[0]["operation_id"] == f"airdrop:legacy:{address}" + + +def test_legacy_confirmation_metadata_does_not_bypass_on_chain_field_validation(monkeypatch) -> None: + address, wrong_receiver = _addresses(2) + legacy_reward = AirdropReward( + airdrop_id="legacy", + address=address, + asa_id=42, + amount_micros=10, + txid="legacy-txid", + confirmed_round=456, + ) + _, manifests = _install_fake_db(monkeypatch, [legacy_reward]) + _seed_manifest( + manifests, + airdrop_id="legacy", + amounts={address: 10}, + ) + service = FakeTransferService( + legacy_transfer=ConfirmedAssetTransfer( + txid="legacy-txid", + sender=airdrop.cometa_public_key, + receiver=wrong_receiver, + asset_id=42, + amount_micros=10, + confirmed_round=456, + ) + ) + + with pytest.raises(airdrop.AirdropConflictError, match="does not match"): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=10, + address_shares={address: 1}, + notes=["hello"], + airdrop_id="legacy", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert service.executed == [] + + +def test_legacy_campaign_cannot_expand_without_explicit_manifest_migration(monkeypatch) -> None: + paid, new_recipient = _addresses(2) + legacy_reward = AirdropReward( + airdrop_id="legacy", + address=paid, + asa_id=42, + amount_micros=10, + txid="legacy-txid", + ) + _install_fake_db(monkeypatch, [legacy_reward]) + service = FakeTransferService() + + with pytest.raises(airdrop.AirdropError, match="no immutable manifest"): + asyncio.run( + airdrop.send_airdrop( + asset_info=_asset(), + total_amount_micros=20, + address_shares={paid: 1, new_recipient: 1}, + notes=["hello"], + airdrop_id="legacy", + transfer_service=service, # type: ignore[arg-type] + ) + ) + + assert service.executed == [] diff --git a/tests/unit/test_algorand_asset_transfer_gateway.py b/tests/unit/test_algorand_asset_transfer_gateway.py new file mode 100644 index 00000000..6f608fd2 --- /dev/null +++ b/tests/unit/test_algorand_asset_transfer_gateway.py @@ -0,0 +1,261 @@ +from dataclasses import replace + +import pytest +from algosdk import account, encoding, transaction + +from flex.application.asset_transfers import ( + AssetTransferRequest, + InvalidAssetTransferError, +) +from flex.blockchain.asset_transfers import AlgorandAssetTransferGateway + + +class FakeAlgod: + def __init__( + self, + *, + min_fee: object = 1_000, + genesis_id: str = "mainnet-v1.0", + genesis_hash: str = "wGHE2Pwdvd7S12BL5FaOP20EGYesN73ktiC1qzkkit8=", + ) -> None: + self.sent: list[transaction.SignedTransaction] = [] + self.min_fee = min_fee + self.genesis_id = genesis_id + self.genesis_hash = genesis_hash + + def suggested_params(self) -> transaction.SuggestedParams: + return transaction.SuggestedParams( + fee=99_999, + first=100, + last=1_100, + gh=self.genesis_hash, + gen=self.genesis_id, + flat_fee=False, + min_fee=self.min_fee, + ) + + def send_transaction(self, signed: transaction.SignedTransaction) -> str: + self.sent.append(signed) + return signed.transaction.get_txid() + + def status(self) -> dict[str, int]: + return {"last-round": 100} + + +class FakeIndexer: + def __init__(self, *, sender: str, receiver: str) -> None: + self.sender = sender + self.receiver = receiver + + def transaction(self, txid: str) -> dict: + return { + "transaction": { + "confirmed-round": 123, + "sender": self.sender, + "asset-transfer-transaction": { + "receiver": self.receiver, + "asset-id": 42, + "amount": 1_000, + }, + } + } + + +def test_gateway_prepares_a_replay_safe_signed_transaction() -> None: + private_key, sender = account.generate_account() + _, receiver = account.generate_account() + algod = FakeAlgod() + gateway = AlgorandAssetTransferGateway( + algod=algod, # type: ignore[arg-type] + indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] + sender=sender, + private_key=private_key, + network="mainnet", + max_fee_microalgos=1_000, + ) + request = AssetTransferRequest( + operation_id="airdrop:summer:recipient", + receiver=receiver, + asset_id=42, + amount_micros=1_000, + note="hello", + ) + + prepared = gateway.prepare(request) + decoded = encoding.msgpack_decode(prepared.signed_transaction) + + assert isinstance(decoded, transaction.SignedTransaction) + assert decoded.transaction.get_txid() == prepared.txid + assert decoded.transaction.receiver == receiver + assert decoded.transaction.index == 42 + assert decoded.transaction.amount == 1_000 + assert decoded.transaction.note == b"hello" + assert len(decoded.transaction.lease) == 32 + assert decoded.transaction.fee == 1_000 + assert prepared.first_valid_round == 100 + assert prepared.last_valid_round == 1_100 + + returned_txid = gateway.broadcast(prepared, request) + + assert returned_txid == prepared.txid + assert len(algod.sent) == 1 + + observed = gateway.lookup_confirmed_transfer(prepared.txid) + + assert observed is not None + assert observed.sender == sender + assert observed.receiver == receiver + assert observed.asset_id == 42 + assert observed.amount_micros == 1_000 + assert observed.confirmed_round == 123 + + +def test_operation_id_deterministically_selects_the_lease() -> None: + private_key, sender = account.generate_account() + _, receiver = account.generate_account() + gateway = AlgorandAssetTransferGateway( + algod=FakeAlgod(), # type: ignore[arg-type] + indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] + sender=sender, + private_key=private_key, + network="mainnet", + max_fee_microalgos=1_000, + ) + request = AssetTransferRequest( + operation_id="lottery:draw:42", + receiver=receiver, + asset_id=7, + amount_micros=1, + ) + + first = encoding.msgpack_decode(gateway.prepare(request).signed_transaction) + second = encoding.msgpack_decode(gateway.prepare(request).signed_transaction) + + assert first.transaction.lease == second.transaction.lease + assert first.transaction.get_txid() == second.transaction.get_txid() + + +def test_gateway_rejects_a_swapped_signed_payload_before_network_io() -> None: + private_key, sender = account.generate_account() + _, receiver = account.generate_account() + _, other_receiver = account.generate_account() + algod = FakeAlgod() + gateway = AlgorandAssetTransferGateway( + algod=algod, # type: ignore[arg-type] + indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] + sender=sender, + private_key=private_key, + network="mainnet", + max_fee_microalgos=1_000, + ) + expected = AssetTransferRequest( + operation_id="airdrop:summer:recipient", + receiver=receiver, + asset_id=42, + amount_micros=1_000, + note="hello", + ) + swapped = gateway.prepare( + AssetTransferRequest( + operation_id=expected.operation_id, + receiver=other_receiver, + asset_id=42, + amount_micros=1_000, + note="hello", + ) + ) + + with pytest.raises(InvalidAssetTransferError, match="do not match"): + gateway.broadcast(swapped, expected) + + assert algod.sent == [] + + +@pytest.mark.parametrize("min_fee", [None, True, -1, 0, 1, 999, 1_001]) +def test_gateway_rejects_an_invalid_or_excessive_network_fee( + min_fee: object, +) -> None: + private_key, sender = account.generate_account() + _, receiver = account.generate_account() + algod = FakeAlgod(min_fee=min_fee) + gateway = AlgorandAssetTransferGateway( + algod=algod, # type: ignore[arg-type] + indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] + sender=sender, + private_key=private_key, + network="mainnet", + max_fee_microalgos=1_000, + ) + + with pytest.raises(InvalidAssetTransferError, match="fee"): + gateway.prepare( + AssetTransferRequest( + operation_id="lottery:fee-policy", + receiver=receiver, + asset_id=7, + amount_micros=1, + ) + ) + + assert algod.sent == [] + + +def test_gateway_rejects_wrong_network_parameters_before_signing() -> None: + private_key, sender = account.generate_account() + _, receiver = account.generate_account() + algod = FakeAlgod( + genesis_id="testnet-v1.0", + genesis_hash="SGO1GKSzyE7IEPItTxCByw9x8FmnrCDexi9/cOUJOiI=", + ) + gateway = AlgorandAssetTransferGateway( + algod=algod, # type: ignore[arg-type] + indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] + sender=sender, + private_key=private_key, + network="mainnet", + max_fee_microalgos=1_000, + ) + + with pytest.raises(InvalidAssetTransferError, match="configured network"): + gateway.prepare( + AssetTransferRequest( + operation_id="lottery:wrong-network", + receiver=receiver, + asset_id=7, + amount_micros=1, + ) + ) + + +def test_gateway_rejects_a_persisted_fee_above_policy_before_network_io() -> None: + private_key, sender = account.generate_account() + _, receiver = account.generate_account() + algod = FakeAlgod() + gateway = AlgorandAssetTransferGateway( + algod=algod, # type: ignore[arg-type] + indexer=FakeIndexer(sender=sender, receiver=receiver), # type: ignore[arg-type] + sender=sender, + private_key=private_key, + network="mainnet", + max_fee_microalgos=1_000, + ) + request = AssetTransferRequest( + operation_id="lottery:persisted-fee", + receiver=receiver, + asset_id=7, + amount_micros=1, + ) + prepared = gateway.prepare(request) + decoded = encoding.msgpack_decode(prepared.signed_transaction) + decoded.transaction.fee = 2_000 + resigned = decoded.transaction.sign(private_key) + tampered = replace( + prepared, + signed_transaction=encoding.msgpack_encode(resigned), + txid=decoded.transaction.get_txid(), + ) + + with pytest.raises(InvalidAssetTransferError, match="fee"): + gateway.broadcast(tampered, request) + + assert algod.sent == [] diff --git a/tests/unit/test_algorand_snapshot_boundaries.py b/tests/unit/test_algorand_snapshot_boundaries.py new file mode 100644 index 00000000..5c22f76d --- /dev/null +++ b/tests/unit/test_algorand_snapshot_boundaries.py @@ -0,0 +1,101 @@ +import asyncio + +import pytest + +from flex.blockchain import info +from flex.domain.algorand import MAX_ALGORAND_UINT + + +def _response(*, assets: list[dict], amount: object = 10, round_number: object = 20) -> dict: + return { + "account": { + "assets": assets, + "amount": amount, + }, + "current-round": round_number, + } + + +def _install_response(monkeypatch, response: dict) -> None: + async def run_sync(func, *args): + del func, args + return response + + monkeypatch.setattr(info, "_run_sync", run_sync) + + +def test_account_snapshot_preserves_full_uint64_domain(monkeypatch) -> None: + _install_response( + monkeypatch, + _response( + assets=[{"asset-id": MAX_ALGORAND_UINT, "amount": MAX_ALGORAND_UINT}], + amount=MAX_ALGORAND_UINT, + round_number=MAX_ALGORAND_UINT, + ), + ) + + snapshot = asyncio.run( + info.get_address_asset_snapshot("ACCOUNT", include_algo=True), + ) + + assert snapshot.balances == { + 0: MAX_ALGORAND_UINT, + MAX_ALGORAND_UINT: MAX_ALGORAND_UINT, + } + assert snapshot.observed_round == MAX_ALGORAND_UINT + + +@pytest.mark.parametrize( + "assets", + [ + [{"asset-id": True, "amount": 1}], + [{"asset-id": 0, "amount": 1}], + [{"asset-id": 1, "amount": -1}], + [{"asset-id": 1, "amount": MAX_ALGORAND_UINT + 1}], + [ + {"asset-id": 1, "amount": 1}, + {"asset-id": 1, "amount": 2}, + ], + ], +) +def test_account_snapshot_rejects_malformed_or_duplicate_holdings( + monkeypatch, + assets: list[dict], +) -> None: + _install_response(monkeypatch, _response(assets=assets)) + + with pytest.raises(RuntimeError): + asyncio.run( + info.get_address_asset_snapshot("ACCOUNT", include_algo=True), + ) + + +@pytest.mark.parametrize( + ("amount", "round_number"), + [ + (True, 1), + (-1, 1), + (MAX_ALGORAND_UINT + 1, 1), + (1, True), + (1, -1), + (1, MAX_ALGORAND_UINT + 1), + ], +) +def test_account_snapshot_rejects_invalid_algo_or_round( + monkeypatch, + amount: object, + round_number: object, +) -> None: + _install_response( + monkeypatch, + _response( + assets=[], + amount=amount, + round_number=round_number, + ), + ) + + with pytest.raises(RuntimeError): + asyncio.run( + info.get_address_asset_snapshot("ACCOUNT", include_algo=True), + ) diff --git a/tests/unit/test_allocation.py b/tests/unit/test_allocation.py new file mode 100644 index 00000000..bbfcc2ac --- /dev/null +++ b/tests/unit/test_allocation.py @@ -0,0 +1,73 @@ +from decimal import Decimal + +import pytest + +from flex.domain.allocation import AllocationError, allocate_proportionally + + +def test_allocation_never_exceeds_the_integer_budget() -> None: + allocations = allocate_proportionally( + 2, + { + "ADDRESS-B": 1, + "ADDRESS-A": 1, + }, + ) + + assert allocations == {"ADDRESS-A": 1, "ADDRESS-B": 1} + assert sum(allocations.values()) == 2 + + +def test_allocation_uses_deterministic_largest_remainders() -> None: + allocations = allocate_proportionally( + 10, + { + "ADDRESS-C": Decimal("1"), + "ADDRESS-B": Decimal("1"), + "ADDRESS-A": Decimal("1"), + }, + ) + + assert allocations == { + "ADDRESS-A": 4, + "ADDRESS-B": 3, + "ADDRESS-C": 3, + } + + +def test_allocation_is_independent_of_input_order() -> None: + forwards = allocate_proportionally( + 100, + {"ADDRESS-A": 0.1, "ADDRESS-B": 0.2}, + ) + backwards = allocate_proportionally( + 100, + {"ADDRESS-B": 0.2, "ADDRESS-A": 0.1}, + ) + + assert forwards == backwards == {"ADDRESS-A": 33, "ADDRESS-B": 67} + + +@pytest.mark.parametrize("total_micros", [True, 0, -1, 1.5]) +def test_allocation_rejects_invalid_budgets(total_micros: object) -> None: + with pytest.raises(AllocationError): + allocate_proportionally(total_micros, {"ADDRESS": 1}) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + "shares", + [ + {}, + {"": 1}, + {"ADDRESS": True}, + {"ADDRESS": 0}, + {"ADDRESS": -1}, + {"ADDRESS": float("nan")}, + {"ADDRESS": float("inf")}, + {"ADDRESS": "not-a-number"}, + {"ADDRESS": Decimal("1e-301")}, + ], +) +def test_allocation_rejects_invalid_shares(shares: dict[str, object]) -> None: + with pytest.raises(AllocationError): + allocate_proportionally(100, shares) # type: ignore[arg-type] diff --git a/tests/unit/test_asset_price_fallback.py b/tests/unit/test_asset_price_fallback.py index c46c141d..f5e3919f 100644 --- a/tests/unit/test_asset_price_fallback.py +++ b/tests/unit/test_asset_price_fallback.py @@ -7,7 +7,12 @@ from env import settings from flex.data import asset_prices from flex.db.model.priced import AssetPrice -from flex.domain.pricing import PriceUnavailableError +from flex.domain.pricing import ( + PriceQuote, + PriceSource, + PriceUnavailableError, + PricingError, +) def _stored_price( @@ -15,6 +20,8 @@ def _stored_price( age_seconds: int, price_algo: float = 2.0, price_usd: float = 0.5, + tinyman_algo_pool_id: int | None = None, + source: str = "vestige", ) -> AssetPrice: observed_at = datetime.now(UTC) - timedelta(seconds=age_seconds) return AssetPrice( @@ -23,7 +30,8 @@ def _stored_price( price_algo=price_algo, price_usd=price_usd, last_update_round=123, - source="vestige", + tinyman_algo_pool_id=tinyman_algo_pool_id, + source=source, observed_at=observed_at, created=observed_at, updated=observed_at, @@ -78,6 +86,75 @@ def test_refresh_failure_rejects_price_older_than_max_stale(monkeypatch) -> None asyncio.run(asset_prices.get_asset_price_not_cached(42)) +def test_refresh_failure_never_serves_legacy_raw_lp_projection(monkeypatch) -> None: + stored = _stored_price(age_seconds=0, tinyman_algo_pool_id=999) + _database(monkeypatch, stored) + monkeypatch.setattr(asset_prices, "update_asset_price", _failed_refresh) + + with pytest.raises(PriceUnavailableError, match="maximum stale window"): + asyncio.run(asset_prices.get_asset_price_not_cached(42)) + + +def test_refresh_failure_never_serves_derived_lp_source_without_pool_marker( + monkeypatch, +) -> None: + stored = _stored_price(age_seconds=0, source=PriceSource.DERIVED_LP.value) + _database(monkeypatch, stored) + monkeypatch.setattr(asset_prices, "update_asset_price", _failed_refresh) + + with pytest.raises(PriceUnavailableError, match="maximum stale window"): + asyncio.run(asset_prices.get_asset_price_not_cached(42)) + + +def test_provider_refresh_removes_legacy_raw_lp_provenance(monkeypatch) -> None: + stored = _stored_price(age_seconds=0, tinyman_algo_pool_id=999) + observed_at = datetime.now(UTC) + quote = PriceQuote.from_raw( + asset_id=42, + algo="3", + usd="0.75", + source=PriceSource.VESTIGE, + observed_at=observed_at, + observed_round=456, + stale_after=timedelta(seconds=settings.asset_prices_ttl), + ) + persisted = [] + + async def fetch_quote(*args, **kwargs) -> PriceQuote: + return quote + + monkeypatch.setattr(asset_prices, "fetch_vestige_price_quote", fetch_quote) + monkeypatch.setattr( + asset_prices, + "_upsert_asset_price", + lambda price: persisted.append(price) is None, + ) + + result = asyncio.run(asset_prices.update_asset_price(stored, current_round=456)) + + assert result.tinyman_algo_pool_id is None + assert result.source == PriceSource.VESTIGE.value + assert result.observed_at == observed_at + assert persisted == [result] + + +def test_legacy_raw_lp_projection_cannot_cross_persistence_boundary() -> None: + stored = _stored_price(age_seconds=0, tinyman_algo_pool_id=999) + + with pytest.raises(PricingError, match="projections are retired"): + asset_prices._upsert_asset_price(stored) + + +def test_derived_lp_source_without_pool_marker_cannot_be_persisted() -> None: + stored = _stored_price( + age_seconds=0, + source=PriceSource.DERIVED_LP.value, + ) + + with pytest.raises(PricingError, match="projections are retired"): + asset_prices._upsert_asset_price(stored) + + def test_invalid_fresh_database_value_is_refreshed(monkeypatch) -> None: stored = _stored_price(age_seconds=0, price_algo=float("nan")) refreshed = _stored_price(age_seconds=0, price_algo=3.0, price_usd=0.75) @@ -105,12 +182,22 @@ def test_list_reads_filter_invalid_and_expired_prices(monkeypatch) -> None: age_seconds=settings.asset_prices_max_stale + 1, ) expired.id = 44 + legacy = _stored_price( + age_seconds=0, + tinyman_algo_pool_id=999, + ) + legacy.id = 45 + derived = _stored_price( + age_seconds=0, + source=PriceSource.DERIVED_LP.value, + ) + derived.id = 46 monkeypatch.setattr( asset_prices, "db", SimpleNamespace( asset_prices=SimpleNamespace( - get_all=lambda: [valid, invalid, expired], + get_all=lambda: [valid, invalid, expired, legacy, derived], ), ), ) diff --git a/tests/unit/test_asset_supply.py b/tests/unit/test_asset_supply.py new file mode 100644 index 00000000..2974ade9 --- /dev/null +++ b/tests/unit/test_asset_supply.py @@ -0,0 +1,337 @@ +from collections.abc import Callable +from copy import deepcopy +from types import SimpleNamespace +from typing import Any + +import pytest + +from flex.blockchain import info +from flex.data import assets as asset_data +from flex.db.model.blockchain import ( + TOTAL_SUPPLY_SOURCE_INDEXER, + UINT64_MAX, + Asset, +) + + +def _asset_response(*, total: int, decimals: int) -> dict[str, Any]: + return { + "asset": { + "params": { + "creator": "CREATOR", + "decimals": decimals, + "name": "Precision Asset", + "reserve": "RESERVE", + "total": total, + "unit-name": "PRECISE", + } + } + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("total", [2**53 + 1, UINT64_MAX]) +async def test_fetch_asset_preserves_full_uint64_supply( + monkeypatch: pytest.MonkeyPatch, + total: int, +) -> None: + async def fake_run_sync(func: Callable[..., Any], *args: Any) -> dict[str, Any]: + del func, args + return _asset_response(total=total, decimals=0) + + monkeypatch.setattr(info, "_run_sync", fake_run_sync) + + asset = await info.fetch_asset(42) + + assert asset.total_supply_micros == total + assert asset.total_supply_base_units == total + + stored = asset.to_dict() + assert stored["total_supply_micros"] == str(total) + assert stored["total_supply_source"] == TOTAL_SUPPLY_SOURCE_INDEXER + restored = Asset.from_dict(stored) + assert restored.total_supply_micros == total + assert restored.total_supply_is_authoritative is True + + +def test_legacy_asset_document_marks_reconstructed_supply_non_authoritative() -> None: + legacy_document = { + "id": 42, + "name": "Legacy Asset", + "decimals": 6, + "unit_name": "OLD", + "creator": "CREATOR", + "reserve": "RESERVE", + "total_supply": 123.456789, + } + + asset = Asset.from_dict(legacy_document) + + assert asset.total_supply_micros == 123_456_789 + assert asset.total_supply_is_authoritative is False + assert "total_supply_micros" not in asset.to_dict() + + +def test_canonical_supply_wins_over_lossy_legacy_float() -> None: + canonical_supply = 2**53 + 1 + + asset = Asset( + id=42, + name="Canonical Asset", + decimals=0, + unit_name="CANON", + creator="CREATOR", + reserve="RESERVE", + total_supply=float(canonical_supply), + total_supply_micros=canonical_supply, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, + ) + + assert int(asset.total_supply) != canonical_supply + assert asset.total_supply_micros == canonical_supply + assert asset.total_supply_is_authoritative is True + + +@pytest.mark.parametrize("invalid_supply", [-1, UINT64_MAX + 1]) +def test_asset_rejects_supply_outside_algorand_uint64(invalid_supply: int) -> None: + with pytest.raises(ValueError, match="total_supply_micros"): + Asset( + id=42, + name="Invalid Asset", + decimals=0, + unit_name="BAD", + creator="CREATOR", + reserve="RESERVE", + total_supply=0, + total_supply_micros=invalid_supply, + ) + + +class _AssetCollection: + def __init__(self, document: dict[str, Any]) -> None: + self.document = document + self.update_query: dict[str, Any] | None = None + + def find_one( + self, + query: dict[str, Any], + projection: dict[str, int] | None = None, + ) -> dict[str, Any] | None: + if "_id" in query and query["_id"] != self.document["_id"]: + return None + result = deepcopy(self.document) + if projection is None: + return result + return {key: value for key, value in result.items() if key == "_id" or projection.get(key)} + + def find_one_and_update( + self, + query: dict[str, Any], + update: dict[str, Any], + **kwargs: Any, + ) -> dict[str, Any] | None: + del kwargs + self.update_query = query + may_migrate = ( + self.document.get("total_supply_micros") is None + or self.document.get("total_supply_source") != TOTAL_SUPPLY_SOURCE_INDEXER + ) + if query["_id"] != self.document["_id"] or not may_migrate: + return None + self.document.update(update["$set"]) + return deepcopy(self.document) + + +class _AssetManager: + def __init__(self, collection: _AssetCollection) -> None: + self.mongodb_collection = collection + + def get_by_primary_key( + self, + asset_id: int, + *, + throw_ex: bool, + ) -> Asset | None: + del asset_id, throw_ex + return Asset.from_dict(deepcopy(self.mongodb_collection.document)) + + +@pytest.mark.asyncio +async def test_financial_supply_backfills_from_indexer_instead_of_legacy_float( + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = _AssetCollection( + { + "_id": "asset-42", + "id": 42, + "total_supply": float(UINT64_MAX), + } + ) + monkeypatch.setattr( + asset_data, + "db", + SimpleNamespace( + assets=SimpleNamespace( + mongodb_collection=collection, + ) + ), + ) + authoritative = Asset( + id=42, + name="Canonical Asset", + decimals=0, + unit_name="CANON", + creator="CREATOR", + reserve="RESERVE", + total_supply=0, + total_supply_micros=UINT64_MAX, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, + ) + fetches: list[int] = [] + + async def fetch_asset(asset_id: int) -> Asset: + fetches.append(asset_id) + return authoritative + + monkeypatch.setattr(asset_data, "fetch_asset", fetch_asset) + + supply = await asset_data.get_asset_total_supply.__wrapped__(42) + + assert supply == UINT64_MAX + assert fetches == [42] + assert collection.document["total_supply_micros"] == str(UINT64_MAX) + assert collection.document["total_supply_source"] == TOTAL_SUPPLY_SOURCE_INDEXER + assert collection.update_query == { + "_id": "asset-42", + "$or": [ + {"total_supply_micros": {"$exists": False}}, + {"total_supply_micros": None}, + {"total_supply_source": {"$ne": TOTAL_SUPPLY_SOURCE_INDEXER}}, + ], + } + + +@pytest.mark.asyncio +async def test_financial_supply_uses_persisted_canonical_units_without_indexer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = _AssetCollection( + { + "_id": "asset-42", + "id": 42, + "total_supply": 1.0, + "total_supply_micros": str(2**53 + 1), + "total_supply_source": TOTAL_SUPPLY_SOURCE_INDEXER, + } + ) + monkeypatch.setattr( + asset_data, + "db", + SimpleNamespace( + assets=SimpleNamespace( + mongodb_collection=collection, + ) + ), + ) + + async def unexpected_fetch(asset_id: int) -> Asset: + pytest.fail(f"unexpected Indexer read for asset {asset_id}") + + monkeypatch.setattr(asset_data, "fetch_asset", unexpected_fetch) + + supply = await asset_data.get_asset_total_supply.__wrapped__(42) + + assert supply == 2**53 + 1 + assert collection.update_query is None + + +@pytest.mark.asyncio +async def test_unverified_persisted_supply_is_overwritten_from_indexer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = _AssetCollection( + { + "_id": "asset-42", + "id": 42, + "total_supply": float(2**53), + "total_supply_micros": str(2**53), + "total_supply_source": None, + } + ) + monkeypatch.setattr( + asset_data, + "db", + SimpleNamespace( + assets=SimpleNamespace( + mongodb_collection=collection, + ) + ), + ) + authoritative_supply = 2**53 + 1 + + async def fetch_asset(asset_id: int) -> Asset: + assert asset_id == 42 + return Asset( + id=42, + name="Canonical Asset", + decimals=0, + unit_name="CANON", + creator="CREATOR", + reserve="RESERVE", + total_supply=0, + total_supply_micros=authoritative_supply, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, + ) + + monkeypatch.setattr(asset_data, "fetch_asset", fetch_asset) + + supply = await asset_data.get_asset_total_supply.__wrapped__(42) + + assert supply == authoritative_supply + assert collection.document["total_supply_micros"] == str(authoritative_supply) + assert collection.document["total_supply_source"] == TOTAL_SUPPLY_SOURCE_INDEXER + + +@pytest.mark.asyncio +async def test_full_asset_read_migrates_legacy_supply_before_returning( + monkeypatch: pytest.MonkeyPatch, +) -> None: + collection = _AssetCollection( + { + "_id": "asset-42", + "id": 42, + "name": "Legacy Asset", + "decimals": 0, + "unit_name": "OLD", + "creator": "CREATOR", + "reserve": "RESERVE", + "total_supply": float(UINT64_MAX), + } + ) + manager = _AssetManager(collection) + monkeypatch.setattr( + asset_data, + "db", + SimpleNamespace(assets=manager), + ) + + async def fetch_asset(asset_id: int) -> Asset: + assert asset_id == 42 + return Asset( + id=42, + name="Canonical Asset", + decimals=0, + unit_name="CANON", + creator="CREATOR", + reserve="RESERVE", + total_supply=0, + total_supply_micros=UINT64_MAX, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, + ) + + monkeypatch.setattr(asset_data, "fetch_asset", fetch_asset) + + asset = await asset_data.get_full_asset.__wrapped__(42) + + assert asset.total_supply_micros == UINT64_MAX + assert asset.total_supply_is_authoritative is True diff --git a/tests/unit/test_asset_transfer_repository.py b/tests/unit/test_asset_transfer_repository.py new file mode 100644 index 00000000..26084fc8 --- /dev/null +++ b/tests/unit/test_asset_transfer_repository.py @@ -0,0 +1,184 @@ +from datetime import UTC, datetime +from unittest.mock import Mock + +import pytest +from bson import Decimal128 +from pymongo import ReturnDocument + +from flex.db.asset_transfer_intents import ( + MongoAssetTransferIntentRepository, + TransferIntentPersistenceError, +) +from flex.db.model.transfers import AssetTransferIntent + + +def _intent() -> AssetTransferIntent: + timestamp = datetime(2026, 1, 1, tzinfo=UTC) + return AssetTransferIntent( + id="airdrop:summer:address", + receiver="address", + asset_id="42", + amount_micros="1000", + note="hello", + signed_transaction="signed", + txid="txid", + first_valid_round=100, + last_valid_round=1_100, + status="prepared", + created=timestamp, + updated=timestamp, + ) + + +def test_repository_installs_unique_and_reconciliation_indexes() -> None: + collection = Mock() + repository = MongoAssetTransferIntentRepository(collection) + + repository.ensure_indexes() + + collection.create_index.assert_any_call("id", unique=True, name="id_unique") + collection.create_index.assert_any_call( + [("status", 1), ("updated", 1)], + name="status_updated_idx", + ) + + +def test_reserve_uses_one_atomic_set_on_insert() -> None: + collection = Mock() + intent = _intent() + collection.find_one_and_update.return_value = intent.to_dict() + repository = MongoAssetTransferIntentRepository(collection) + + reserved = repository.reserve(intent) + + assert reserved == intent + collection.find_one_and_update.assert_called_once_with( + {"id": intent.id}, + {"$setOnInsert": intent.to_dict()}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + + +def test_confirmation_is_guarded_by_operation_and_transaction_id() -> None: + collection = Mock() + confirmed = _intent() + confirmed.status = "confirmed" + confirmed.confirmed_round = 777 + collection.find_one_and_update.return_value = confirmed.to_dict() + repository = MongoAssetTransferIntentRepository(collection) + + result = repository.mark_confirmed( + confirmed.id, + confirmed.txid, + confirmed.confirmed_round, + ) + + assert result.status == "confirmed" + assert result.confirmed_round == 777 + args, kwargs = collection.find_one_and_update.call_args + assert args[0] == { + "id": confirmed.id, + "txid": confirmed.txid, + "status": {"$ne": "confirmed"}, + } + assert args[1]["$set"]["status"] == "confirmed" + assert args[1]["$set"]["confirmed_round"] == Decimal128("777") + assert kwargs == {"return_document": ReturnDocument.AFTER} + + +def test_repeated_identical_confirmation_preserves_terminal_evidence() -> None: + collection = Mock() + confirmed = _intent() + confirmed.status = "confirmed" + confirmed.confirmed_round = 777 + confirmed.confirmed_at = datetime(2026, 1, 2, tzinfo=UTC) + collection.find_one_and_update.return_value = None + collection.find_one.return_value = confirmed.to_dict() + repository = MongoAssetTransferIntentRepository(collection) + + result = repository.mark_confirmed(confirmed.id, confirmed.txid, 777) + + assert result == confirmed + assert result.confirmed_at == datetime(2026, 1, 2, tzinfo=UTC) + query = collection.find_one_and_update.call_args.args[0] + assert query["status"] == {"$ne": "confirmed"} + collection.find_one.assert_called_once_with({"id": confirmed.id}) + + +@pytest.mark.parametrize( + ("current", "txid", "round_number", "message"), + [ + (None, "txid", 777, "no longer exists"), + (_intent(), "other-txid", 777, "now belongs to transaction"), + (_intent(), "txid", 778, "already confirmed in round"), + (_intent(), "txid", 777, "remains in status"), + ], +) +def test_confirmation_cas_miss_fails_closed( + current: AssetTransferIntent | None, + txid: str, + round_number: int, + message: str, +) -> None: + if current is not None and message == "already confirmed in round": + current.status = "confirmed" + current.confirmed_round = 777 + collection = Mock() + collection.find_one_and_update.return_value = None + collection.find_one.return_value = None if current is None else current.to_dict() + repository = MongoAssetTransferIntentRepository(collection) + + with pytest.raises(TransferIntentPersistenceError, match=message): + repository.mark_confirmed("airdrop:summer:address", txid, round_number) + + +@pytest.mark.parametrize( + "update", + [ + lambda repository, intent: repository.record_attempt(intent.id, intent.txid), + lambda repository, intent: repository.mark_submitted(intent.id, intent.txid), + lambda repository, intent: repository.record_error(intent.id, intent.txid, "late error"), + ], +) +def test_non_confirmation_updates_cannot_mutate_confirmed_intent(update) -> None: + collection = Mock() + confirmed = _intent() + confirmed.status = "confirmed" + confirmed.attempt_count = 4 + confirmed.confirmed_round = 777 + collection.find_one_and_update.return_value = None + collection.find_one.return_value = confirmed.to_dict() + repository = MongoAssetTransferIntentRepository(collection) + + result = update(repository, confirmed) + + assert result == confirmed + query = collection.find_one_and_update.call_args.args[0] + assert query == { + "id": confirmed.id, + "txid": confirmed.txid, + "status": {"$ne": "confirmed"}, + } + collection.find_one.assert_called_once_with({"id": confirmed.id}) + + +@pytest.mark.parametrize( + "update", + [ + lambda repository, intent: repository.record_attempt(intent.id, "other-txid"), + lambda repository, intent: repository.mark_submitted(intent.id, "other-txid"), + lambda repository, intent: repository.record_error(intent.id, "other-txid", "late error"), + ], +) +def test_non_confirmation_update_rejects_transaction_mismatch(update) -> None: + collection = Mock() + confirmed = _intent() + confirmed.status = "confirmed" + confirmed.confirmed_round = 777 + collection.find_one_and_update.return_value = None + collection.find_one.return_value = confirmed.to_dict() + repository = MongoAssetTransferIntentRepository(collection) + + with pytest.raises(TransferIntentPersistenceError, match="changed while updating"): + update(repository, confirmed) diff --git a/tests/unit/test_asset_transfers.py b/tests/unit/test_asset_transfers.py new file mode 100644 index 00000000..a4381569 --- /dev/null +++ b/tests/unit/test_asset_transfers.py @@ -0,0 +1,445 @@ +from dataclasses import replace +from datetime import UTC, datetime + +import pytest + +from flex.application.asset_transfers import ( + AssetTransferConflictError, + AssetTransferError, + AssetTransferExpiredError, + AssetTransferPendingError, + AssetTransferRequest, + AssetTransferService, + ConfirmedAssetTransfer, + InvalidAssetTransferError, + PreparedAssetTransfer, + TransferStatus, +) +from flex.db.model.transfers import AssetTransferIntent + + +class InMemoryIntentRepository: + def __init__(self) -> None: + self.intents: dict[str, AssetTransferIntent] = {} + + def get(self, operation_id: str) -> AssetTransferIntent | None: + return self.intents.get(operation_id) + + def reserve(self, intent: AssetTransferIntent) -> AssetTransferIntent: + return self.intents.setdefault(intent.id, intent) + + def record_attempt(self, operation_id: str, txid: str) -> AssetTransferIntent: + intent = self._intent(operation_id, txid) + intent.attempt_count += 1 + return intent + + def mark_submitted(self, operation_id: str, txid: str) -> AssetTransferIntent: + intent = self._intent(operation_id, txid) + intent.status = TransferStatus.SUBMITTED + return intent + + def mark_confirmed( + self, + operation_id: str, + txid: str, + confirmed_round: int, + ) -> AssetTransferIntent: + intent = self._intent(operation_id, txid) + intent.status = TransferStatus.CONFIRMED + intent.confirmed_round = confirmed_round + return intent + + def record_error( + self, + operation_id: str, + txid: str, + error: str, + ) -> AssetTransferIntent: + intent = self._intent(operation_id, txid) + intent.last_error = error + return intent + + def _intent(self, operation_id: str, txid: str) -> AssetTransferIntent: + intent = self.intents[operation_id] + assert intent.txid == txid + return intent + + +class ConfirmOnUpdateRepository(InMemoryIntentRepository): + def __init__(self, update_name: str) -> None: + super().__init__() + self.update_name = update_name + + def _confirm(self, intent: AssetTransferIntent, update_name: str) -> AssetTransferIntent: + if self.update_name == update_name: + intent.status = TransferStatus.CONFIRMED + intent.confirmed_round = 777 + return intent + + def record_attempt(self, operation_id: str, txid: str) -> AssetTransferIntent: + return self._confirm( + super().record_attempt(operation_id, txid), + "record_attempt", + ) + + def mark_submitted(self, operation_id: str, txid: str) -> AssetTransferIntent: + return self._confirm( + super().mark_submitted(operation_id, txid), + "mark_submitted", + ) + + def record_error( + self, + operation_id: str, + txid: str, + error: str, + ) -> AssetTransferIntent: + return self._confirm( + super().record_error(operation_id, txid, error), + "record_error", + ) + + +class ReconcileReloadRepository(InMemoryIntentRepository): + def __init__(self, reloaded: AssetTransferIntent | None) -> None: + super().__init__() + self.reloaded = reloaded + self.get_count = 0 + + def get(self, operation_id: str) -> AssetTransferIntent | None: + self.get_count += 1 + if self.get_count == 1: + return super().get(operation_id) + return self.reloaded + + +class FakeTransferGateway: + def __init__(self) -> None: + self.prepare_count = 0 + self.broadcasted: list[str] = [] + self.current = 100 + self.confirmed_round = 101 + self.lookup_round: int | None = None + self.broadcast_error: Exception | None = None + self.wait_error: Exception | None = None + self.returned_txid: str | None = None + + def prepare(self, request: AssetTransferRequest) -> PreparedAssetTransfer: + self.prepare_count += 1 + return PreparedAssetTransfer( + signed_transaction=f"signed-{request.operation_id}", + txid=f"txid-{request.operation_id}", + first_valid_round=100, + last_valid_round=1_100, + ) + + def broadcast( + self, + prepared: PreparedAssetTransfer, + request: AssetTransferRequest, + ) -> str: + self.broadcasted.append(prepared.signed_transaction) + if self.broadcast_error is not None: + raise self.broadcast_error + if self.returned_txid is not None: + return self.returned_txid + return prepared.signed_transaction.replace("signed-", "txid-") + + def wait_for_confirmation(self, txid: str) -> int: + if self.wait_error is not None: + raise self.wait_error + return self.confirmed_round + + def lookup_confirmed_round(self, txid: str) -> int | None: + return self.lookup_round + + def lookup_confirmed_transfer(self, txid: str) -> ConfirmedAssetTransfer | None: + if self.lookup_round is None: + return None + return ConfirmedAssetTransfer( + txid=txid, + sender="SENDER", + receiver="ADDRESS", + asset_id=42, + amount_micros=1_000, + confirmed_round=self.lookup_round, + ) + + def current_round(self) -> int: + return self.current + + +def _request(**changes: object) -> AssetTransferRequest: + values = { + "operation_id": "airdrop:summer:ADDRESS", + "receiver": "ADDRESS", + "asset_id": 42, + "amount_micros": 1_000, + "note": "thank you", + } + values.update(changes) + return AssetTransferRequest(**values) # type: ignore[arg-type] + + +def _intent(**changes: object) -> AssetTransferIntent: + timestamp = datetime(2026, 1, 1, tzinfo=UTC) + values = { + "id": "airdrop:summer:ADDRESS", + "receiver": "ADDRESS", + "asset_id": "42", + "amount_micros": "1000", + "note": "thank you", + "signed_transaction": "persisted-signed-transaction", + "txid": "persisted-txid", + "first_valid_round": 1, + "last_valid_round": 1_000, + "status": TransferStatus.SUBMITTED, + "created": timestamp, + "updated": timestamp, + } + values.update(changes) + return AssetTransferIntent(**values) # type: ignore[arg-type] + + +def test_retry_rebroadcasts_the_same_persisted_signed_transaction() -> None: + repository = InMemoryIntentRepository() + gateway = FakeTransferGateway() + gateway.wait_error = TimeoutError("confirmation timeout") + service = AssetTransferService(repository, gateway) + + with pytest.raises(AssetTransferPendingError): + service.execute(_request()) + + persisted = repository.intents["airdrop:summer:ADDRESS"] + assert persisted.txid == "txid-airdrop:summer:ADDRESS" + assert persisted.signed_transaction == "signed-airdrop:summer:ADDRESS" + + gateway.wait_error = None + receipt = service.execute(_request()) + + assert receipt.txid == persisted.txid + assert gateway.prepare_count == 1 + assert gateway.broadcasted == [ + "signed-airdrop:summer:ADDRESS", + "signed-airdrop:summer:ADDRESS", + ] + assert persisted.attempt_count == 2 + + +def test_ambiguous_broadcast_is_resolved_by_confirmation() -> None: + repository = InMemoryIntentRepository() + gateway = FakeTransferGateway() + gateway.broadcast_error = TimeoutError("connection dropped after submit") + service = AssetTransferService(repository, gateway) + + receipt = service.execute(_request()) + + assert receipt.confirmed_round == 101 + assert repository.intents[receipt.operation_id].status == TransferStatus.CONFIRMED + + +def test_confirmed_retry_does_not_contact_the_gateway() -> None: + repository = InMemoryIntentRepository() + confirmed = _intent( + status=TransferStatus.CONFIRMED, + confirmed_round=500, + ) + repository.intents[confirmed.id] = confirmed + gateway = FakeTransferGateway() + service = AssetTransferService(repository, gateway) + + receipt = service.execute(_request()) + + assert receipt.already_confirmed is True + assert receipt.confirmed_round == 500 + assert gateway.prepare_count == 0 + assert gateway.broadcasted == [] + + +def test_concurrent_confirmation_before_broadcast_returns_terminal_receipt() -> None: + repository = ConfirmOnUpdateRepository("record_attempt") + existing = _intent() + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + service = AssetTransferService(repository, gateway) + + receipt = service.execute(_request()) + + assert receipt.already_confirmed is True + assert receipt.confirmed_round == 777 + assert gateway.broadcasted == [] + + +@pytest.mark.parametrize("update_name", ["mark_submitted", "record_error"]) +def test_concurrent_confirmation_after_broadcast_is_not_reported_pending( + update_name: str, +) -> None: + repository = ConfirmOnUpdateRepository(update_name) + existing = _intent() + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + if update_name == "record_error": + gateway.wait_error = TimeoutError("another worker confirmed") + else: + gateway.returned_txid = existing.txid + service = AssetTransferService(repository, gateway) + + receipt = service.execute(_request()) + + assert receipt.already_confirmed is True + assert receipt.confirmed_round == 777 + assert gateway.broadcasted == ["persisted-signed-transaction"] + + +def test_idempotency_key_cannot_be_reused_for_another_amount() -> None: + repository = InMemoryIntentRepository() + existing = _intent() + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + service = AssetTransferService(repository, gateway) + + with pytest.raises(AssetTransferConflictError): + service.execute(_request(amount_micros=1_001)) + + assert gateway.broadcasted == [] + + +def test_expired_intent_reconciles_a_confirmed_transaction() -> None: + repository = InMemoryIntentRepository() + existing = _intent(last_valid_round=99) + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + gateway.current = 100 + gateway.lookup_round = 88 + service = AssetTransferService(repository, gateway) + + receipt = service.execute(_request()) + + assert receipt.already_confirmed is True + assert receipt.confirmed_round == 88 + assert gateway.broadcasted == [] + + +def test_expired_unconfirmed_intent_requires_manual_reconciliation() -> None: + repository = InMemoryIntentRepository() + existing = _intent(last_valid_round=99) + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + gateway.current = 100 + service = AssetTransferService(repository, gateway) + + with pytest.raises(AssetTransferExpiredError): + service.execute(_request()) + + assert gateway.prepare_count == 0 + assert gateway.broadcasted == [] + assert "manual reconciliation" in existing.last_error + + +def test_reconcile_marks_an_observed_transaction_confirmed() -> None: + repository = InMemoryIntentRepository() + existing = _intent() + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + gateway.lookup_round = 777 + service = AssetTransferService(repository, gateway) + + result = service.reconcile(existing.id) + + assert result.status == TransferStatus.CONFIRMED + assert result.confirmed_round == 777 + + +def test_reconcile_returns_confirmation_committed_during_chain_lookup() -> None: + confirmed = _intent( + status=TransferStatus.CONFIRMED, + confirmed_round=778, + ) + repository = ReconcileReloadRepository(confirmed) + existing = _intent() + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + service = AssetTransferService(repository, gateway) + + result = service.reconcile(existing.id) + + assert repository.get_count == 2 + assert result.status == TransferStatus.CONFIRMED + assert result.txid == existing.txid + assert result.confirmed_round == 778 + + +def test_reconcile_fails_if_the_intent_disappears_during_chain_lookup() -> None: + repository = ReconcileReloadRepository(None) + existing = _intent() + repository.intents[existing.id] = existing + service = AssetTransferService(repository, FakeTransferGateway()) + + with pytest.raises(AssetTransferError, match="disappeared during reconciliation"): + service.reconcile(existing.id) + + +def test_reconcile_fails_if_the_persisted_transaction_changes() -> None: + replacement = _intent(txid="replacement-txid") + repository = ReconcileReloadRepository(replacement) + existing = _intent() + repository.intents[existing.id] = existing + service = AssetTransferService(repository, FakeTransferGateway()) + + with pytest.raises(AssetTransferConflictError, match="changed transaction"): + service.reconcile(existing.id) + + +def test_reconcile_builds_nonterminal_result_from_reloaded_intent() -> None: + reloaded = _intent( + status=TransferStatus.PREPARED, + last_valid_round=99, + ) + repository = ReconcileReloadRepository(reloaded) + existing = _intent(last_valid_round=1_000) + repository.intents[existing.id] = existing + gateway = FakeTransferGateway() + gateway.current = 100 + service = AssetTransferService(repository, gateway) + + result = service.reconcile(existing.id) + + assert result.status == "expired_unconfirmed" + assert result.confirmed_round is None + + +def test_full_algorand_uint64_values_are_persisted_without_bson_integers() -> None: + repository = InMemoryIntentRepository() + gateway = FakeTransferGateway() + service = AssetTransferService(repository, gateway) + + request = _request(asset_id=2**64 - 1, amount_micros=2**64 - 1) + service.execute(request) + + intent = repository.intents[request.operation_id] + assert intent.asset_id == str(2**64 - 1) + assert intent.amount_micros == str(2**64 - 1) + assert intent.asset_id_int == 2**64 - 1 + assert intent.amount_micros_int == 2**64 - 1 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("operation_id", ""), + ("operation_id", "x" * 201), + ("receiver", ""), + ("asset_id", True), + ("asset_id", 0), + ("asset_id", 2**64), + ("amount_micros", True), + ("amount_micros", 0), + ("amount_micros", 2**64), + ("note", "x" * 1_001), + ], +) +def test_transfer_request_rejects_invalid_values(field: str, value: object) -> None: + request = _request() + + with pytest.raises(InvalidAssetTransferError): + replace(request, **{field: value}) diff --git a/tests/unit/test_bson_uint64_models.py b/tests/unit/test_bson_uint64_models.py new file mode 100644 index 00000000..75a04595 --- /dev/null +++ b/tests/unit/test_bson_uint64_models.py @@ -0,0 +1,133 @@ +from datetime import UTC, datetime + +import pytest +from bson import BSON, Decimal128 + +from flex.db.model.blockchain import TOTAL_SUPPLY_SOURCE_INDEXER, Asset, LpToken +from flex.db.model.liquidity_pools import LpState +from flex.db.model.priced import AssetPrice +from flex.db.model.transfers import AssetTransferIntent + +UINT64_MAX = 2**64 - 1 + + +def test_lp_token_round_trips_full_uint64_identifiers() -> None: + token = LpToken( + id=UINT64_MAX, + asset1_id=UINT64_MAX - 1, + asset2_id=0, + dex_provider="tinyman", + address="POOL", + pool_id=UINT64_MAX, + ) + + restored = LpToken.from_dict( + BSON(BSON.encode(token.to_dict())).decode(), + ) + + assert restored.id == UINT64_MAX + assert restored.asset1_id == UINT64_MAX - 1 + assert restored.pool_id == UINT64_MAX + assert LpToken.encode_query({"id": UINT64_MAX}) == { + "id": Decimal128(str(UINT64_MAX)), + } + + +def test_asset_round_trips_full_uint64_identifier_and_supply() -> None: + asset = Asset( + id=UINT64_MAX, + name="MAX", + decimals=0, + unit_name="MAX", + creator="CREATOR", + reserve="RESERVE", + total_supply=0, + total_supply_micros=UINT64_MAX, + total_supply_source=TOTAL_SUPPLY_SOURCE_INDEXER, + ) + + restored = Asset.from_dict( + BSON(BSON.encode(asset.to_dict())).decode(), + ) + + assert restored.id == UINT64_MAX + assert restored.total_supply_micros == UINT64_MAX + assert Asset.encode_query({"id": UINT64_MAX}) == { + "id": Decimal128(str(UINT64_MAX)), + } + + +def test_asset_price_round_trips_full_uint64_identifiers_and_round() -> None: + observed_at = datetime(2026, 1, 1, tzinfo=UTC) + price = AssetPrice( + id=UINT64_MAX, + price_usd=1, + price_algo=2, + last_update_round=UINT64_MAX, + name="MAX", + tinyman_algo_pool_id=UINT64_MAX, + source="tinyman", + observed_at=observed_at, + ) + + restored = AssetPrice.from_dict( + BSON(BSON.encode(price.to_dict())).decode(), + ) + + assert restored.id == UINT64_MAX + assert restored.last_update_round == UINT64_MAX + assert restored.tinyman_algo_pool_id == UINT64_MAX + assert AssetPrice.encode_query({"id": UINT64_MAX}) == { + "id": Decimal128(str(UINT64_MAX)), + } + + +def test_transfer_intent_round_trips_full_uint64_rounds() -> None: + timestamp = datetime(2026, 1, 1, tzinfo=UTC) + intent = AssetTransferIntent( + id="airdrop:max-round", + receiver="receiver", + asset_id="1", + amount_micros="1", + note=None, + signed_transaction="signed", + txid="txid", + first_valid_round=UINT64_MAX - 1, + last_valid_round=UINT64_MAX, + status="confirmed", + confirmed_round=UINT64_MAX, + created=timestamp, + updated=timestamp, + ) + + encoded = intent.to_dict() + restored = AssetTransferIntent.from_dict( + BSON(BSON.encode(encoded)).decode(), + ) + + assert encoded["first_valid_round"] == Decimal128(str(UINT64_MAX - 1)) + assert encoded["last_valid_round"] == Decimal128(str(UINT64_MAX)) + assert encoded["confirmed_round"] == Decimal128(str(UINT64_MAX)) + assert restored.first_valid_round == UINT64_MAX - 1 + assert restored.last_valid_round == UINT64_MAX + assert restored.confirmed_round == UINT64_MAX + + +def test_lp_state_rejects_invalid_balance_before_initial_insert() -> None: + with pytest.raises(ValueError, match="asset1_reserve_micros"): + LpState( + id=1, + token_id=99, + asset1_id=7, + asset2_id=0, + dex_provider="tinyman", + address="POOL", + last_updated_round=1, + asset1_reserve_micros=True, + asset2_reserve_micros=1, + total_tokens_micros=1, + asset1_reserve=0, + asset2_reserve=0, + total_tokens=0, + token_price_algo=0, + ) diff --git a/tests/unit/test_database_indexes.py b/tests/unit/test_database_indexes.py index f83482c8..e3685670 100644 --- a/tests/unit/test_database_indexes.py +++ b/tests/unit/test_database_indexes.py @@ -4,8 +4,12 @@ import pytest from flex.db.indexes import ( + create_unique_id_index_fail_closed, deduplicate_and_create_unique_id_index, + delete_unverified_legacy_lp_prices, + ensure_airdrop_indexes, ensure_database_indexes, + ensure_sync_state_singleton, ) @@ -15,13 +19,18 @@ def _manager(collection: Mock) -> SimpleNamespace: def _database(**collections: Mock) -> SimpleNamespace: names = ( + "airdrop_manifests", + "assets", "asset_prices", + "asset_transfer_intents", "pool_transactions", "lp_transactions", "lp_states", "pool_states", "user_states", "lp_tokens", + "airdrop_rewards", + "sync_states", ) return SimpleNamespace(**{name: _manager(collections.get(name, Mock())) for name in names}) @@ -62,39 +71,246 @@ def test_deduplication_keeps_newest_record_before_creating_index() -> None: ) < collection.mock_calls.index(call.create_index("id", unique=True, name="id_unique")) +def test_immutable_ledger_duplicates_abort_without_deleting_evidence() -> None: + collection = Mock() + collection.aggregate.return_value = [ + { + "_id": "operation", + "count": 2, + "keep_id": "confirmed", + "all_ids": ["confirmed", "prepared"], + } + ] + + with pytest.raises(RuntimeError, match="duplicate immutable ID"): + create_unique_id_index_fail_closed( + collection, + collection_name="asset_transfer_intents", + ) + + collection.delete_many.assert_not_called() + collection.create_index.assert_not_called() + + def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: unique_collections = { name: Mock() for name in ( + "airdrop_manifests", + "assets", "asset_prices", + "asset_transfer_intents", "pool_transactions", "lp_transactions", + "lp_tokens", ) } for collection in unique_collections.values(): collection.aggregate.return_value = [] + lp_states = Mock() + lp_states.aggregate.return_value = [] + sync_states = Mock() + sync_states.find.return_value = [] + airdrop_rewards = Mock() + airdrop_rewards.aggregate.return_value = [] - database = _database(**unique_collections) + database = _database( + **unique_collections, + lp_states=lp_states, + sync_states=sync_states, + airdrop_rewards=airdrop_rewards, + ) + database.asset_prices.mongodb_collection.delete_many.return_value = SimpleNamespace( + deleted_count=1, + ) removed = ensure_database_indexes(database) assert removed == { + "airdrop_manifests": 0, + "assets": 0, "asset_prices": 0, + "asset_transfer_intents": 0, "pool_transactions": 0, "lp_transactions": 0, + "lp_tokens": 0, } for collection in unique_collections.values(): collection.create_index.assert_called_once_with("id", unique=True, name="id_unique") + database.asset_prices.mongodb_collection.delete_many.assert_called_once_with( + { + "$or": [ + { + "tinyman_algo_pool_id": { + "$exists": True, + "$ne": None, + } + }, + {"source": "derived_lp"}, + ] + } + ) + assert database.asset_prices.mongodb_collection.mock_calls.index( + call.delete_many( + { + "$or": [ + { + "tinyman_algo_pool_id": { + "$exists": True, + "$ne": None, + } + }, + {"source": "derived_lp"}, + ] + } + ) + ) < database.asset_prices.mongodb_collection.mock_calls.index( + call.aggregate( + [ + { + "$sort": { + "id": 1, + "observed_at": -1, + "updated": -1, + "_id": -1, + } + }, + { + "$group": { + "_id": "$id", + "count": {"$sum": 1}, + "keep_id": {"$first": "$_id"}, + "all_ids": {"$push": "$_id"}, + } + }, + {"$match": {"count": {"$gt": 1}}}, + ], + allowDiskUse=True, + ) + ) - database.lp_states.mongodb_collection.create_index.assert_called_once_with("token_id", name="token_id_idx") + assert database.lp_states.mongodb_collection.create_index.call_args_list == [ + call("token_id", unique=True, name="token_id_unique"), + call("address", unique=True, name="address_unique"), + ] database.pool_states.mongodb_collection.create_index.assert_called_once_with("pool_id", name="pool_id_idx") database.user_states.mongodb_collection.create_index.assert_called_once_with("address", name="address_idx") - database.lp_tokens.mongodb_collection.create_index.assert_called_once_with("id", name="lp_token_id_idx") + database.airdrop_rewards.mongodb_collection.create_index.assert_called_once_with( + "operation_id", + unique=True, + name="operation_id_unique", + partialFilterExpression={"operation_id": {"$type": "string"}}, + ) + database.sync_states.mongodb_collection.create_index.assert_called_once_with( + "id", + unique=True, + name="id_unique", + ) + + +def test_standalone_airdrop_indexes_fail_closed_on_duplicate_operations() -> None: + manifests = Mock() + manifests.aggregate.return_value = [] + rewards = Mock() + rewards.aggregate.return_value = [ + { + "_id": "airdrop:summer:wallet", + "count": 2, + "keep_id": "first", + "all_ids": ["first", "second"], + } + ] + + with pytest.raises(RuntimeError, match="duplicate 'operation_id'"): + ensure_airdrop_indexes( + _database( + airdrop_manifests=manifests, + airdrop_rewards=rewards, + ) + ) + + manifests.create_index.assert_called_once_with( + "id", + unique=True, + name="id_unique", + ) + rewards.create_index.assert_not_called() + rewards.delete_many.assert_not_called() + + +def test_legacy_lp_price_cleanup_reports_deleted_rows() -> None: + collection = Mock() + collection.delete_many.return_value = SimpleNamespace(deleted_count=3) + + removed = delete_unverified_legacy_lp_prices( + _database(asset_prices=collection), + ) + + assert removed == 3 + collection.delete_many.assert_called_once_with( + { + "$or": [ + { + "tinyman_algo_pool_id": { + "$exists": True, + "$ne": None, + } + }, + {"source": "derived_lp"}, + ] + } + ) + + +def test_single_legacy_sync_cursor_is_migrated_without_guessing_between_competitors() -> None: + collection = Mock() + collection.find.return_value = [ + { + "_id": "mongo-id", + "id": "legacy-random-id", + "last_round": 123, + } + ] + database = _database(sync_states=collection) + + ensure_sync_state_singleton(database) + + collection.update_one.assert_called_once_with( + {"_id": "mongo-id"}, + {"$set": {"id": "main"}}, + ) + collection.create_index.assert_called_once_with( + "id", + unique=True, + name="id_unique", + ) + + +def test_competing_sync_cursors_fail_closed() -> None: + collection = Mock() + collection.find.return_value = [ + {"_id": "a", "id": "a", "last_round": 100}, + {"_id": "b", "id": "b", "last_round": 101}, + ] + + with pytest.raises(RuntimeError, match="competing checkpoints"): + ensure_sync_state_singleton( + _database(sync_states=collection), + ) + + collection.update_one.assert_not_called() + collection.create_index.assert_not_called() def test_correctness_critical_index_failure_is_not_swallowed() -> None: database = _database() + database.airdrop_manifests.mongodb_collection.aggregate.return_value = [] + database.airdrop_rewards.mongodb_collection.aggregate.return_value = [] + database.assets.mongodb_collection.aggregate.return_value = [] database.asset_prices.mongodb_collection.aggregate.return_value = [] + database.asset_prices.mongodb_collection.delete_many.return_value = SimpleNamespace( + deleted_count=0, + ) database.asset_prices.mongodb_collection.create_index.side_effect = RuntimeError("index build failed") with pytest.raises(RuntimeError, match="index build failed"): diff --git a/tests/unit/test_financial_model_regressions.py b/tests/unit/test_financial_model_regressions.py new file mode 100644 index 00000000..e47f510a --- /dev/null +++ b/tests/unit/test_financial_model_regressions.py @@ -0,0 +1,83 @@ +import asyncio +from datetime import UTC, datetime +from types import SimpleNamespace + +import app as api_module +from flex.db.model.blockchain import AssetInfo +from flex.db.model.pools import StakingPool + + +def test_staking_pool_uses_reward_token_decimals() -> None: + now = datetime(2026, 1, 1, tzinfo=UTC) + pool = StakingPool( + description="different decimals", + address="POOL", + stake_token=AssetInfo( + id=1, + name="STAKE", + unit_name="STK", + decimals=6, + ), + reward_token=AssetInfo( + id=2, + name="REWARD", + unit_name="RWD", + decimals=2, + ), + reward_amount_micros=123, + algo_reward_amount_micros=0, + begin_block=1, + end_block=2, + lock_length_blocks=0, + deploy_date=now, + begin_date=now, + end_date=now, + id=3, + ) + + assert pool.to_info().reward_amount == 1.23 + + +def test_contract_sorting_does_not_mutate_cached_repository_list( + monkeypatch, +) -> None: + cached = [ + SimpleNamespace( + id=1, + end_date=None, + metadata={"cache": {"global": {"totalStaked": "0x1"}}}, + ), + SimpleNamespace( + id=2, + end_date=None, + metadata={"cache": {"global": {"totalStaked": "0x1"}}}, + ), + ] + monkeypatch.setattr( + api_module, + "get_contracts_by_type", + lambda contract_type: cached, + ) + + newest_first = asyncio.run( + api_module.get_contracts( + type=None, + max_count=None, + new_first=True, + without_old_pools=False, + include_address_pools=None, + ) + ) + original_order = asyncio.run( + api_module.get_contracts( + type=None, + max_count=None, + new_first=False, + without_old_pools=False, + include_address_pools=None, + ) + ) + + assert [contract.id for contract in newest_first] == [2, 1] + assert [contract.id for contract in original_order] == [1, 2] + assert [contract.id for contract in cached] == [1, 2] diff --git a/tests/unit/test_lp_api_contract.py b/tests/unit/test_lp_api_contract.py index 3fa09e11..c3dd11ef 100644 --- a/tests/unit/test_lp_api_contract.py +++ b/tests/unit/test_lp_api_contract.py @@ -10,7 +10,12 @@ from flex.db.model.priced import AssetPrice -def _lp_price(token_id: int, *, age_seconds: int = 0) -> AssetPrice: +def _lp_price( + token_id: int, + *, + age_seconds: int = 0, + source: str = "vestige", +) -> AssetPrice: observed_at = datetime.now(UTC) - timedelta(seconds=age_seconds) return AssetPrice( id=token_id, @@ -18,7 +23,7 @@ def _lp_price(token_id: int, *, age_seconds: int = 0) -> AssetPrice: price_algo=1.25 if token_id == 8 else 2.5, price_usd=0.42 if token_id == 8 else 0.84, last_update_round=123, - source="derived_lp", + source=source, observed_at=observed_at, created=observed_at, updated=observed_at, @@ -90,7 +95,7 @@ def get_many_by_query(self, query: dict) -> list[SimpleNamespace]: self.calls.append(query) return [ _lp_price(8), - _lp_price(13), + _lp_price(13, source="derived_lp"), ] asset_prices = AssetPrices() @@ -115,11 +120,7 @@ def get_many_by_query(self, query: dict) -> list[SimpleNamespace]: "token_price_usd": 0.42, }, "5": None, - "13": { - "token_id": 13, - "token_price_algo": 2.5, - "token_price_usd": 0.84, - }, + "13": None, }, } @@ -147,6 +148,29 @@ def test_lp_single_response_remains_backward_compatible(monkeypatch) -> None: } +def test_lp_endpoint_hides_retired_derived_price_without_pool_marker( + monkeypatch, +) -> None: + asset_prices = SimpleNamespace( + get_many_by_query=lambda query: [ + _lp_price(8, source="derived_lp"), + ], + ) + monkeypatch.setattr( + api, + "db", + SimpleNamespace(asset_prices=asset_prices), + ) + + response = asyncio.run( + api.handle_get_lp_state_priced( + api.LpStatePricedRequest(lp_token_id=8), + ), + ) + + assert response is None + + def test_lp_endpoint_hides_price_older_than_max_stale(monkeypatch) -> None: asset_prices = SimpleNamespace( get_many_by_query=lambda query: [ diff --git a/tests/unit/test_lp_projection_domain.py b/tests/unit/test_lp_projection_domain.py new file mode 100644 index 00000000..95549e54 --- /dev/null +++ b/tests/unit/test_lp_projection_domain.py @@ -0,0 +1,27 @@ +from flex.domain.lp_projection import lp_balance_delta + + +def test_algo_pool_fee_debits_the_economic_algo_reserve() -> None: + delta = lp_balance_delta( + token_id=99, + asset1_id=7, + asset2_id=0, + event_asset_id=0, + event_pool_delta_micros=-1_000, + ) + + assert delta.field == "asset2_reserve_micros" + assert delta.amount == -1_000 + + +def test_token_token_pool_fee_debits_the_operational_algo_balance() -> None: + delta = lp_balance_delta( + token_id=99, + asset1_id=7, + asset2_id=8, + event_asset_id=0, + event_pool_delta_micros=-1_000, + ) + + assert delta.field == "operational_algo_balance_micros" + assert delta.amount == -1_000 diff --git a/tests/unit/test_lp_projection_repository.py b/tests/unit/test_lp_projection_repository.py new file mode 100644 index 00000000..5f6f399b --- /dev/null +++ b/tests/unit/test_lp_projection_repository.py @@ -0,0 +1,697 @@ +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from datetime import UTC, datetime, timedelta +from threading import RLock +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from bson import BSON, Decimal128 + +from flex.db.lp_projection import ( + LpProjectionPersistenceError, + LpProjectionResult, + MongoLpProjectionRepository, +) +from flex.db.model.blockchain import SyncState +from flex.db.model.liquidity_pools import LpState, LpTransaction +from flex.db.sync_coordinator import ( + MongoSyncCoordinator, + SyncCoordinatorError, +) +from flex.domain.lp_projection import lp_event_order, lp_round_end_order + + +def _numeric(value): + return value.to_decimal() if isinstance(value, Decimal128) else value + + +def _matches(document: dict, query: dict) -> bool: + for field, expected in query.items(): + if field == "$or": + if not any(_matches(document, branch) for branch in expected): + return False + continue + if isinstance(expected, dict): + if "$exists" in expected: + if (field in document) is not expected["$exists"]: + return False + continue + actual = _numeric(document.get(field)) + if "$gte" in expected and actual < _numeric(expected["$gte"]): + return False + if "$lte" in expected and actual > _numeric(expected["$lte"]): + return False + if "$gt" in expected and actual <= _numeric(expected["$gt"]): + return False + if "$lt" in expected and actual >= _numeric(expected["$lt"]): + return False + continue + if document.get(field) != expected: + return False + return True + + +class AtomicCollection: + def __init__(self, documents: list[dict] | None = None) -> None: + self.documents = [deepcopy(document) for document in documents or []] + self.lock = RLock() + self.fail_next_upsert = False + + def find_one(self, query, projection=None): + with self.lock: + document = next((item for item in self.documents if _matches(item, query)), None) + if document is None: + return None + if projection is None: + return deepcopy(document) + return { + field: deepcopy(document[field]) + for field, include in projection.items() + if include and field in document + } + + def find_one_and_update( + self, + query, + update, + *, + upsert=False, + return_document=None, + ): + del return_document + with self.lock: + document = next((item for item in self.documents if _matches(item, query)), None) + if document is None and upsert: + if self.fail_next_upsert: + self.fail_next_upsert = False + raise RuntimeError("injected marker failure") + document = { + field: value + for field, value in query.items() + if not field.startswith("$") and not isinstance(value, dict) + } + document.update(deepcopy(update.get("$setOnInsert", {}))) + self.documents.append(document) + if document is None: + return None + for field, value in update.get("$inc", {}).items(): + document[field] = Decimal128(_numeric(document[field]) + _numeric(value)) + document.update(deepcopy(update.get("$set", {}))) + return deepcopy(document) + + def update_one(self, query, update): + with self.lock: + document = next((item for item in self.documents if _matches(item, query)), None) + if document is not None: + document.update(deepcopy(update.get("$set", {}))) + return SimpleNamespace(matched_count=int(document is not None)) + + +class BsonValidatingCollection(AtomicCollection): + def find_one_and_update(self, query, update, **kwargs): + BSON.encode( + { + "query": query, + "update": update, + } + ) + return super().find_one_and_update( + query, + update, + **kwargs, + ) + + +def _state( + *, + cursor: str | None = None, + last_round: int = 99, + reserve: int = 100, +) -> LpState: + return LpState( + id=1, + token_id=99, + asset1_id=7, + asset2_id=0, + dex_provider="tinyman", + address="POOL", + last_updated_round=last_round, + last_event_order=cursor, + asset1_reserve_micros=reserve, + asset2_reserve_micros=100, + total_tokens_micros=100, + asset1_reserve=0, + asset2_reserve=0, + total_tokens=0, + token_price_algo=0, + ) + + +def _transaction( + event_id: str = "TX@POOL", + *, + asa_id: int = 7, + amount: int = 10, + round_number: int = 100, + position: int = 1, +) -> LpTransaction: + return LpTransaction( + id=event_id, + pool_address="POOL", + user_address="USER", + asa_id=asa_id, + delta_amount_micros=amount, + confirmed_round=round_number, + event_position=position, + ) + + +def _repository( + state: LpState, + events: list[LpTransaction] | None = None, +) -> tuple[MongoLpProjectionRepository, AtomicCollection, AtomicCollection]: + states = AtomicCollection([state.to_dict()]) + markers = AtomicCollection([event.to_dict() for event in events or []]) + return ( + MongoLpProjectionRepository( + states=states, # type: ignore[arg-type] + events=markers, # type: ignore[arg-type] + ), + states, + markers, + ) + + +def test_crash_after_state_cas_repairs_marker_without_reapplying_delta() -> None: + transaction = _transaction() + repository, states, markers = _repository( + _state(cursor=lp_round_end_order(99)), + ) + markers.fail_next_upsert = True + + with pytest.raises(RuntimeError, match="injected marker"): + repository.project(transaction) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 110 + assert markers.documents == [] + + replay = repository.project(transaction) + + assert replay.result is LpProjectionResult.ALREADY_APPLIED + assert replay.state.asset1_reserve_micros == 110 + assert len(markers.documents) == 1 + + +def test_concurrent_replay_changes_the_balance_once() -> None: + transaction = _transaction() + repository, states, markers = _repository( + _state(cursor=lp_round_end_order(99)), + ) + + with ThreadPoolExecutor(max_workers=8) as executor: + outcomes = list(executor.map(repository.project, [transaction] * 20)) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 110 + assert len(markers.documents) == 1 + assert sum(outcome.result is LpProjectionResult.APPLIED for outcome in outcomes) == 1 + + +def test_replay_refreshes_stale_state_before_declaring_marker_divergence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + transaction = _transaction() + stale_state = _state(cursor=lp_round_end_order(99)) + repository, _, _ = _repository( + _state( + cursor=transaction.event_order, + last_round=100, + reserve=110, + ), + events=[transaction], + ) + original_get_state = MongoLpProjectionRepository._get_state + read_count = 0 + + def stale_then_current( + current_repository: MongoLpProjectionRepository, + pool_address: str, + ) -> LpState: + nonlocal read_count + read_count += 1 + if read_count == 1: + return stale_state + return original_get_state(current_repository, pool_address) + + monkeypatch.setattr( + MongoLpProjectionRepository, + "_get_state", + stale_then_current, + ) + + outcome = repository.project(transaction) + + assert outcome.result is LpProjectionResult.ALREADY_APPLIED + assert outcome.state.asset1_reserve_micros == 110 + assert read_count == 2 + + +def test_later_event_cannot_hide_an_unrecorded_earlier_event() -> None: + later = _transaction("Z@POOL", position=2) + earlier = _transaction("A@POOL", position=1) + repository, states, _ = _repository( + _state(cursor=lp_round_end_order(99)), + ) + + repository.project(later) + + with pytest.raises(LpProjectionPersistenceError, match="advanced past"): + repository.project(earlier) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 110 + + +def test_snapshot_cursor_covers_legacy_events_without_guessing_markers() -> None: + transaction = _transaction(round_number=100) + repository, states, markers = _repository( + _state(cursor=lp_round_end_order(100), last_round=100), + ) + + outcome = repository.project(transaction) + + assert outcome.result is LpProjectionResult.SNAPSHOT_COVERED + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 100 + assert markers.documents == [] + + +def test_legacy_state_without_cutover_snapshot_fails_closed() -> None: + repository, states, markers = _repository( + _state(cursor=None, last_round=100), + ) + + with pytest.raises(LpProjectionPersistenceError, match="authoritative snapshot"): + repository.project(_transaction(round_number=101)) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 100 + assert markers.documents == [] + + +def test_projection_rejects_underflow_and_preserves_cursor() -> None: + initial_cursor = lp_round_end_order(99) + transaction = _transaction(amount=-101) + repository, states, _ = _repository( + _state(cursor=initial_cursor), + ) + + with pytest.raises(LpProjectionPersistenceError, match="underflow or overflow"): + repository.project(transaction) + + stored = LpState.from_dict(states.documents[0]) + assert stored.asset1_reserve_micros == 100 + assert stored.last_event_order == initial_cursor + + +def test_existing_event_id_with_different_payload_fails_closed() -> None: + expected = _transaction(amount=10) + conflicting = _transaction(amount=11) + repository, states, _ = _repository( + _state(cursor=lp_round_end_order(99)), + events=[conflicting], + ) + + with pytest.raises(LpProjectionPersistenceError, match="different immutable data"): + repository.project(expected) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 100 + + +def test_existing_marker_cannot_advance_a_state_without_its_delta() -> None: + transaction = _transaction() + initial_cursor = lp_round_end_order(99) + repository, states, _ = _repository( + _state(cursor=initial_cursor), + events=[transaction], + ) + + with pytest.raises( + LpProjectionPersistenceError, + match="recorded but state", + ): + repository.project(transaction) + + stored = LpState.from_dict(states.documents[0]) + assert stored.asset1_reserve_micros == 100 + assert stored.last_event_order == initial_cursor + + +def test_uint64_amounts_use_decimal128_without_precision_loss() -> None: + amount = 2**64 - 2 + repository, states, _ = _repository( + _state( + cursor=lp_round_end_order(99), + reserve=1, + ), + ) + + outcome = repository.project( + _transaction(amount=amount), + ) + + assert outcome.state.asset1_reserve_micros == 2**64 - 1 + assert isinstance(states.documents[0]["asset1_reserve_micros"], Decimal128) + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 2**64 - 1 + + +def test_uint64_identifiers_and_cursors_are_bson_safe() -> None: + maximum = 2**64 - 1 + transaction = LpTransaction( + id="TX@POOL", + pool_address="POOL", + user_address="USER", + asa_id=maximum, + delta_amount_micros=maximum, + confirmed_round=maximum, + event_position=maximum, + ) + + encoded = BSON.encode(transaction.to_dict()) + restored = LpTransaction.from_dict(BSON(encoded).decode()) + + assert restored.asa_id == maximum + assert restored.delta_amount_micros == maximum + assert restored.confirmed_round == maximum + assert restored.event_position == maximum + + +def test_operational_algo_balance_is_bson_safe_and_backward_compatible() -> None: + maximum = 2**64 - 1 + state = _state() + state.operational_algo_balance_micros = maximum + + encoded = BSON.encode(state.to_dict()) + restored = LpState.from_dict(BSON(encoded).decode()) + + assert restored.operational_algo_balance_micros == maximum + + legacy_payload = state.to_dict() + legacy_payload.pop("operational_algo_balance_micros") + restored_legacy = LpState.from_dict(legacy_payload) + + assert restored_legacy.operational_algo_balance_micros == 0 + + +def test_uint64_repository_queries_and_updates_are_bson_safe() -> None: + maximum = 2**64 - 1 + state = _state( + cursor=lp_round_end_order(maximum - 1), + last_round=maximum - 1, + ) + state.token_id = maximum + states = BsonValidatingCollection([state.to_dict()]) + markers = BsonValidatingCollection() + repository = MongoLpProjectionRepository( + states=states, # type: ignore[arg-type] + events=markers, # type: ignore[arg-type] + ) + + outcome = repository.project( + _transaction( + amount=1, + round_number=maximum, + position=maximum, + ), + ) + + assert outcome.result is LpProjectionResult.APPLIED + assert outcome.state.last_updated_round == maximum + BSON.encode( + LpState.encode_query( + { + "token_id": { + "$in": [maximum], + } + } + ) + ) + + +def test_token_token_pool_fee_updates_only_operational_algo_balance() -> None: + state = _state( + cursor=lp_round_end_order(99), + reserve=100, + ) + state.asset2_id = 8 + state.asset2_reserve_micros = 200 + state.operational_algo_balance_micros = 5_000 + repository, states, _ = _repository(state) + + outcome = repository.project( + _transaction( + event_id="TX#fee@POOL", + asa_id=0, + amount=-1_000, + ), + ) + + assert outcome.result is LpProjectionResult.APPLIED + assert outcome.state.asset1_reserve_micros == 100 + assert outcome.state.asset2_reserve_micros == 200 + assert outcome.state.operational_algo_balance_micros == 4_000 + persisted = LpState.from_dict(states.documents[0]) + assert persisted.operational_algo_balance_micros == 4_000 + assert isinstance( + states.documents[0]["operational_algo_balance_micros"], + Decimal128, + ) + + +def test_event_position_is_sorted_numerically_within_a_round() -> None: + assert lp_event_order(100, "TX-2", 2) < lp_event_order(100, "TX-10", 10) + + +def test_snapshot_cannot_overwrite_a_newer_event_cursor() -> None: + newer_cursor = lp_event_order(101, "TX@POOL", 1) + repository, states, _ = _repository( + _state(cursor=newer_cursor, last_round=101, reserve=150), + ) + + result = repository.replace_snapshot( + _state(cursor=None, last_round=100, reserve=90), + observed_round=100, + ) + + assert result.last_event_order == newer_cursor + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 150 + + +def test_balance_snapshot_does_not_publish_derived_price_fields() -> None: + observed_at = datetime(2026, 1, 1, tzinfo=UTC) + current = _state( + cursor=lp_round_end_order(100), + last_round=100, + reserve=100, + ) + current.asset1_reserve = 1.0 + current.asset2_reserve = 2.0 + current.total_tokens = 3.0 + current.token_price_algo = 4.0 + current.derived_observed_at = observed_at + current.asset2_id = 8 + current.operational_algo_balance_micros = 5_000 + repository, states, _ = _repository(current) + + incoming = _state(cursor=None, last_round=101, reserve=150) + incoming.asset1_reserve = 10.0 + incoming.asset2_reserve = 20.0 + incoming.total_tokens = 30.0 + incoming.token_price_algo = 40.0 + incoming.derived_observed_at = observed_at + timedelta(minutes=1) + incoming.asset2_id = 8 + incoming.operational_algo_balance_micros = 6_000 + + result = repository.replace_snapshot( + incoming, + observed_round=101, + ) + + assert result.asset1_reserve_micros == 150 + assert result.operational_algo_balance_micros == 6_000 + assert result.asset1_reserve == 1.0 + assert result.asset2_reserve == 2.0 + assert result.total_tokens == 3.0 + assert result.token_price_algo == 4.0 + assert result.derived_observed_at == observed_at + persisted = LpState.from_dict(states.documents[0]) + assert persisted.token_price_algo == 4.0 + assert persisted.derived_observed_at == observed_at + + +def test_same_round_snapshot_with_different_balances_fails_closed() -> None: + snapshot_cursor = lp_round_end_order(100) + repository, states, _ = _repository( + _state(cursor=snapshot_cursor, last_round=100, reserve=100), + ) + + with pytest.raises(LpProjectionPersistenceError, match="conflicts"): + repository.replace_snapshot( + _state(cursor=None, last_round=100, reserve=101), + observed_round=100, + ) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 100 + + +@pytest.mark.parametrize( + ("field_name", "invalid_value"), + [ + ("asset1_reserve_micros", -1), + ("asset2_reserve_micros", True), + ("total_tokens_micros", 2**64), + ("operational_algo_balance_micros", 1.5), + ], +) +def test_invalid_snapshot_is_rejected_before_mongo_write( + field_name: str, + invalid_value: object, +) -> None: + state = _state(cursor=None) + setattr(state, field_name, invalid_value) + states = SimpleNamespace( + find_one_and_update=Mock(), + ) + repository = MongoLpProjectionRepository( + states=states, # type: ignore[arg-type] + events=AtomicCollection(), # type: ignore[arg-type] + ) + + with pytest.raises(LpProjectionPersistenceError, match=field_name): + repository.replace_snapshot(state, observed_round=100) + + states.find_one_and_update.assert_not_called() + + +def test_event_marker_counterparty_is_immutable() -> None: + expected = _transaction() + conflicting = _transaction() + conflicting.user_address = "OTHER-USER" + repository, states, _ = _repository( + _state(cursor=lp_round_end_order(99)), + events=[conflicting], + ) + + with pytest.raises(LpProjectionPersistenceError, match="immutable data"): + repository.project(expected) + + assert LpState.from_dict(states.documents[0]).asset1_reserve_micros == 100 + + +def test_expired_round_lease_uses_fencing_owner_and_checkpoint_cas() -> None: + now = datetime(2026, 1, 1, tzinfo=UTC) + collection = AtomicCollection( + [ + SyncState( + last_round=99, + ).to_dict() + ] + ) + coordinator = MongoSyncCoordinator( + collection=collection, # type: ignore[arg-type] + lease_duration=timedelta(seconds=30), + ) + + first = coordinator.claim_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now, + ) + blocked = coordinator.claim_round( + owner="worker-b", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=1), + ) + takeover = coordinator.claim_round( + owner="worker-b", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=31), + ) + + assert first is not None + assert blocked is None + assert takeover is not None + with pytest.raises(SyncCoordinatorError, match="lost its claim"): + coordinator.complete_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=32), + ) + + completed = coordinator.complete_round( + owner="worker-b", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=32), + ) + + assert completed.last_round == 100 + + +def test_expired_round_lease_cannot_complete_without_takeover() -> None: + now = datetime(2026, 1, 1, tzinfo=UTC) + collection = AtomicCollection([SyncState(last_round=99).to_dict()]) + coordinator = MongoSyncCoordinator( + collection=collection, # type: ignore[arg-type] + lease_duration=timedelta(seconds=30), + ) + + claimed = coordinator.claim_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now, + ) + + assert claimed is not None + with pytest.raises(SyncCoordinatorError, match="lost its claim"): + coordinator.complete_round( + owner="worker-a", + expected_last_round=99, + round_number=100, + now=now + timedelta(seconds=31), + ) + assert SyncState.from_dict(collection.documents[0]).last_round == 99 + + +def test_uint64_sync_checkpoint_operations_are_bson_safe() -> None: + maximum = 2**64 - 1 + now = datetime(2026, 1, 1, tzinfo=UTC) + collection = BsonValidatingCollection( + [ + SyncState( + last_round=maximum - 1, + ).to_dict() + ] + ) + coordinator = MongoSyncCoordinator( + collection=collection, # type: ignore[arg-type] + ) + + claimed = coordinator.claim_round( + owner="worker", + expected_last_round=maximum - 1, + round_number=maximum, + now=now, + ) + completed = coordinator.complete_round( + owner="worker", + expected_last_round=maximum - 1, + round_number=maximum, + now=now, + ) + + assert claimed is not None + assert claimed.claimed_round == maximum + assert completed.last_round == maximum diff --git a/tests/unit/test_lp_token_identity.py b/tests/unit/test_lp_token_identity.py new file mode 100644 index 00000000..84a286b0 --- /dev/null +++ b/tests/unit/test_lp_token_identity.py @@ -0,0 +1,51 @@ +from types import SimpleNamespace + +import pytest + +from flex.data import lp_tokens +from flex.db.model.blockchain import LpToken + + +def _token(*, address: str = "POOL") -> LpToken: + return LpToken( + id=99, + pool_id=123, + asset1_id=7, + asset2_id=0, + address=address, + dex_provider="tinyman", + ) + + +def test_lp_token_registration_returns_matching_atomic_winner( + monkeypatch: pytest.MonkeyPatch, +) -> None: + candidate = _token() + manager = SimpleNamespace(get_or_create=lambda item: item) + monkeypatch.setattr( + lp_tokens, + "db", + SimpleNamespace(lp_tokens=manager), + ) + + assert lp_tokens.persist_lp_token(candidate) is candidate + + +def test_lp_token_registration_fails_closed_on_identity_conflict( + monkeypatch: pytest.MonkeyPatch, +) -> None: + candidate = _token() + manager = SimpleNamespace( + get_or_create=lambda item: _token(address="OTHER_POOL"), + ) + monkeypatch.setattr( + lp_tokens, + "db", + SimpleNamespace(lp_tokens=manager), + ) + + with pytest.raises( + lp_tokens.LpTokenIdentityConflictError, + match="different pool metadata", + ): + lp_tokens.persist_lp_token(candidate) diff --git a/tests/unit/test_lp_transaction_projection.py b/tests/unit/test_lp_transaction_projection.py index 10def713..79bbbbc6 100644 --- a/tests/unit/test_lp_transaction_projection.py +++ b/tests/unit/test_lp_transaction_projection.py @@ -1,21 +1,33 @@ import asyncio from types import SimpleNamespace +import pytest + from flex import sync_pools from flex.data import lp_states -from flex.domain.transactions import ASSET_TRANSFER_TX +from flex.db.lp_projection import ( + LpProjectionOutcome, + LpProjectionResult, +) +from flex.db.model.liquidity_pools import LpState, LpTransaction +from flex.domain.algorand import MAX_ALGORAND_UINT +from flex.domain.transactions import ASSET_TRANSFER_TX, PAYMENT_TX def test_lp_to_lp_transfer_updates_both_scoped_projections(monkeypatch) -> None: monkeypatch.setattr( sync_pools, - "get_all_lp_state_addresses", - lambda: {"POOL-A", "POOL-B"}, + "get_lp_tracked_assets_by_address", + lambda: { + "POOL-A": frozenset({0, 7, 99}), + "POOL-B": frozenset({0, 7, 98}), + }, ) raw_transaction = { "id": "TX", "sender": "POOL-A", "confirmed-round": 123, + "fee": 0, ASSET_TRANSFER_TX: { "asset-id": 7, "amount": 5, @@ -43,6 +55,7 @@ def test_lp_to_lp_transfer_updates_both_scoped_projections(monkeypatch) -> None: asset2_reserve_micros=100, total_tokens_micros=100, last_updated_round=0, + last_event_order=None, ), "POOL-B": SimpleNamespace( id=2, @@ -54,39 +67,601 @@ def test_lp_to_lp_transfer_updates_both_scoped_projections(monkeypatch) -> None: asset2_reserve_micros=200, total_tokens_micros=200, last_updated_round=0, + last_event_order=None, ), } - created = [] - updated = [] + + class FakeProjectionRepository: + def __init__(self) -> None: + self.projected = [] + + def project(self, transaction): + self.projected.append(transaction) + state = states[transaction.pool_address] + state.asset1_reserve_micros += transaction.delta_amount_micros + state.last_event_order = transaction.event_order + return LpProjectionOutcome( + state=state, + result=LpProjectionResult.APPLIED, + ) + + def get_state(self, pool_address): + return states[pool_address] + + def update_derived_fields(self, state, *, expected_cursor): + raise AssertionError("ledger projection must not persist derived prices") + + repository = FakeProjectionRepository() monkeypatch.setattr( lp_states, "db", SimpleNamespace( - lp_transactions=SimpleNamespace( - exists=lambda **kwargs: False, - create_many=lambda values: created.extend(values), - ), - lp_states=SimpleNamespace( - get_one=lambda **kwargs: states[kwargs["address"]], - update=updated.append, - ), + lp_transactions=SimpleNamespace(mongodb_collection=object()), + lp_states=SimpleNamespace(mongodb_collection=object()), ), ) + monkeypatch.setattr( + lp_states, + "MongoLpProjectionRepository", + lambda **kwargs: repository, + ) + + recalculation_calls = 0 - async def unchanged(state): - return state + async def forbidden_recalculation(state): + nonlocal recalculation_calls + recalculation_calls += 1 + raise AssertionError("ledger projection must not recalculate prices") + # Keep this sentinel even though the production symbol has been removed: + # the test fails if a future implementation reintroduces and calls it. monkeypatch.setattr( lp_states, "recalculate_lp_state_price_algo_with_micros", - unchanged, + forbidden_recalculation, + raising=False, ) result = asyncio.run( - lp_states.update_lp_states_with_transactions(projections), + lp_states.update_lp_states_with_transactions( + projections, + expected_round=123, + ), ) assert states["POOL-A"].asset1_reserve_micros == 95 assert states["POOL-B"].asset1_reserve_micros == 205 assert {state.address for state in result} == {"POOL-A", "POOL-B"} - assert [tx.id for tx in created] == ["TX@POOL-A", "TX@POOL-B"] + assert [tx.id for tx in repository.projected] == ["TX@POOL-A", "TX@POOL-B"] + assert recalculation_calls == 0 + + +def test_lp_self_transfer_has_zero_projection(monkeypatch) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": 0, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "POOL", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert projections == [] + + +def test_lp_asset_transfer_projects_pool_fee_as_separate_algo_debit( + monkeypatch, +) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "USER", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert [(tx.id, tx.asa_id, tx.delta_amount_micros) for tx in projections] == [ + ("TX@POOL", 7, -5), + ("TX#fee@POOL", 0, -1_000), + ] + + +def test_token_token_pool_fee_is_a_separately_projectable_algo_event( + monkeypatch, +) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 8, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "USER", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert [(tx.id, tx.asa_id, tx.delta_amount_micros) for tx in projections] == [ + ("TX@POOL", 7, -5), + ("TX#fee@POOL", 0, -1_000), + ] + + +def test_lp_payment_projects_amount_and_fee_as_distinct_algo_debits( + monkeypatch, +) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": 1_000, + PAYMENT_TX: { + "amount": 5, + "receiver": "USER", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert [(tx.id, tx.asa_id, tx.delta_amount_micros) for tx in projections] == [ + ("TX@POOL", 0, -5), + ("TX#fee@POOL", 0, -1_000), + ] + + +def test_lp_self_transfer_still_projects_pool_fee(monkeypatch) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "POOL", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert [(tx.id, tx.pool_address, tx.asa_id, tx.delta_amount_micros) for tx in projections] == [ + ("TX#fee@POOL", "POOL", 0, -1_000), + ] + + +def test_unrelated_asset_dust_cannot_block_lp_projection(monkeypatch) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 8, 99})}, + ) + raw_transaction = { + "id": "DUST", + "sender": "ATTACKER", + "confirmed-round": 123, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 666, + "amount": 1, + "receiver": "POOL", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert projections == [] + + +def test_unrelated_clawback_cannot_block_pool_fee_projection(monkeypatch) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 8, 99})}, + ) + raw_transaction = { + "id": "UNRELATED-CLAWBACK", + "sender": "POOL", + "confirmed-round": 123, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 666, + "amount": 1, + "sender": "VICTIM", + "receiver": "ATTACKER", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert [(transaction.id, transaction.asa_id, transaction.delta_amount_micros) for transaction in projections] == [ + ("UNRELATED-CLAWBACK#fee@POOL", 0, -1_000), + ] + + +@pytest.mark.parametrize( + "invalid_fee", + [None, -1, MAX_ALGORAND_UINT + 1, 1.5, True, "1000"], +) +def test_lp_projection_rejects_invalid_algorand_fees( + monkeypatch, + invalid_fee, +) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": invalid_fee, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "USER", + }, + } + + with pytest.raises(ValueError, match="fee must be an Algorand uint64"): + asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + +@pytest.mark.parametrize( + ("field", "invalid_value"), + [ + ("amount", True), + ("amount", -1), + ("amount", MAX_ALGORAND_UINT + 1), + ("amount", "5"), + ("asset-id", True), + ("asset-id", 0), + ("asset-id", MAX_ALGORAND_UINT + 1), + ], +) +def test_lp_projection_rejects_invalid_asset_transfer_uint64_fields( + monkeypatch, + field: str, + invalid_value: object, +) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + transfer = { + "asset-id": 7, + "amount": 5, + "receiver": "USER", + } + transfer[field] = invalid_value + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": 123, + "fee": 1_000, + ASSET_TRANSFER_TX: transfer, + } + + with pytest.raises(ValueError, match=field): + asyncio.run(sync_pools.process_lp_transactions([raw_transaction])) + + +@pytest.mark.parametrize( + "invalid_round", + [True, -1, MAX_ALGORAND_UINT + 1, 1.5, "123", None], +) +def test_lp_projection_rejects_invalid_confirmed_round_before_negation( + monkeypatch, + invalid_round: object, +) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "POOL", + "confirmed-round": invalid_round, + "fee": 1_000, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "USER", + }, + } + + with pytest.raises(ValueError, match="confirmed-round"): + asyncio.run(sync_pools.process_lp_transactions([raw_transaction])) + + +def test_lp_sync_module_has_no_raw_balance_price_publisher() -> None: + assert not hasattr(sync_pools, "update_asset_prices") + assert not hasattr(sync_pools, "create_and_update_asset_prices") + + +def test_token_token_snapshot_tracks_operational_algo_without_repricing( + monkeypatch, +) -> None: + state = LpState( + id=1, + address="POOL", + token_id=99, + asset1_id=7, + asset2_id=8, + dex_provider="tinyman", + last_updated_round=122, + asset1_reserve_micros=1, + asset2_reserve_micros=2, + total_tokens_micros=3, + operational_algo_balance_micros=4, + asset1_reserve=1.0, + asset2_reserve=2.0, + total_tokens=3.0, + token_price_algo=12.5, + ) + + async def snapshot(address, *, include_algo): + assert address == "POOL" + assert include_algo is True + return SimpleNamespace( + balances={0: 5_000, 7: 100, 8: 200, 99: 20}, + observed_round=123, + ) + + async def total_supply(asset_id): + assert asset_id == 99 + return 100 + + class FakeRepository: + def replace_snapshot(self, requested_state, *, observed_round): + assert observed_round == 123 + return requested_state + + monkeypatch.setattr(lp_states, "get_address_asset_snapshot", snapshot) + monkeypatch.setattr(lp_states, "get_asset_total_supply", total_supply) + monkeypatch.setattr( + lp_states, + "db", + SimpleNamespace( + lp_transactions=SimpleNamespace(mongodb_collection=object()), + lp_states=SimpleNamespace(mongodb_collection=object()), + ), + ) + monkeypatch.setattr( + lp_states, + "MongoLpProjectionRepository", + lambda **kwargs: FakeRepository(), + ) + + updated = asyncio.run(lp_states.update_lp_state(state)) + + assert state.asset1_reserve_micros == 1 + assert state.asset2_reserve_micros == 2 + assert state.total_tokens_micros == 3 + assert state.operational_algo_balance_micros == 4 + assert updated.asset1_reserve_micros == 100 + assert updated.asset2_reserve_micros == 200 + assert updated.total_tokens_micros == 80 + assert updated.operational_algo_balance_micros == 5_000 + assert updated.token_price_algo == 12.5 + + +def test_lp_projection_rejects_clawback_semantics(monkeypatch) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "CLAWBACK", + "confirmed-round": 123, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "POOL", + "sender": "VICTIM", + }, + } + + with pytest.raises(ValueError, match="clawback/close"): + asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + +def test_lp_projection_ignores_unrelated_clawback(monkeypatch) -> None: + monkeypatch.setattr( + sync_pools, + "get_lp_tracked_assets_by_address", + lambda: {"POOL": frozenset({0, 7, 99})}, + ) + raw_transaction = { + "id": "TX", + "sender": "CLAWBACK", + "confirmed-round": 123, + ASSET_TRANSFER_TX: { + "asset-id": 7, + "amount": 5, + "receiver": "OTHER", + "sender": "VICTIM", + }, + } + + projections = asyncio.run( + sync_pools.process_lp_transactions([raw_transaction]), + ) + + assert projections == [] + + +def test_lp_batch_preflight_rejects_conflicts_before_projection() -> None: + original = LpTransaction( + id="TX@POOL", + pool_address="POOL", + user_address="USER", + asa_id=7, + delta_amount_micros=5, + confirmed_round=123, + event_position=1, + ) + conflicting = LpTransaction( + id="TX@POOL", + pool_address="POOL", + user_address="USER", + asa_id=7, + delta_amount_micros=6, + confirmed_round=123, + event_position=1, + ) + + with pytest.raises( + lp_states.LpProjectionPersistenceError, + match="conflicting data", + ): + lp_states._preflight_lp_transactions( + [original, conflicting], + expected_round=123, + ) + + +def test_lp_batch_preflight_deduplicates_exact_replay() -> None: + transaction = LpTransaction( + id="TX@POOL", + pool_address="POOL", + user_address="USER", + asa_id=7, + delta_amount_micros=5, + confirmed_round=123, + event_position=1, + ) + + canonical = lp_states._preflight_lp_transactions( + [transaction, transaction], + expected_round=123, + ) + + assert canonical == [transaction] + + +def test_snapshot_checkpoint_uses_slowest_indexer_observation() -> None: + states = [ + SimpleNamespace( + last_updated_round=105, + last_event_order="00000000000000000105:~", + ), + SimpleNamespace( + last_updated_round=102, + last_event_order="00000000000000000102:~", + ), + ] + + checkpoint = sync_pools._snapshot_checkpoint_round( + previous_round=100, + node_round=110, + lp_states=states, + ) + + assert checkpoint == 102 + + +def test_snapshot_checkpoint_without_lp_states_uses_node_round() -> None: + checkpoint = sync_pools._snapshot_checkpoint_round( + previous_round=100, + node_round=110, + lp_states=[], + ) + + assert checkpoint == 110 + + +def test_snapshot_checkpoint_rejects_indexer_regression() -> None: + states = [ + SimpleNamespace( + last_updated_round=99, + last_event_order="00000000000000000099:~", + ) + ] + + with pytest.raises( + sync_pools.SyncCoordinatorError, + match="Indexer snapshots lag", + ): + sync_pools._snapshot_checkpoint_round( + previous_round=100, + node_round=110, + lp_states=states, + ) + + +def test_snapshot_checkpoint_does_not_skip_rest_of_partial_round() -> None: + states = [ + SimpleNamespace( + last_updated_round=101, + last_event_order=("00000000000000000101:00000000000000000002:TX@POOL"), + ) + ] + + checkpoint = sync_pools._snapshot_checkpoint_round( + previous_round=100, + node_round=110, + lp_states=states, + ) + + assert checkpoint == 100 diff --git a/tests/unit/test_nft_lottery_payouts.py b/tests/unit/test_nft_lottery_payouts.py new file mode 100644 index 00000000..923a2f0b --- /dev/null +++ b/tests/unit/test_nft_lottery_payouts.py @@ -0,0 +1,728 @@ +from copy import deepcopy +from hashlib import sha256 +from types import SimpleNamespace + +import pytest +from pymongo.errors import OperationFailure + +from flex.application.asset_transfers import AssetTransferReceipt + + +def _matches(document: dict, query: dict) -> bool: + for field, expected in query.items(): + if field == "$or": + if not any(_matches(document, branch) for branch in expected): + return False + continue + if isinstance(expected, dict): + if "$exists" in expected: + if (field in document) is not expected["$exists"]: + return False + continue + if "$ne" in expected: + if document.get(field) == expected["$ne"]: + return False + continue + if "$in" in expected: + if document.get(field) not in expected["$in"]: + return False + continue + if "$eq" in expected: + if document.get(field) != expected["$eq"]: + return False + continue + if document.get(field) != expected: + return False + return True + + +class FakeLotteryCollection: + def __init__( + self, + documents: list[dict], + *, + index_information: dict[str, dict] | None = None, + ) -> None: + self.documents = documents + self.indexes: list[tuple[tuple, dict]] = [] + self.dropped_indexes: list[str] = [] + self.index_events: list[tuple[str, str]] = [] + self.find_one_and_update_calls: list[tuple[dict, dict]] = [] + self._index_information = deepcopy( + index_information + or { + "_id_": { + "key": [("_id", 1)], + } + } + ) + + def aggregate(self, pipeline, **kwargs): + del kwargs + matching_ids: dict[str, list[object]] = {} + for document in self.documents: + draw_id = document.get("id") + if isinstance(draw_id, str) and draw_id: + matching_ids.setdefault(draw_id, []).append(document.get("_id")) + return [ + { + "_id": draw_id, + "count": len(document_ids), + "document_ids": document_ids, + } + for draw_id, document_ids in matching_ids.items() + if len(document_ids) > 1 + ][: pipeline[-1].get("$limit", 10)] + + def index_information(self): + return deepcopy(self._index_information) + + def create_index(self, *args, **kwargs) -> str: + self.indexes.append((args, kwargs)) + index_name = kwargs["name"] + self._index_information[index_name] = { + "key": [(args[0], 1)], + "unique": kwargs.get("unique", False), + "partialFilterExpression": deepcopy(kwargs.get("partialFilterExpression")), + } + self.index_events.append(("create", index_name)) + return index_name + + def drop_index(self, name: str) -> None: + self.dropped_indexes.append(name) + self._index_information.pop(name) + self.index_events.append(("drop", name)) + + def find(self, query): + return [deepcopy(document) for document in self.documents if _matches(document, query)] + + def find_one(self, query): + document = next((item for item in self.documents if _matches(item, query)), None) + return deepcopy(document) if document is not None else None + + def find_one_and_update(self, query, update, **kwargs): + self.find_one_and_update_calls.append((deepcopy(query), deepcopy(update))) + document = next((item for item in self.documents if _matches(item, query)), None) + if document is None: + return None + document.update(update.get("$set", {})) + return deepcopy(document) + + def update_one(self, query, update): + document = next((item for item in self.documents if _matches(item, query)), None) + if document is not None: + document.update(update.get("$set", {})) + matched_count = int(document is not None) + return SimpleNamespace( + matched_count=matched_count, + modified_count=matched_count, + ) + + +class ClaimBeforePrepareCollection(FakeLotteryCollection): + def update_one(self, query, update): + if "payout_operation_id" in query: + self.documents[0].update( + { + "claimed": True, + "payout_operation_id": "nft:manual-reconciliation", + "payout_txid": "manual-chain-tx", + "payout_status": "confirmed", + } + ) + return super().update_one(query, update) + + +class ConcurrentLegacyDropCollection(FakeLotteryCollection): + def drop_index(self, name: str) -> None: + self._index_information.pop(name, None) + self.index_events.append(("concurrent_drop", name)) + raise OperationFailure("index not found", code=27) + + +def _legacy_draw_id(document_id: str) -> str: + return f"legacy-{sha256(f'lottery-draw:{document_id}'.encode()).hexdigest()}" + + +def _receipt(asset_id: int, idempotency_key: str) -> AssetTransferReceipt: + return AssetTransferReceipt( + operation_id=f"nft:{idempotency_key}", + txid=f"tx-{asset_id}", + confirmed_round=777, + already_confirmed=False, + ) + + +def test_new_lottery_draw_starts_with_pending_payout(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection([]) + manager = SimpleNamespace( + collection=collection, + create=lambda draw: draw, + ) + monkeypatch.setattr(nft_lottery, "lottery_draws", manager) + + draw = nft_lottery._create_draw( + lottery_name="summer", + prize=101, + wallet="WALLET-A", + timestamp=1.0, + ) + + assert draw.id + assert draw.payout_status == nft_lottery.LotteryPayoutStatus.PENDING + assert collection.indexes == [ + ( + ("id",), + { + "unique": True, + "name": nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME, + "partialFilterExpression": { + "id": { + "$type": "string", + "$gt": "", + } + }, + }, + ) + ] + + +def test_lottery_index_upgrade_builds_replacement_before_dropping_legacy( + monkeypatch, +) -> None: + from api import nft_lottery + + documents = [ + {"_id": "legacy-a"}, + {"_id": "legacy-b", "id": ""}, + {"_id": "current", "id": "draw-current"}, + ] + collection = FakeLotteryCollection( + deepcopy(documents), + index_information={ + "_id_": {"key": [("_id", 1)]}, + "id_unique": { + "key": [("id", 1)], + "unique": True, + "partialFilterExpression": {"id": {"$type": "string"}}, + }, + }, + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + + nft_lottery.ensure_lottery_indexes() + + assert collection.documents == documents + assert collection.index_events == [ + ("create", nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME), + ("drop", "id_unique"), + ] + replacement = collection.index_information()[nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME] + assert replacement["partialFilterExpression"] == nft_lottery.LOTTERY_DRAW_ID_FILTER + + +def test_lottery_index_upgrade_preserves_duplicate_evidence_and_fails_closed( + monkeypatch, +) -> None: + from api import nft_lottery + + documents = [ + {"_id": "mongo-a", "id": "duplicate"}, + {"_id": "mongo-b", "id": "duplicate"}, + ] + collection = FakeLotteryCollection(deepcopy(documents)) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + + with pytest.raises(RuntimeError, match="preserved 1 duplicate group"): + nft_lottery.ensure_lottery_indexes() + + assert collection.documents == documents + assert collection.indexes == [] + assert collection.dropped_indexes == [] + + +def test_lottery_index_upgrade_converges_after_concurrent_legacy_drop( + monkeypatch, +) -> None: + from api import nft_lottery + + collection = ConcurrentLegacyDropCollection( + [{"_id": "current", "id": "draw-current"}], + index_information={ + "_id_": {"key": [("_id", 1)]}, + "id_unique": { + "key": [("id", 1)], + "unique": True, + "partialFilterExpression": {"id": {"$type": "string"}}, + }, + }, + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + + nft_lottery.ensure_lottery_indexes() + + replacement = collection.index_information()[nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME] + assert replacement["partialFilterExpression"] == nft_lottery.LOTTERY_DRAW_ID_FILTER + assert collection.index_events == [ + ("create", nft_lottery.LOTTERY_DRAW_ID_INDEX_NAME), + ("concurrent_drop", "id_unique"), + ] + + +def test_legacy_lottery_payouts_require_reconciliation_and_never_send(monkeypatch) -> None: + # Import after test collection helpers so module-level infrastructure stays + # outside the behavioral assertion. + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "draw-a", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + }, + { + "_id": "draw-b", + "wallet": "WALLET-B", + "prize": 202, + "timestamp": 2.0, + "lottery_name": "summer", + "claimed": False, + "payout_txid": "legacy-chain-tx", + }, + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 0 + assert result["error_count"] == 2 + assert calls == [] + assert not any(document["claimed"] for document in collection.documents) + assert all(document["id"].startswith("legacy-") for document in collection.documents) + assert all( + document["payout_status"] == nft_lottery.LotteryPayoutStatus.RECONCILIATION_REQUIRED + for document in collection.documents + ) + assert collection.documents[1]["payout_txid"] == "legacy-chain-tx" + + retry = nft_lottery.send_all_prizes() + + assert retry["sent_count"] == 0 + assert retry["error_count"] == 2 + assert calls == [] + + +def test_existing_draw_without_payout_state_is_legacy_and_fails_closed(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "mongo-a", + "id": "pre-status-draw", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 0 + assert result["error_count"] == 1 + assert calls == [] + assert collection.documents[0]["payout_status"] == nft_lottery.LotteryPayoutStatus.RECONCILIATION_REQUIRED + + +def test_empty_legacy_identity_and_status_are_migrated_fail_closed(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "mongo-a", + "id": "", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + "payout_status": "", + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + monkeypatch.setattr( + nft_lottery, + "send_nft", + lambda *args, **kwargs: pytest.fail("legacy draw must not broadcast"), + ) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 0 + assert result["error_count"] == 1 + assert collection.documents[0]["id"].startswith("legacy-") + assert collection.documents[0]["payout_status"] == nft_lottery.LotteryPayoutStatus.RECONCILIATION_REQUIRED + + +def test_missing_claimed_is_atomically_normalized_and_fails_closed(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "mongo-a", + "id": "draw-a", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + monkeypatch.setattr( + nft_lottery, + "send_nft", + lambda *args, **kwargs: pytest.fail("uncertain legacy draw must not broadcast"), + ) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 0 + assert result["error_count"] == 1 + assert collection.documents[0]["claimed"] is False + assert collection.documents[0]["payout_status"] == nft_lottery.LotteryPayoutStatus.RECONCILIATION_REQUIRED + migration_query, migration_update = collection.find_one_and_update_calls[0] + assert migration_query["claimed"] == {"$exists": False} + assert migration_update["$set"] == { + "id": "draw-a", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.RECONCILIATION_REQUIRED, + } + + +def test_null_claimed_with_exact_durable_operation_is_safe_to_resume(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "mongo-a", + "id": "draw-a", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": None, + "payout_operation_id": "nft:lottery:draw-a", + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 1 + assert result["error_count"] == 0 + assert calls == [("WALLET-A", 101, "lottery:draw-a")] + assert collection.documents[0]["claimed"] is True + assert collection.documents[0]["payout_status"] == nft_lottery.LotteryPayoutStatus.CONFIRMED + migration_query, migration_update = collection.find_one_and_update_calls[0] + assert migration_query["claimed"] == {"$eq": None} + assert migration_update["$set"] == { + "id": "draw-a", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.PREPARED, + } + + +def test_malformed_legacy_draw_does_not_block_safe_payouts(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "malformed", + "prize": 101, + "claimed": False, + }, + { + "_id": "mongo-b", + "id": "draw-b", + "wallet": "WALLET-B", + "prize": 202, + "timestamp": 2.0, + "lottery_name": "summer", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + }, + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 1 + assert result["error_count"] == 1 + assert calls == [("WALLET-B", 202, "lottery:draw-b")] + assert "migration failed" in result["results"][0]["error"] + assert collection.documents[1]["claimed"] is True + + +def test_legacy_lottery_payout_with_matching_operation_is_safe_to_resume(monkeypatch) -> None: + from api import nft_lottery + + document_id = "draw-a" + legacy_id = _legacy_draw_id(document_id) + collection = FakeLotteryCollection( + [ + { + "_id": document_id, + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + "payout_operation_id": f"nft:lottery:{legacy_id}", + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 1 + assert result["error_count"] == 0 + assert calls == [("WALLET-A", 101, f"lottery:{legacy_id}")] + assert collection.documents[0]["claimed"] is True + assert collection.documents[0]["payout_status"] == nft_lottery.LotteryPayoutStatus.CONFIRMED + + +def test_failed_new_lottery_payout_becomes_unresolved(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "mongo-a", + "id": "draw-a", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + + def fail_send_nft(address: str, asset_id: int, *, idempotency_key: str): + raise RuntimeError("indexer unavailable") + + monkeypatch.setattr(nft_lottery, "send_nft", fail_send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 0 + assert result["error_count"] == 1 + assert collection.documents[0]["payout_operation_id"] == "nft:lottery:draw-a" + assert collection.documents[0]["payout_status"] == nft_lottery.LotteryPayoutStatus.UNRESOLVED + assert collection.documents[0]["send_error"] == "indexer unavailable" + + +def test_claim_won_before_reservation_never_broadcasts_again(monkeypatch) -> None: + from api import nft_lottery + + collection = ClaimBeforePrepareCollection( + [ + { + "_id": "mongo-a", + "id": "draw-a", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + } + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 0 + assert result["error_count"] == 0 + assert result["results"][0]["already_claimed"] is True + assert result["results"][0]["txid"] == "manual-chain-tx" + assert calls == [] + + +def test_new_lottery_payouts_use_stable_ids_and_claim_exact_documents(monkeypatch) -> None: + from api import nft_lottery + + collection = FakeLotteryCollection( + [ + { + "_id": "mongo-a", + "id": "draw-a", + "wallet": "WALLET-A", + "prize": 101, + "timestamp": 1.0, + "lottery_name": "summer", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + }, + { + "_id": "mongo-b", + "id": "draw-b", + "wallet": "WALLET-B", + "prize": 202, + "timestamp": 2.0, + "lottery_name": "summer", + "claimed": False, + "payout_status": nft_lottery.LotteryPayoutStatus.PENDING, + }, + ] + ) + monkeypatch.setattr( + nft_lottery, + "lottery_draws", + SimpleNamespace(collection=collection), + ) + calls: list[tuple[str, int, str]] = [] + + def send_nft(address: str, asset_id: int, *, idempotency_key: str): + calls.append((address, asset_id, idempotency_key)) + return _receipt(asset_id, idempotency_key) + + monkeypatch.setattr(nft_lottery, "send_nft", send_nft) + + result = nft_lottery.send_all_prizes() + + assert result["sent_count"] == 2 + assert result["error_count"] == 0 + assert calls == [ + ("WALLET-A", 101, "lottery:draw-a"), + ("WALLET-B", 202, "lottery:draw-b"), + ] + assert [document["payout_txid"] for document in collection.documents] == [ + "tx-101", + "tx-202", + ] + assert all(document["claimed"] for document in collection.documents) + assert all( + document["payout_status"] == nft_lottery.LotteryPayoutStatus.CONFIRMED for document in collection.documents + ) + + first_calls = list(calls) + retry = nft_lottery.send_all_prizes() + + assert retry["sent_count"] == 0 + assert calls == first_calls diff --git a/tests/unit/test_price_background.py b/tests/unit/test_price_background.py index 85ad41df..f0219905 100644 --- a/tests/unit/test_price_background.py +++ b/tests/unit/test_price_background.py @@ -4,46 +4,42 @@ import pytest from api import background -from flex.data import lp_prices +from flex.data import lp_registry -def test_empty_asset_catalog_still_updates_lp_prices(monkeypatch) -> None: - monkeypatch.setattr(background.settings, "background_asset_prices_update", True) - monkeypatch.setattr( - background, - "db", - SimpleNamespace( - assets=SimpleNamespace(get_all=lambda: []), - asset_prices=SimpleNamespace(get_all=lambda: []), - ), +def _empty_price_database() -> SimpleNamespace: + return SimpleNamespace( + assets=SimpleNamespace(get_all=lambda: []), + asset_prices=SimpleNamespace(get_all=lambda: []), ) + + +def test_retired_lp_price_flag_cannot_enable_a_publisher(monkeypatch) -> None: + monkeypatch.setattr(background.settings, "background_asset_prices_update", True) + monkeypatch.setattr(background.settings, "background_lp_prices_update", True) + monkeypatch.setattr(background, "db", _empty_price_database()) monkeypatch.setattr(background, "get_current_round", lambda: 321) async def no_lp_definitions() -> list[dict]: return [] - updated_rounds: list[int] = [] - - async def record_lp_update(current_round: int) -> None: - updated_rounds.append(current_round) - monkeypatch.setattr( background, "get_lp_token_definitions", no_lp_definitions, ) - monkeypatch.setattr(background, "update_lp_token_prices", record_lp_update) one_shot = background.update_asset_prices_background.__wrapped__.__wrapped__ asyncio.run(one_shot()) - assert updated_rounds == [321] + assert not hasattr(background, "update_lp_token_prices") -def test_lp_registry_failure_cannot_overwrite_lp_with_external_price( +def test_lp_registry_failure_keeps_generic_refresh_fail_closed( monkeypatch, ) -> None: monkeypatch.setattr(background.settings, "background_asset_prices_update", True) + monkeypatch.setattr(background.settings, "background_lp_prices_update", False) monkeypatch.setattr( background, "db", @@ -62,11 +58,6 @@ async def failed_lp_registry() -> list[dict]: async def unexpected_asset_refresh(*args, **kwargs): raise AssertionError("regular asset refresh must fail closed") - updated_rounds: list[int] = [] - - async def record_lp_update(current_round: int) -> None: - updated_rounds.append(current_round) - monkeypatch.setattr( background, "get_lp_token_definitions", @@ -77,29 +68,38 @@ async def record_lp_update(current_round: int) -> None: "create_asset_price", unexpected_asset_refresh, ) - monkeypatch.setattr(background, "update_lp_token_prices", record_lp_update) one_shot = background.update_asset_prices_background.__wrapped__.__wrapped__ asyncio.run(one_shot()) - assert updated_rounds == [321] - -def test_incomplete_lp_registry_fails_closed(monkeypatch) -> None: - contract = SimpleNamespace( - metadata={ +@pytest.mark.parametrize( + "metadata", + [ + { "stake_token_id": 999, "asset1_id": 7, "asset2_id": 0, }, + { + "stake_token_id": 999, + }, + ], +) +def test_incomplete_lp_registry_fails_closed( + monkeypatch, + metadata: dict, +) -> None: + contract = SimpleNamespace( + metadata=metadata, ) monkeypatch.setattr( - lp_prices, + lp_registry, "get_contracts_by_type", lambda contract_type: [contract], ) monkeypatch.setattr( - lp_prices, + lp_registry, "db", SimpleNamespace( lp_tokens=SimpleNamespace( @@ -110,7 +110,7 @@ def test_incomplete_lp_registry_fails_closed(monkeypatch) -> None: ) with pytest.raises( - lp_prices.LpTokenRegistryError, - match=r"incomplete.*999", + lp_registry.LpTokenRegistryError, + match=r"classification is incomplete.*999", ): - asyncio.run(lp_prices.get_lp_token_definitions.__wrapped__()) + asyncio.run(lp_registry.get_lp_token_definitions.__wrapped__()) diff --git a/tests/unit/test_price_router.py b/tests/unit/test_price_router.py index a312dfcf..78726a1f 100644 --- a/tests/unit/test_price_router.py +++ b/tests/unit/test_price_router.py @@ -1,26 +1,32 @@ import asyncio from datetime import UTC, datetime, timedelta -from decimal import Decimal from types import SimpleNamespace import httpx import pytest from env import settings -from flex.data import lp_prices -from flex.domain.pricing import PriceQuote, PriceSource, PriceUnavailableError +from flex.domain.pricing import PriceSource, PriceUnavailableError from flex.meta_error import MetaError from flex.providers import price_router, vestige from flex.providers.vestige import Price -def _record(*, age_seconds: int, algo: float = 2.0, usd: float = 0.5): +def _record( + *, + age_seconds: int, + algo: float = 2.0, + usd: float = 0.5, + tinyman_algo_pool_id: int | None = None, + source: str = PriceSource.VESTIGE.value, +): observed_at = datetime.now(UTC) - timedelta(seconds=age_seconds) return SimpleNamespace( id=42, price_algo=algo, price_usd=usd, - source=PriceSource.VESTIGE.value, + source=source, + tinyman_algo_pool_id=tinyman_algo_pool_id, observed_at=observed_at, updated=observed_at, ) @@ -127,71 +133,67 @@ def test_provider_outage_uses_only_bounded_stale_database_price(monkeypatch) -> assert result == Price(algo=2.0, usd=0.5) -def test_bounded_stale_quote_preserves_original_provenance(monkeypatch) -> None: - stale_age = settings.asset_prices_ttl + 1 - record = _record(age_seconds=stale_age) - _database(monkeypatch, record) +def test_provider_outage_never_uses_legacy_raw_lp_projection(monkeypatch) -> None: + _database( + monkeypatch, + _record(age_seconds=0, tinyman_algo_pool_id=999), + ) monkeypatch.setattr( vestige, "vestige_full_asset_price_not_cached", _provider_failure, ) - monkeypatch.setattr(price_router, "_asset_price_from_tinyman", _provider_failure) - - quote = asyncio.run(price_router.get_asset_price_quote(42)) + monkeypatch.setattr( + price_router, + "_asset_price_from_tinyman", + _provider_failure, + ) - assert quote.source is PriceSource.VESTIGE - assert quote.observed_at == record.observed_at - assert quote.to_legacy_floats() == (2.0, 0.5) + with pytest.raises(PriceUnavailableError, match="no acceptable fallback"): + asyncio.run(price_router.get_asset_price(42)) -def test_derived_lp_keeps_bounded_stale_dependency_timestamp(monkeypatch) -> None: - record = _record(age_seconds=settings.asset_prices_ttl + 1) - _database(monkeypatch, record) +def test_provider_outage_never_uses_derived_lp_source_without_pool_marker( + monkeypatch, +) -> None: + _database( + monkeypatch, + _record( + age_seconds=0, + source=PriceSource.DERIVED_LP.value, + ), + ) monkeypatch.setattr( vestige, "vestige_full_asset_price_not_cached", _provider_failure, ) - monkeypatch.setattr(price_router, "_asset_price_from_tinyman", _provider_failure) - asset_quote = asyncio.run(price_router.get_asset_price_quote(42)) - algo_quote = PriceQuote.from_raw( - asset_id=0, - algo=1, - usd=Decimal("0.25"), - source=PriceSource.VESTIGE, - stale_after=timedelta(minutes=5), + monkeypatch.setattr( + price_router, + "_asset_price_from_tinyman", + _provider_failure, ) - persisted = [] - async def calculate_with_quote(lp_def): - assert lp_def == {"lp_token_id": 99} - return Decimal("4"), asset_quote + with pytest.raises(PriceUnavailableError, match="no acceptable fallback"): + asyncio.run(price_router.get_asset_price(42)) - async def asset_details(asset_id: int): - assert asset_id == 99 - return SimpleNamespace(name="LP") +def test_bounded_stale_quote_preserves_original_provenance(monkeypatch) -> None: + stale_age = settings.asset_prices_ttl + 1 + record = _record(age_seconds=stale_age) + _database(monkeypatch, record) monkeypatch.setattr( - lp_prices, - "_calculate_lp_token_price_algo_with_quote", - calculate_with_quote, - ) - monkeypatch.setattr(lp_prices, "get_asset_details", asset_details) - monkeypatch.setattr(lp_prices, "_upsert_asset_price", persisted.append) - - updated = asyncio.run( - lp_prices._update_single_lp( - {"lp_token_id": 99}, - algo_quote, - current_round=123, - ), + vestige, + "vestige_full_asset_price_not_cached", + _provider_failure, ) + monkeypatch.setattr(price_router, "_asset_price_from_tinyman", _provider_failure) + + quote = asyncio.run(price_router.get_asset_price_quote(42)) - assert updated is True - assert len(persisted) == 1 - assert persisted[0].observed_at == record.observed_at - assert persisted[0].source == PriceSource.DERIVED_LP.value + assert quote.source is PriceSource.VESTIGE + assert quote.observed_at == record.observed_at + assert quote.to_legacy_floats() == (2.0, 0.5) def test_price_older_than_max_stale_is_rejected(monkeypatch) -> None: diff --git a/tests/unit/test_pricing_domain.py b/tests/unit/test_pricing_domain.py index 3bc7d6e2..ea4030a7 100644 --- a/tests/unit/test_pricing_domain.py +++ b/tests/unit/test_pricing_domain.py @@ -7,7 +7,7 @@ import pytest from algosdk.error import IndexerHTTPError -from pymongo.errors import PyMongoError +from pymongo.errors import DuplicateKeyError, PyMongoError from flex.application.price_refresh import ( PriceDataError, @@ -15,9 +15,9 @@ validate_provider_quote, ) from flex.data import asset_prices as asset_price_data -from flex.data import lp_prices as lp_price_data from flex.db.model.priced import AssetPrice from flex.domain.pricing import ( + MAX_OBSERVATION_CLOCK_SKEW, InvalidPriceError, PriceQuote, PriceSource, @@ -160,6 +160,26 @@ def test_provider_boundary_rejects_misattributed_quote() -> None: ) +def test_provider_boundary_rejects_future_dated_quote() -> None: + quote = PriceQuote.from_raw( + asset_id=42, + algo="1", + usd="0.25", + source=PriceSource.VESTIGE, + observed_at=datetime.now(UTC) + MAX_OBSERVATION_CLOCK_SKEW + timedelta(seconds=1), + stale_after=timedelta(minutes=1), + ) + + with pytest.raises(PriceDataError, match="too far in the future"): + validate_provider_quote( + quote, + asset_id=42, + source=PriceSource.VESTIGE, + fresh_for=timedelta(minutes=1), + observed_round=123, + ) + + @pytest.mark.parametrize( "overrides", [ @@ -208,6 +228,29 @@ def test_standalone_freshness_uses_the_same_inclusive_boundary() -> None: ) +def test_future_observation_is_never_treated_as_fresh() -> None: + now = datetime(2026, 1, 1, 12, tzinfo=UTC) + observed_at = now + MAX_OBSERVATION_CLOCK_SKEW + timedelta(microseconds=1) + quote = PriceQuote.from_raw( + asset_id=42, + algo="1", + usd="0.25", + source=PriceSource.VESTIGE, + observed_at=observed_at, + stale_after=timedelta(minutes=5), + ) + + assert ( + is_observation_stale( + observed_at, + fresh_for=timedelta(minutes=5), + now=now, + ) + is True + ) + assert quote.is_stale(now=now) is True + + def _stored_asset_price( *, updated: datetime, @@ -342,14 +385,21 @@ async def fetch_quote(*args, **kwargs) -> PriceQuote: update_one.assert_called_once() selector, update = update_one.call_args.args - assert selector == { - "id": stored.id, - "$or": [ - {"observed_at": {"$exists": False}}, - {"observed_at": None}, - {"observed_at": {"$lte": observed_at}}, - ], - } + assert selector == AssetPrice.encode_query( + { + "id": stored.id, + "$or": [ + {"observed_at": {"$exists": False}}, + {"observed_at": None}, + {"observed_at": {"$lte": observed_at}}, + { + "observed_at": { + "$gt": refreshed.updated + MAX_OBSERVATION_CLOCK_SKEW, + } + }, + ], + } + ) assert update["$set"]["source"] == PriceSource.VESTIGE.value assert update["$set"]["observed_at"] == observed_at assert update_one.call_args.kwargs == {"upsert": False} @@ -357,6 +407,37 @@ async def fetch_quote(*args, **kwargs) -> PriceQuote: collection.insert_one.assert_not_called() +def test_future_dated_asset_price_is_rejected_before_database_io( + monkeypatch, +) -> None: + future = datetime.now(UTC) + MAX_OBSERVATION_CLOCK_SKEW + timedelta(seconds=1) + price = _stored_asset_price( + updated=future, + observed_at=future, + ) + collection = SimpleNamespace( + update_one=Mock(), + find_one=Mock(), + insert_one=Mock(), + ) + monkeypatch.setattr( + asset_price_data, + "db", + SimpleNamespace( + asset_prices=SimpleNamespace( + mongodb_collection=collection, + ), + ), + ) + + with pytest.raises(InvalidPriceError, match="too far in the future"): + asset_price_data._upsert_asset_price(price) + + collection.update_one.assert_not_called() + collection.find_one.assert_not_called() + collection.insert_one.assert_not_called() + + def test_older_asset_price_observation_cannot_replace_newer_record( monkeypatch, ) -> None: @@ -383,10 +464,52 @@ def test_older_asset_price_observation_cannot_replace_newer_record( persisted = asset_price_data._upsert_asset_price(older) assert persisted is False - collection.find_one.assert_called_once_with({"id": older.id}, {"_id": 1}) + collection.find_one.assert_called_once_with( + AssetPrice.encode_query({"id": older.id}), + {"_id": 1}, + ) collection.insert_one.assert_not_called() +def test_newer_initial_price_retries_after_concurrent_older_insert( + monkeypatch, +) -> None: + observed_at = datetime(2026, 1, 1, 12, tzinfo=UTC) + newer = _stored_asset_price( + updated=observed_at, + observed_at=observed_at, + ) + collection = SimpleNamespace( + update_one=Mock( + side_effect=[ + SimpleNamespace(matched_count=0), + SimpleNamespace(matched_count=1), + ] + ), + find_one=Mock(return_value=None), + insert_one=Mock(side_effect=DuplicateKeyError("concurrent insert")), + ) + monkeypatch.setattr( + asset_price_data, + "db", + SimpleNamespace( + asset_prices=SimpleNamespace( + mongodb_collection=collection, + ), + ), + ) + + persisted = asset_price_data._upsert_asset_price(newer) + + assert persisted is True + assert collection.update_one.call_count == 2 + first_selector, first_update = collection.update_one.call_args_list[0].args + retry_selector, retry_update = collection.update_one.call_args_list[1].args + assert retry_selector == first_selector + assert retry_update == {"$set": first_update["$set"]} + assert collection.update_one.call_args_list[1].kwargs == {"upsert": False} + + def test_batch_refresh_isolates_expected_asset_metadata_failure(monkeypatch) -> None: async def batch_prices(asset_ids: list[int]) -> dict[int, vestige.Price]: return {asset_id: vestige.Price(algo=0.25, usd=1.0) for asset_id in asset_ids} @@ -486,76 +609,3 @@ async def asset_details(asset_id: int): ) assert result == [winner] - - -def test_lp_data_adapter_passes_raw_uint64_values_to_exact_domain_pricer(monkeypatch) -> None: - asset1_id = 7 - lp_token_id = 99 - reserve_micros = 2**53 + 1 - pool_balance_micros = 100 - total_supply_micros = reserve_micros + pool_balance_micros - responses = iter( - [ - { - "params": { - "creator": "FACTORY", - "reserve": "POOL", - "total": total_supply_micros, - "decimals": 8, - }, - }, - { - "assets": [ - {"asset-id": asset1_id, "amount": reserve_micros}, - {"asset-id": lp_token_id, "amount": pool_balance_micros}, - ], - "amount": 0, - }, - ], - ) - run_sync_calls: list[tuple[object, tuple[object, ...]]] = [] - - async def run_sync(func, *args): - run_sync_calls.append((func, args)) - return next(responses) - - async def get_details(asset_id: int): - assert asset_id == asset1_id - return SimpleNamespace(decimals=6) - - async def get_price_quote(asset_id: int): - assert asset_id == asset1_id - return PriceQuote.from_raw( - asset_id=asset1_id, - algo=Decimal("0.5"), - usd=Decimal("0.125"), - source=PriceSource.VESTIGE, - stale_after=timedelta(minutes=5), - ) - - exact_pricer = Mock(return_value=Decimal("1")) - monkeypatch.setattr(lp_price_data, "_run_sync", run_sync) - monkeypatch.setattr(lp_price_data, "get_asset_details", get_details) - monkeypatch.setattr( - lp_price_data.price_router, - "get_asset_price_quote", - get_price_quote, - ) - monkeypatch.setattr(lp_price_data, "calculate_lp_token_price_algo_exact", exact_pricer) - - result = asyncio.run( - lp_price_data.calculate_lp_token_price_algo( - {"lp_token_id": lp_token_id, "asset1_id": asset1_id}, - ), - ) - - assert result == Decimal("1") - assert run_sync_calls[1][1] == ("POOL",) - exact_pricer.assert_called_once_with( - asset1_price_algo=Decimal("0.5"), - asset1_reserve_micros=reserve_micros, - asset1_decimals=6, - total_lp_supply_micros=total_supply_micros, - pool_lp_balance_micros=pool_balance_micros, - lp_token_decimals=8, - ) diff --git a/tests/unit/test_stats_pricing.py b/tests/unit/test_stats_pricing.py new file mode 100644 index 00000000..068ae5ff --- /dev/null +++ b/tests/unit/test_stats_pricing.py @@ -0,0 +1,49 @@ +import asyncio +from datetime import UTC, datetime +from types import SimpleNamespace + +from flex.data import stats +from flex.db.model.pools import PoolType +from flex.db.model.priced import AssetPrice + + +def test_tvl_never_uses_legacy_raw_lp_projection(monkeypatch) -> None: + legacy_price = AssetPrice( + id=42, + name="LEGACY", + price_algo=999.0, + price_usd=999.0, + last_update_round=123, + tinyman_algo_pool_id=777, + source="tinyman", + observed_at=datetime.now(UTC), + ) + pool = SimpleNamespace( + pool_id=1, + stake_token=SimpleNamespace(id=42), + total_staked=10, + ) + requested_asset_ids = [] + + async def safe_price(asset_id: int): + requested_asset_ids.append(asset_id) + return SimpleNamespace(price_usd=2.0) + + monkeypatch.setattr( + stats, + "db", + SimpleNamespace( + pool_states=SimpleNamespace( + get_many=lambda **query: [pool], + ), + asset_prices=SimpleNamespace(get_all=lambda: [legacy_price]), + ), + ) + monkeypatch.setattr(stats, "get_asset_price", safe_price) + + total = asyncio.run( + stats.calculate_total_tvl_usd_for_type(PoolType.FARMING), + ) + + assert total == 20.0 + assert requested_asset_ids == [42] diff --git a/tests/unit/test_tinyman_price_projection.py b/tests/unit/test_tinyman_price_projection.py deleted file mode 100644 index a5f64fc9..00000000 --- a/tests/unit/test_tinyman_price_projection.py +++ /dev/null @@ -1,107 +0,0 @@ -import asyncio -from datetime import UTC, datetime, timedelta -from types import SimpleNamespace - -import pytest - -from flex.data import tinyman_lps -from flex.domain.pricing import ( - InvalidLiquidityPoolError, - PriceQuote, - PriceSource, -) - - -def _lp_state( - *, - asset_reserve: int = 4_000_000, - observed_at: datetime | None = None, -): - return SimpleNamespace( - id=555, - token_id=99, - asset1_id=7, - asset2_id=0, - is_algo_pool=True, - asset1_reserve_micros=asset_reserve, - asset2_reserve_micros=2_000_000, - total_tokens_micros=2_000_000, - token_price_algo=0, - last_updated_round=123, - updated=observed_at or datetime(2026, 1, 1, tzinfo=UTC), - ) - - -def _algo_quote(*, observed_at: datetime | None = None) -> PriceQuote: - return PriceQuote.from_raw( - asset_id=0, - algo=1, - usd="0.5", - source=PriceSource.VESTIGE, - stale_after=timedelta(minutes=5), - observed_at=observed_at, - ) - - -def test_tinyman_projection_persists_validated_provenance(monkeypatch) -> None: - pool_observed_at = datetime(2026, 1, 1, tzinfo=UTC) - state = _lp_state(observed_at=pool_observed_at) - lp_updates = [] - persisted = [] - - async def asset_details(asset_id: int): - if asset_id == 7: - return SimpleNamespace(decimals=6, name="ASSET") - assert asset_id == 99 - return SimpleNamespace(decimals=6, name="LP") - - monkeypatch.setattr(tinyman_lps, "get_asset_details", asset_details) - monkeypatch.setattr( - tinyman_lps, - "db", - SimpleNamespace(lp_states=SimpleNamespace(update=lp_updates.append)), - ) - monkeypatch.setattr( - tinyman_lps, - "_upsert_asset_price", - persisted.append, - ) - - result = asyncio.run( - tinyman_lps.update_tinyman_algo_lp_state_and_prices( - state, - algo_quote=_algo_quote( - observed_at=pool_observed_at + timedelta(minutes=1), - ), - ), - ) - - assert state.token_price_algo == 2.0 - assert lp_updates == [state] - assert persisted == [result] - assert result.price_algo == 0.5 - assert result.price_usd == 0.25 - assert result.source == PriceSource.TINYMAN.value - assert result.observed_at == pool_observed_at - - -def test_tinyman_projection_rejects_empty_reserve_without_writing( - monkeypatch, -) -> None: - state = _lp_state(asset_reserve=0) - persisted = [] - monkeypatch.setattr( - tinyman_lps, - "_upsert_asset_price", - persisted.append, - ) - - with pytest.raises(InvalidLiquidityPoolError, match="must be positive"): - asyncio.run( - tinyman_lps.update_tinyman_algo_lp_state_and_prices( - state, - algo_quote=_algo_quote(), - ), - ) - - assert persisted == [] diff --git a/tests/unit/test_wallet_transfers.py b/tests/unit/test_wallet_transfers.py new file mode 100644 index 00000000..00452be7 --- /dev/null +++ b/tests/unit/test_wallet_transfers.py @@ -0,0 +1,42 @@ +from api import wallet +from flex.application.asset_transfers import ( + AssetTransferReceipt, + AssetTransferRequest, +) + + +def test_send_nft_requires_and_namespaces_a_stable_idempotency_key(monkeypatch) -> None: + requests = [] + + class FakeService: + def execute(self, request): + requests.append(request) + return AssetTransferReceipt( + operation_id=request.operation_id, + txid="TXID", + confirmed_round=123, + already_confirmed=False, + ) + + monkeypatch.setattr( + wallet, + "get_asset_transfer_service", + lambda: FakeService(), + ) + + receipt = wallet.send_nft( + "RECEIVER", + 42, + idempotency_key="lottery:draw-1", + ) + + assert receipt.txid == "TXID" + assert requests == [ + AssetTransferRequest( + operation_id="nft:lottery:draw-1", + receiver="RECEIVER", + asset_id=42, + amount_micros=1, + note=None, + ) + ]