diff --git a/.claude/AGENTS.md b/.claude/AGENTS.md deleted file mode 100644 index d29f6f22..00000000 --- a/.claude/AGENTS.md +++ /dev/null @@ -1,65 +0,0 @@ -# Backend Agent Briefing - -Read this file and the root `AGENTS.md` before changing code. `BOARD.md` records -work status; it does not override repository safety rules or the current user request. - -## Working Order - -1. Inspect the relevant implementation and its tests. -2. Check frontend consumers in `~/dev/cometa/metafarm-frontend/src/providers/` - before changing an endpoint or response shape. -3. Preserve external behavior unless the task explicitly authorizes a contract change. -4. Add or update a regression test with the implementation. -5. Run the narrow checks while iterating, then the complete quality gate. - -## Architecture Boundaries - -- `app.py` owns application assembly, route registration, and process orchestration. -- Pure pricing and transaction invariants belong in `flex/domain/`. -- MongoDB and provider adapters stay outside domain modules. -- Synchronous SDK or HTTP work must not block an async request path. -- Financial amounts remain integer base units or `Decimal` until an explicit - compatibility boundary. -- Circuit breakers are process-local resilience controls, not cross-process coordination. - -## Safety Boundaries - -- Never expose or commit mnemonics, tokens, private keys, `.env`, recovery data, - or full sensitive payloads. -- Do not execute signing, refunds, deployments, database migrations, or production - scripts unless the user explicitly requests that external action. -- Validate on-chain and provider payloads before persistence. -- Preserve idempotency for transaction replay and financial background jobs. -- Treat confirmed payouts and complete manifests as terminal states. -- The raw-balance LP publisher is retired. Do not recreate it; token-token fee - ALGO is operational balance, not an economic reserve. -- Do not silently substitute zero for unavailable chain state or price data. - -## Quality Gate - -```bash -pipenv verify -make sync -make quality -``` - -CI validates Compose but does not certify the image as immutable-deploy ready: -it runs the full quality gate on Python 3.12 and 3.14, builds the digest-pinned -Python-only image, smoke-tests the non-root runtime, scans it with Trivy, and -checks financial persistence invariants against a disposable MongoDB service. -Keep the focused lint, type, and coverage ratchets honest. - -`make run` uses the production-equivalent `python app.py` entrypoint, including -critical indexes, migrations only when `MIGRATE=true`, and configured workers. -Use `make run-api` only for API-only hot reload. - -Financial projector design and known standalone-Mongo limits are documented in -`docs/architecture/lp-projection.md`; outbound payout recovery is documented in -`docs/architecture/outbound-asset-transfers.md`. - -## Cross-Project Changes - -The canonical API contract and deploy-state log live in `~/dev/cometa/CLAUDE.md`. -For an API change, update the backend, frontend provider/types, tests, and canonical -contract as one logical unit. Record completed scoped work in `BOARD.md` without -rewriting unrelated user changes. diff --git a/.dockerignore b/.dockerignore index 185a23f0..206fd5ff 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,60 +1,46 @@ -.git -.gitignore -.claude -.mcp.json -.archive -.coverage -.coverage.* -.env -.env.* +# Send only production runtime sources to the Docker daemon. +** +!Pipfile +!Pipfile.lock +!app.py +!env.py +!telegram_bot.py +!api/ +!api/**/ +!api/**/*.py +!blockchain/ +!blockchain/**/ +!blockchain/**/*.py +!bot/ +!bot/**/ +!bot/**/*.py +!core/ +!core/**/ +!core/**/*.py +!dexes/ +!dexes/**/ +!dexes/**/*.py +!flex/ +!flex/**/ +!flex/**/*.py +!scripts/ +!scripts/run.sh + +# Exclude local artifacts that may exist inside allowed source directories. +**/__pycache__/ +**/*.py[cod] +**/.DS_Store **/.env **/.env.* -**/.npmrc -**/node_modules -.idea -.mypy_cache -.pytest_cache -.ruff_cache -**/.mypy_cache -**/.pytest_cache -**/.ruff_cache -.venv -venv -api_calls_*.json -api_endpoints_* -bot_log -contract_operations_*.json -downloaded_logs -all_log_entries_*.json -sample_jsons -temp_data -tmp -test_refund.py -check_vestige_hack.py -sample.py -verify_pool.sh -verify_algorand_credentials.py -metapunks/arweave_key.json -*.md -docs/ -scripts/missing_contracts.json -scripts/recover_contracts.py -scripts/recovery_results.json -scripts/restore_checkpoint.json -scripts/restore_missing.py -scripts/restore_results.json -__pycache__ -**/__pycache__ -*.pyc -**/*.pyc -*.pyo -**/*.pyo -.DS_Store -**/.DS_Store -*.log **/*.log -*.sh -!scripts/*.sh -log_analyzer.py -analyze_logs.sh -download_and_analyze_logs.sh +**/*.bson +**/*.dump +**/*.jks +**/*.key +**/*.keystore +**/*.mnemonic +**/*.p12 +**/*.pem +**/*.pfx +**/*.seed +**/backups/** diff --git a/.env.example b/.env.example index 1722a701..fa1e6a73 100644 --- a/.env.example +++ b/.env.example @@ -2,13 +2,12 @@ ALGO_NETWORK=mainnet SERVER_PORT=8000 WORKERS_NUM=1 -MIGRATE=false # Algorand (never commit real values) ALGO_MNEMONIC= REKEYED_MNEMONIC= ALGOD_ADDRESS=http://algod:8080 -ALGOD_TOKEN= +ALGOD_TOKEN=local-development-token ALGO_INDEXER_ADDRESS=https://mainnet-idx.algonode.cloud OUTBOUND_ASSET_TRANSFER_MAX_FEE_MICROALGOS=1000 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 2d04c2e0..8c0f15cd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,7 +4,7 @@ ## Why - + ## Risk checklist diff --git a/.gitignore b/.gitignore index bc8df320..270d0780 100644 --- a/.gitignore +++ b/.gitignore @@ -1,270 +1,79 @@ -# Byte-compiled / optimized / DLL files +# Python bytecode and build output __pycache__/ *.py[cod] -*$py.class - -# C extensions *.so - -# Distribution / packaging +*.egg-info/ .Python build/ -develop-eggs/ dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ sdist/ -var/ wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec -# Installer logs -pip-log.txt -pip-delete-this-directory.txt +# Virtual environments and package tools +.venv/ +venv/ +env/ +ENV/ +__pypackages__/ -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.ruff_cache/ +# Tests, coverage, and static analysis .coverage .coverage.* -.cache -nosetests.xml coverage.xml -*.cover -*.py,cover +htmlcov/ .hypothesis/ +.mypy_cache/ +.nox/ .pytest_cache/ +.pyre/ +.ruff_cache/ +.tox/ +test-results/ -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments +# Local configuration and credentials .env .env.* !.env.example -.env.main -*.mnemonic -*.seed -*.pem +.mcp.json +*.jks *.key +*.keystore +*.mnemonic *.p12 +*.pem *.pfx -*.jks -*.keystore -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Database exports and operator backups -dump/ -backups/ -*.bson -*.dump -*.sql.gz - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ +*.seed -.idea +# Editors and operating systems .DS_Store **/.DS_Store -/tmp/ -/scripts/missing_contracts.json -/scripts/recovery_results.json -/scripts/restore_checkpoint.json -/scripts/restore_results.json -metapunks/arweave_key.json -metapunks/copy_chosen_nfts.sh -metapunks/filter_nft_files.sh -metapunks/image_urls.txt -metapunks/model_urls.txt -metapunks/remove_used.sh -metapunks/ybg_2_id_mapping.json -metapunks/ybg_first_50_ids.json -metapunks/ybg_info.json -metapunks/ybg_metapunks_numbers.json -sample_jsons/algo_brdini_farm.json -sample_jsons/arm_algo_pact_create_info.json -sample_jsons/bloom_ipt_farm_create.json -sample_jsons/bndri_algo_farm.json -sample_jsons/cjesus_cat_staking.json -sample_jsons/lp_tx.json -sample_jsons/meta_ballsack_deploy.json -sample_jsons/meta_niko_farm_info.json -sample_jsons/meta_pact_nft_lottery.json -sample_jsons/meta_pool_info.json -sample_jsons/meta_pool_txns.json -.archive/cometa_2746104200.json -.archive/cosmic_snapshot_2.json -.archive/cosmic_snapshot_3.json -.archive/cosmic_snapshot.json -.archive/cosmic_stakes_26672956.json -.archive/cosmic_stakes_27330358.json -.archive/cosmic_stakes_27461042.json -.archive/cosmic_stakes_27494994.json -.archive/cosmic_stakes_old_27330238.json -.archive/cosmic_stakes_prev_27461042.json -.archive/free_punks.txt -.archive/latest_cosg_pool_28810058.json -.archive/lol.json -.archive/meta_lotteries.json -.archive/nft_list_1.txt -.archive/opyx_pool_28858365.json -.archive/opyx_pool_29858365.json -bot/.DS_Store -bot/.env -bot_log/bot.log.2022-09-15 -bot_log/bot.log.2022-09-17 -bot_log/bot.log.2022-09-26 -core/.DS_Store -temp_data/airdrop_amounts_algo 200_37601573.json -temp_data/airdrop_amounts_algo 200_37603224.json -temp_data/airdrop_amounts_algo 200_37603344.json -temp_data/airdrop_amounts_algo 200_37603373.json -temp_data/airdrop_amounts_algo 200_37603507.json -temp_data/airdrop_amounts_algo 200_37604069.json -temp_data/airdrop_amounts_token for testing_39225461.json -temp_data/airdrop_amounts_token for testing_39226450.json -temp_data/airdrop_amounts_token for testing_39226505.json -temp_data/airdrop_amounts_token for testing_39226635.json -temp_data/airdrop_amounts_token for testing_39226652.json -temp_data/airdrop_amounts_token for testing_39226671.json -temp_data/airdrop_amounts_token for testing_39226683.json -temp_data/airdrop_txns_algo 200_37603373.json -temp_data/airdrop_txns_algo 200_37603507.json -temp_data/airdrop_txns_algo 200_37604069.json -temp_data/airdrop_txns_token for testing_39226505.json -temp_data/airdrop_txns_token for testing_39226683.json -temp_data/algofi_airdrop.txt -temp_data/cometa_info.html -temp_data/coop_meta_farm.json -temp_data/deploy_requests.json -temp_data/kek.json -temp_data/meta_coop_nft_lottery.json -temp_data/meta_staked_airdrop.py -temp_data/metapunk_holders.txt -temp_data/outage_opted_in.json -temp_data/pool_costs_37598572.json -temp_data/pool_fees_35720000_37530000.json -temp_data/response_1712150137492.json -temp_data/response_1712151196711.json -temp_data/response_1712153435846.json -temp_data/response_1712153944657.json -temp_data/test_optins_balances_count.json -temp_data/test_optins_balances.json -temp_data/test_optins.json -temp_data/user_pools_snapshot.json -temp_data/user_pools.json -temp_data/user_txns_35720000_37530000.json +.idea/ +.vscode/ +*.swp -# Analysis artifacts +# Runtime and operator output +*.log +backups/ +bot_log/ +dump/ +downloaded_logs/ +sample_jsons/ +temp_data/ +tmp/ +.archive/ +*.bson +*.dump *.json.bak +*.sql.gz all_log_entries_*.json analysis_report_*.txt api_calls_*.json api_endpoints_*/ contract_operations_*.json -downloaded_logs/ -sample_jsons/ -test-results/ - -# Local agent/tool configuration -.mcp.json -# Sensitive scripts with hardcoded keys -test_refund.py -check_vestige_hack.py -sample.py -# Retired credential-bearing operator helper; history is scrubbed separately. -/verify_pool.sh - -# Local operator scratch files; never publish from this checkout. -/analyze_logs.sh -/download_and_analyze_logs.sh -/log_analyzer.py -/docs/impeccable-analysis.md -/docs/pagination-plan.md -/scripts/recover_contracts.py -/scripts/restore_missing.py +# Known operator credential and recovery paths +/metapunks/arweave_key.json +/scripts/missing_contracts.json +/scripts/recovery_results.json +/scripts/restore_checkpoint.json +/scripts/restore_results.json diff --git a/AGENTS.md b/AGENTS.md index 827bc42e..759ee5d4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,31 +2,52 @@ ## Project Structure & Module Organization -`app.py` creates the FastAPI application and defines routes; `env.py` owns configuration. Keep database models and background jobs in `api/`, Algorand helpers in `blockchain/`, and shared logic in `core/`. DEX adapters belong in `dexes/`, while Flex-specific APIs, providers, migrations, and pricing data live under `flex/`. Telegram and Farcaster integrations are in `bot/` and `farcaster/`; Node.js interoperability is isolated in `js/`. Operational utilities belong in `scripts/`. The repository currently has no committed `tests/` suite. - -Before changing an API contract, inspect the frontend consumers in `~/dev/cometa/metafarm-frontend/src/providers/` and update the canonical contract in `~/dev/cometa/CLAUDE.md`. +`app.py` assembles the FastAPI application and background workers; `env.py` +defines environment-backed settings. Product routes and legacy models live in +`api/`, Algorand clients in `blockchain/`, and shared persistence and resilience +code in `core/`. New financial use cases are split across `flex/application/`, +pure invariants in `flex/domain/`, and MongoDB adapters in `flex/db/`. Keep DEX +integration details in `dexes/` or `flex/providers/`. Tests mirror these +boundaries under `tests/unit/` and `tests/integration/`. ## Build, Test, and Development Commands -- `pipenv install` installs Python 3.12 dependencies from `Pipfile`. -- `pipenv run uvicorn app:app --reload --port 8000` starts the local API with reload. -- `docker-compose up -d --build` rebuilds and starts the service stack. -- `docker-compose logs -f app` follows application logs. -- `pipenv run pytest tests/ -v` runs tests after the pytest infrastructure is installed. -- `scripts/redeploy.sh` pulls, rebuilds, and restarts services on the VPS; do not use it as a local development command. +- `make sync` installs the locked Python 3.12 development environment. +- `make run-api` starts an API-only reload loop on port 8000. +- `make run` starts the production-equivalent application entrypoint. +- `make quality` runs lint, format, type, and test checks. +- `docker compose up -d --build` starts the local application, MongoDB, and + Algorand services. + +Copy `.env.example` to `.env` before running the application. Use only generated, +unfunded accounts for development. ## Coding Style & Naming Conventions -Use four-space indentation and standard Python naming: `snake_case` for functions and modules, `PascalCase` for classes, and uppercase names for constants. Add type annotations to public functions and prefer explicit models over loosely shaped dictionaries. Use a module-level logger (`logger = logging.getLogger(__name__)`) instead of `print()`. Read secrets and environment-specific values through `env.py`; never hardcode them. Avoid blocking SDK or HTTP calls in async request paths. +Use four spaces and Ruff formatting. Name functions and modules with +`snake_case`, classes with `PascalCase`, and constants with `UPPER_SNAKE_CASE`. +Type public interfaces. Keep financial amounts as integer base units or +`Decimal`; never round-trip them through `float`. Domain modules must remain +independent of network and database clients. Use module-level logging instead of +`print()`, and move blocking SDK calls outside async request paths. ## Testing Guidelines -New tests should use `pytest`, `pytest-asyncio`, and `httpx.AsyncClient`; use `mongomock` or a dedicated test database for MongoDB code. Name files `tests/test_.py` and test functions `test_`. Prioritize atomic lottery claims, contract CRUD, price fetching, and failure paths. Every bug fix should include a regression test when practical. +Use pytest and name tests `test_`. Every bug fix requires a regression +test. Prefer deterministic unit tests with injected clocks and mocked adapter +boundaries. Use `MONGODB_TEST_URI` only for disposable real-MongoDB integration +tests covering uniqueness, compare-and-set behavior, and crash recovery. -## Commit & Pull Request Guidelines +## Commits & Pull Requests -Use concise, lowercase imperative subjects such as `fix null bytes in on-chain state keys`. Update `BOARD.md` before committing related work. Keep commits focused, then push unless explicitly told not to. Pull requests should explain the behavior change, link the task or issue, list verification commands, and call out API, configuration, database, or deployment effects. Include frontend updates when response shapes or endpoints change. +Use focused, lowercase imperative subjects, for example +`fix stale price fallback`. Pull requests must explain the failure mode and +outcome, link a GitHub issue when available, list exact verification commands, +and identify API, migration, configuration, rollout, or rollback effects. ## Security & Data Integrity -Validate all external and on-chain input. Preserve Algorand amounts and identifiers without lossy numeric conversion. Do not log credentials, tokens, private keys, or full sensitive payloads. +Validate every external and on-chain payload before persistence. Outbound asset +operations must remain idempotent and reconcile on-chain before completion. +Never commit or log mnemonics, private keys, tokens, database exports, recovery +artifacts, or unredacted production payloads. diff --git a/BOARD.md b/BOARD.md deleted file mode 100644 index 0a574d51..00000000 --- a/BOARD.md +++ /dev/null @@ -1,60 +0,0 @@ -# Cometa Backend — Task Board - -> Last updated: 2026-07-19 - -## Conventions - -- **ID format**: `CB-NNN` (sequential, never reuse) -- **Statuses**: `todo` | `in_progress` | `blocked` | `done` -- **Priorities**: `critical` | `high` | `medium` | `low` -- **Tags**: `security` | `backend` | `infra` | `dx` | `arch` | `perf` -- Next available ID: **CB-097** - -## Active - -| ID | Task | Status | Priority | Tags | Definition of done | -| --- | --- | --- | --- | --- | --- | -| CB-073 | Public repository hardening | in_progress | critical | security, dx | Credentials rotated, history sanitized, secret protection enabled, clean-clone scan passes | -| CB-074 | Atomic event projection | in_progress | critical | backend, arch | LP projector is crash-safe; replace the disabled legacy staking projector with verified grouped events and recovery tests | -| CB-075 | Isolate transaction signing | todo | high | security, arch | Read-only API boundary; authenticated policy-limited signing service | -| CB-076 | Async persistence boundary | todo | high | backend, perf | Storage outages cannot block the event loop; timeouts and readiness covered | -| CB-078 | Replay-safe outbound asset payouts | done | critical | security, backend, arch | Exact allocations, immutable airdrop manifests, persisted signed intents, on-chain reconciliation, and regression tests | -| CB-079 | Crash-safe LP projection | done | critical | backend, arch | Decimal128 balances, ordered per-state CAS cursor, fenced round checkpoint, snapshot guards, and crash/concurrency tests | -| CB-080 | Financial read-model regressions | done | medium | backend | Reward decimals use the reward asset and request ordering never mutates cached contract state | -| CB-081 | Fence expired sync workers | done | critical | backend, arch | A worker cannot commit a financial round after its lease expires, with unit and real-Mongo regressions | -| CB-082 | Verify Mongo financial invariants | done | high | backend, infra | CI proves CAS replay, marker repair, BSON promotion, uniqueness, and lease fencing against a disposable pinned MongoDB | -| CB-083 | Refresh public engineering docs | done | medium | dx, arch | Runtime commands, Python support, architecture boundaries, API shapes, and cross-project contract are current | -| CB-084 | Disable unverified LP pricing | done | critical | backend, arch | Raw-account-balance publisher is removed; startup purges its legacy rows and every stored-price read rejects them | -| CB-085 | Remove repository credibility drift | done | medium | dx, arch | Public claims match verified behavior, local sync selects Python 3.12, and the unrelated EVM sample is removed | -| CB-086 | Reconcile legacy lottery payouts | done | critical | security, backend | Pre-intent lottery draws fail closed until manual reconciliation; new draws use durable payout states | -| CB-087 | Preserve terminal payout states | done | high | backend, arch | Confirmed transfer intents and complete airdrop manifests cannot regress under stale concurrent workers | -| CB-088 | Separate LP ledger from pricing | done | critical | backend, arch | Raw balances never publish prices; fees are replay-safe events and operational ALGO is isolated from reserves | -| CB-089 | Migrate canonical asset supply | done | high | backend, arch | Financial reads trust only Indexer-provenanced base units, migrate atomically, and fail closed on duplicate asset IDs | -| CB-090 | Claim staking lottery entitlements atomically | done | critical | backend, arch | Concurrent and crash-recovery paths converge on one draw generation in real MongoDB | -| CB-091 | Bound outbound signer fees and network | done | critical | security, backend | Suggested and persisted transactions enforce a configured fee ceiling and canonical genesis before network I/O | -| CB-092 | Validate Algorand numeric boundaries | done | high | backend, arch | Indexer events and snapshots reject coercion, negatives, duplicates, and uint64 overflow before persistence | -| CB-093 | Reserve one-of-one lottery inventory | todo | high | backend, arch | A re-enabled lottery atomically reserves NFT inventory and reconciles release/finalization | -| CB-094 | Replace retired LP pricing with verified adapters | todo | high | backend, arch | DEX-specific app state proves economic reserves; donation and excess-balance adversarial tests pass | -| CB-095 | Reject poisoned price chronology | done | high | backend, arch | Future-dated quotes fail before persistence and a valid quote replaces legacy future timestamps | -| CB-096 | Enforce standalone financial indexes | done | high | backend, arch | Operator airdrops and LP discovery require unique immutable business keys before concurrent upserts | - -## Completed milestones - -| Area | Status | Evidence | -| --- | --- | --- | -| Quality foundation | done | Ruff, strict domain mypy, pytest, focused coverage ratchet, GitHub Actions | -| Precision-safe pricing | done | Decimal observations, provenance, freshness policy, guarded legacy boundary | -| Provider resilience | done | Typed fallback errors, bounded stale data, retry classification, circuit breaker | -| Replay identity | done | Deterministic nested event IDs and collection-level uniqueness constraints | -| LP financial ledger | done | Marker-gap recovery, uint64-safe BSON operations, full-block preflight, snapshot coverage guards, and fenced round CAS | -| Container baseline | done | Digest-pinned Alpine base, multi-stage non-root runtime, healthcheck, image exclusions, Trivy CI gate | -| API request hardening | done | Fail-closed configured header checks, trusted hosts, explicit CORS policy, bounded selectors and wallet expansion | -| Native Reach decoding | done | Versioned global/local codecs, exact-width integers, deterministic layout tests, no private npm runtime | -| Legacy runtime removal | done | CB-077: Node/Reach sidecar and production source bind mount removed | - -## Working agreement - -- Track sensitive security details in private GitHub advisories, not this public board. -- Add a regression test for every correctness or reliability fix. -- Keep financial values as integer micros or `Decimal` until an explicit API boundary. -- Update the frontend and shared contract documentation with every API shape change. diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index d1dbb6a9..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1,145 +0,0 @@ -# Cometa Backend - -Backend for Cometa — an Algorand DeFi platform handling liquidity pools, token swaps, staking rewards, NFT lotteries, and DEX aggregation (Tinyman, Pact, HumbleSwap, Vestige). - -## Stack - -- **Language**: Python 3.12 production runtime; Python 3.14 compatibility CI; Pipenv -- **Framework**: FastAPI + Uvicorn -- **Database**: MongoDB (pymongo) -- **Cache**: process-local TTL caches (Redis migration is roadmap work) -- **Blockchain**: Algorand (py-algorand-sdk, algosdk) -- **Contract state**: versioned native Reach decoder over Algorand state -- **Deployment**: Docker Compose on VPS -- **Image**: digest-pinned Python 3.12 Alpine - -## Project Structure - -``` -app.py — FastAPI composition, routes, startup, and workers -env.py — Settings via pydantic-settings (from .env) -api/ — Product API, background work, wallets, and disabled lottery surface -blockchain/ — Algorand node/indexer adapters -core/ — Shared authentication, persistence, and resilience -dexes/ — DEX-specific integrations -flex/application/ — Financial use-case orchestration -flex/domain/ — Pure allocation, pricing, projection, and identity rules -flex/db/ — Mongo models, BSON codecs, repositories, and indexes -flex/tools/ — Operator tools such as manifest-driven airdrops -bot/ — Telegram bot logic -scripts/ — Deployment and management shell scripts -tests/ — Unit tests plus opt-in real-service integration tests -docs/ — Architecture decisions, operations, and audit reports -``` - -## Key Commands - -```bash -# Local development -pipenv verify -make sync -make run # production-equivalent startup, including indexes/workers -make run-api # API-only Uvicorn reload; no migrations or workers - -# Quality gate -make quality - -# Current VPS-compatible Docker stack -docker compose up -d --build -docker compose logs -f app - -# Production VPS (uses the parent ~/cometa/docker-compose.yml stack) -scripts/redeploy.sh # pull + rebuild + restart the backend service -``` - -## Rules - -- Environment config via `env.py` (pydantic-settings) — never hardcode secrets -- Use `logging` module, never `print()` — logger per module -- Background tasks in `api/background.py` — use exponential backoff for retries -- Decode only explicitly supported Reach contract versions in `flex/blockchain/contract_state.py` -- Route registration and process orchestration stay in `app.py`; Flex routes live in `flex/api.py` -- Legacy contract persistence models live in `core/db/model.py`; `api/db_model.py` - contains only the public contract-type enum. Maintained Flex models and - repositories live under `flex/db/`. -- Asset prices use process-local TTL caches — see `dexes/` for provider calls -- Preserve financial values as `Decimal` or integer base units until an explicit compatibility boundary -- Persist maintained Flex financial `uint64` fields through the BSON codecs in - `flex/db/bson.py`; do not copy legacy int64/float compatibility shapes -- Outbound transfers must persist immutable signed intent before broadcast and reconcile on-chain before completion -- Legacy lottery draws without a matching durable operation remain `reconciliation_required` -- LP events must enter through the complete-round preflight and `MongoLpProjectionRepository` -- Keep token-token fee funding in `operational_algo_balance_micros`; never mix it into economic reserves -- Trust `total_supply_micros` only with `total_supply_source=indexer`; otherwise - refetch and persist both fields atomically before a financial supply read -- Keep `SYNC_STAKING_POOLS=false` until full Algorand application-group validation is implemented -- Raw LP account balances never publish prices; `BACKGROUND_LP_PRICES_UPDATE` - is a retired compatibility setting and cannot restore the removed publisher -- New pricing and transaction invariants belong in pure modules under `flex/domain/` -- Run the strict mypy target before changing `core/circuit_breaker.py` or `flex/domain/` - -## Cross-Project Sync - -The parent `~/dev/cometa/CLAUDE.md` is loaded automatically (Claude Code ancestor chain). It contains shared API contract, deploy state, and breaking changes log. Update it after any API or deploy change. - -## Linked Projects - -### Frontend: metafarm-frontend - -- **Path**: `~/dev/cometa/metafarm-frontend` -- **Repo**: `MetaLabsOG/metafarm-frontend` -- **Stack**: React 18 + TypeScript, Effector (state), styled-components, react-query, CRA + custom webpack -- **Connection**: Frontend connects via `REACT_APP_COMETA_API_URL` (prod: `https://api.cometa.farm/`) - -Both projects are developed in parallel. **Any API change in the backend must be reflected in the frontend and vice versa.** Before modifying an API endpoint, check how the frontend uses it (`src/providers/` directory). - -### API Contract (frontend → backend) - -See parent `~/dev/cometa/CLAUDE.md` for the canonical API contract table. That file is the single source of truth — update it there, not here. - -### Cross-Project Workflow - -- **Adding a new endpoint**: implement in backend → add provider call in frontend → test both -- **Changing response shape**: update backend model → update frontend types/effector store → verify UI -- **Adding a new field to contracts**: update `ContractInfo` model → update frontend `ContractState` type -- **Price/asset changes**: backend `flex/data/` → frontend `src/providers/` + `src/common/store/prices.ts` - -## Testing - -Fast tests are organized by boundary under `tests/unit/`; real-service checks live -under `tests/integration/`. Run the same local quality gate used by CI before committing: - -```bash -make sync -make quality -``` - -- Use `pytest-asyncio` for async endpoint tests -- Use `httpx.AsyncClient` with ASGI transport for endpoint integration tests -- Keep domain tests pure and deterministic; inject clocks and provider functions -- Priority: event replay/idempotency, price freshness, contract CRUD, authorization -- Exercise crash boundaries, concurrent replay, BSON `uint64` limits, and - Indexer-lag cutovers for financial projections -- Use a dedicated test database only for integration tests; never point tests at production -- Set `MONGODB_TEST_URI` only to run the opt-in MongoDB integration suite -- Every bug fix should include a regression test when practical - -## Commit Discipline - -- Always commit after completing a task or logical unit of work — never leave finished work uncommitted -- Use lowercase verb, concise English: `add`, `fix`, `update`, `remove`, `refactor` -- Push after committing unless explicitly told not to -- If changes need review, commit anyway — better to fix in a follow-up than leave uncommitted -- Update task board status before committing the related work - -## Task Board - -Tasks in `BOARD.md`. Format: pantheon. - -## Optional Diagnostic Tools - -MongoDB, Algorand, and DEX inspection connectors may be available in some agent -sessions. Treat them as optional and read-only by default. Derive database and -network targets from the active environment; never assume localhost is a safe -database and never sign or broadcast a transaction without explicit task-scoped -authorization. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 207079bd..68f74f02 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,19 +5,22 @@ safety, and frontend compatibility—not only make the happy path pass. ## Development workflow -Create a focused branch from `main`, copy `.env.example` to `.env`, and install -the locked development environment: +Create a focused branch from `main`, prepare local settings, and install the +locked development environment: ```bash +cp .env.example .env make sync -make run +make run-api ``` -`make run` uses the production-equivalent entrypoint. Use `make run-api` for -API-only hot reload when migrations and background workers are intentionally -out of scope. +Before starting the API, follow the +[`README` quick start](README.md#quick-start) to generate an unfunded +development mnemonic and configure the MongoDB and Algorand endpoints. +`make run-api` is the safe API-only development loop. `make run` uses the +production-equivalent entrypoint and may start configured background workers. -Before opening a pull request, run the same gate used by CI: +Before opening a pull request, run the local Python quality gate: ```bash make quality @@ -29,6 +32,10 @@ job. Keep I/O in adapters or application services; prefer pure functions and immutable value objects for pricing, transaction parsing, and other financial rules. +CI additionally verifies the lockfile and Compose configuration, builds and +scans the production image, scans reachable Git history for secrets, and runs +financial integration tests against disposable MongoDB. + ## Tests Every bug fix needs a regression test. Add fast unit tests under `tests/unit/` @@ -56,18 +63,19 @@ MONGODB_TEST_URI=mongodb://127.0.0.1:27017 \ ## API compatibility -The frontend lives in `../metafarm-frontend` and calls this service through -`src/providers/`. Before changing an endpoint, inspect its consumer. A response -shape or field change must update backend tests, frontend types/providers, and -the shared API contract in the parent Cometa documentation. +The public +[`metafarm-frontend`](https://github.com/MetaLabsOG/metafarm-frontend) +repository consumes this service through `src/providers/`. Before changing an +endpoint, inspect its consumer. A response shape or field change must update +backend tests and the corresponding frontend types and provider calls. ## Commits and pull requests Use concise, imperative commit subjects beginning with a lowercase verb, for example `fix stale price fallback` or `add transaction replay test`. -A pull request should explain the outcome and failure mode, link its issue or -board item, list exact verification commands, and call out migrations, +A pull request should explain the outcome and failure mode, link its issue when +available, list exact verification commands, and call out migrations, configuration changes, rollout order, and rollback steps. Include screenshots only when the change affects user-visible frontend behavior. diff --git a/Makefile b/Makefile index bd64858d..833632fc 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,3 @@ -PYTHON_LINT_PATHS := \ - api/background.py api/nft_lottery.py api/wallet.py app.py blockchain/indexer.py bot/log.py env.py telegram_bot.py \ - core/circuit_breaker.py core/cometa.py core/decorators.py core/util.py \ - flex/__init__.py flex/api.py flex/application flex/blockchain/asset_transfers.py \ - flex/blockchain/contract_state.py flex/blockchain/info.py flex/data/asset_prices.py \ - flex/data/lp_registry.py flex/data/lp_states.py flex/data/lp_tokens.py flex/data/pool_state.py \ - flex/data/transactions.py \ - flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/base_entity.py \ - flex/db/classes/bson_uint64.py flex/db/classes/collection_manager.py \ - flex/db/indexes.py flex/db/lp_projection.py \ - flex/db/sync_coordinator.py flex/db/model/airdrop.py flex/db/model/blockchain.py flex/db/model/liquidity_pools.py \ - flex/db/model/pools.py flex/db/model/priced.py flex/db/model/transfers.py flex/domain flex/providers/pact.py \ - flex/providers/price_router.py \ - flex/providers/vestige.py flex/sync_pools.py flex/sync_state.py \ - flex/migrations/fix_dex_providers.py \ - flex/tools/airdrop.py scripts/verify_algorand_credentials.py tests - PYTHON_MODERN_PATHS := \ api/background.py api/nft_lottery.py api/wallet.py core/circuit_breaker.py flex/application \ flex/blockchain/asset_transfers.py \ @@ -24,21 +7,6 @@ PYTHON_MODERN_PATHS := \ flex/db/model/airdrop.py flex/db/model/priced.py flex/db/model/transfers.py \ flex/domain flex/providers/pact.py flex/providers/price_router.py tests/unit tests/integration -PYTHON_FORMAT_PATHS := \ - api/background.py api/nft_lottery.py api/wallet.py app.py blockchain/indexer.py bot/log.py \ - core/circuit_breaker.py core/cometa.py core/util.py \ - flex/api.py flex/application flex/blockchain/asset_transfers.py flex/blockchain/contract_state.py flex/blockchain/info.py \ - flex/data/asset_prices.py flex/data/lp_registry.py \ - flex/data/lp_states.py flex/data/lp_tokens.py flex/data/pool_state.py \ - flex/data/transactions.py flex/db/asset_transfer_intents.py flex/db/bson.py flex/db/classes/base_entity.py \ - flex/db/classes/bson_uint64.py flex/db/classes/collection_manager.py flex/db/indexes.py flex/db/lp_projection.py \ - flex/db/sync_coordinator.py flex/db/model/airdrop.py flex/db/model/blockchain.py \ - flex/db/model/liquidity_pools.py flex/db/model/pools.py flex/db/model/priced.py flex/db/model/transfers.py \ - flex/domain flex/providers/pact.py flex/providers/price_router.py \ - flex/providers/vestige.py flex/sync_pools.py flex/sync_state.py telegram_bot.py \ - flex/migrations/fix_dex_providers.py \ - flex/tools/airdrop.py tests/conftest.py tests/unit tests/integration - .PHONY: sync run run-api lint format format-check typecheck test quality sync: @@ -52,14 +20,14 @@ run-api: pipenv run uvicorn app:app --reload --port 8000 lint: - pipenv run ruff check $(PYTHON_LINT_PATHS) + pipenv run ruff check . pipenv run ruff check --select ASYNC,C4,DTZ,RUF,UP $(PYTHON_MODERN_PATHS) format: - pipenv run ruff format $(PYTHON_FORMAT_PATHS) + pipenv run ruff format . format-check: - pipenv run ruff format --check $(PYTHON_FORMAT_PATHS) + pipenv run ruff format --check . typecheck: pipenv run mypy diff --git a/README.md b/README.md index f1c7a5aa..182d60e9 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ staking programs, liquidity pools, and token markets. This service turns several eventually consistent data sources—Algorand nodes, the indexer, DEX APIs, and MongoDB—into stable, query-oriented API models for the product frontend. -## Why this repository is interesting +## Engineering guarantees | Engineering concern | Implementation | | --- | --- | @@ -45,9 +45,8 @@ MongoDB—into stable, query-oriented API models for the product frontend. | **Versioned chain decoding** | Reach 0.1.11 state is decoded natively from Algorand with explicit per-version layouts, exact-width integers, and fail-closed schema validation. | | **Supply-chain hardening** | The digest-pinned Alpine image is multi-stage, non-root, and Python-only; CI smoke-tests it and rejects high/critical vulnerabilities or embedded secrets. | -The codebase combines a production system's real constraints with incremental -modernization: pure domain modules and strict typing sit beside legacy adapters, -and the quality gate expands as those adapters gain isolated test seams. +Pure domain modules own financial invariants, while application services and +adapters isolate network, persistence, and provider behavior. ## Architecture @@ -74,9 +73,6 @@ The decision and extension rules are documented in Financial write paths are documented separately in [`docs/architecture/outbound-asset-transfers.md`](docs/architecture/outbound-asset-transfers.md) and [`docs/architecture/lp-projection.md`](docs/architecture/lp-projection.md). -The latest internal multi-agent engineering review, including resolved and -intentionally open items, is in -[`docs/audit/01-audit-architecture-financial-2026-07-19.md`](docs/audit/01-audit-architecture-financial-2026-07-19.md). ### Reliability boundaries @@ -85,6 +81,7 @@ intentionally open items, is in | Provider quote → stored price | Positive, finite decimal values with source and observation timestamp | | Cached price → API response | Explicit freshness window; expired data is rejected instead of silently relabelled | | Chain event → LP read model | Full-block preflight, fee-aware uint64 ledger, CAS cursor, marker repair, and round fencing | +| Contract registration → pool identity | Fail-closed unique IDs and a retry-safe cross-database saga; unsafe legacy staking replay is never run in the request path | | Raw LP account balance → price | Prohibited; economic reserves require a verified DEX-specific adapter | | Asset payout → Algorand | Validate genesis and fee ceiling; persist signed intent first; rebroadcast identical bytes; reconcile before completion | | Selected sync chain SDK → async request path | Bounded executor hand-off | @@ -96,7 +93,7 @@ intentionally open items, is in ### Requirements -- Python 3.12 and [Pipenv](https://pipenv.pypa.io/) for the production-equivalent environment +- Python 3.12 and [Pipenv](https://github.com/pypa/pipenv) for the production-equivalent environment - Python 3.14 is also exercised by CI as a forward-compatibility gate - MongoDB - access to an Algorand node/indexer @@ -114,17 +111,19 @@ make run-api ``` `make run-api` is the safe API-only development loop. `make run` executes the -production-equivalent entrypoint: critical indexes, -optional migrations when `MIGRATE=true`, configured workers, and Uvicorn. For -the full entrypoint, review every worker flag first. A development mnemonic is -still required by legacy Python transaction adapters; use a generated, unfunded -account only. +production-equivalent entrypoint: critical indexes, configured workers, and +Uvicorn. For the full entrypoint, review every worker flag first. A development +mnemonic is still required by legacy Python transaction adapters; use a +generated, unfunded account only. Verify the service: ```bash curl --fail http://127.0.0.1:8000/status # {"version":"2.1.0","algo_network":"mainnet"} + +# Verify Algod and Indexer connectivity without printing credentials. +pipenv run python scripts/verify_algorand_credentials.py ``` For a containerized environment: @@ -134,9 +133,9 @@ docker compose up -d --build docker compose logs -f app ``` -This Compose stack starts persistent MongoDB and a full Algorand node. Inspect -the network, volume paths, and validated `MONGODB_IMAGE`/`ALGOD_IMAGE` values -before using it outside an isolated development host. +This Compose stack starts persistent MongoDB and a full Algorand node from the +digest-pinned defaults in `docker-compose.yml`. Inspect the network and volume +paths before using it outside an isolated development host. ## Quality gate @@ -148,18 +147,18 @@ This single command runs: - Ruff linting and formatting checks; - strict mypy checks on modern domain boundaries; -- the complete Python test suite with branch coverage, including deterministic Reach - state-codec and security-boundary tests. +- the hermetic Python suite with branch coverage, including deterministic Reach + state-codec and security-boundary tests. Real-MongoDB integration tests run + when `MONGODB_TEST_URI` is configured. CI repeats those checks on Python 3.12 and 3.14 for every pull request and every push to `main`, verifies the lockfile and Compose configuration, builds and smoke-tests the production image, scans it with Trivy, and exercises financial repository invariants against a digest-pinned MongoDB service. A pinned -TruffleHog gate fetches and scans every published Git ref for verified or unresolved -credentials and feeds the stable required `python` status. The focused -coverage ratchet is currently 75%; -it measures maintained domain and infrastructure modules rather than presenting -a misleading whole-repository number. +TruffleHog gate fetches and scans every published Git ref for verified or +unresolved credentials and feeds the stable required `python` status. CI +enforces at least 75% branch coverage across the critical domain and +infrastructure modules listed in `Makefile`. Useful individual targets are `make lint`, `make format-check`, `make typecheck`, and `make test`. @@ -176,14 +175,14 @@ Useful individual targets are `make lint`, `make format-check`, | `POST /lp/state/priced` | Read LP token prices; missing or stale batch entries are returned as `null` | | `GET /stats/tvl` | Protocol TVL snapshot | -The production API intentionally disables interactive OpenAPI pages. Endpoint -changes must remain compatible with the linked frontend; see +Interactive OpenAPI pages are intentionally disabled in every environment. +Endpoint changes must remain compatible with the linked frontend; see [`CONTRIBUTING.md`](CONTRIBUTING.md) for the cross-project checklist. ## Repository map ```text -app.py FastAPI composition, routes, and lifespan +app.py FastAPI composition, routes, and process startup api/ Product-facing API and background orchestration blockchain/ Algorand node and indexer adapters core/ Shared authentication, persistence, and resilience @@ -195,7 +194,7 @@ flex/providers/ Market-data provider adapters flex/db/ MongoDB models, repositories, and indexes tests/unit/ Fast regression and boundary tests tests/integration/ Opt-in tests against disposable real services -scripts/ Deployment and legacy operational utilities +scripts/ Container entrypoint and connectivity utilities ``` ## Configuration and security @@ -205,6 +204,10 @@ variables. `.env.example` contains names and safe placeholders only. Never commi wallet mnemonics, API tokens, `.env` files, database exports, unredacted logs, or recovery artifacts. +Each `NEW_DB_NAME` must belong to exactly one Algorand network. Contract, pool, +and state IDs are network-scoped and protected by unique MongoDB indexes; do not +point mainnet and testnet processes at the same Flex database. + Report vulnerabilities privately using the process in [`SECURITY.md`](SECURITY.md). For development conventions, regression-test expectations, and the pull-request checklist, see diff --git a/TDR.md b/TDR.md deleted file mode 100644 index c35eb5f2..00000000 --- a/TDR.md +++ /dev/null @@ -1,49 +0,0 @@ -Hola Algorand! We are honoured with Targeted DeFi Rewards this time. -And we have a master plan which we want to discuss with you. Please tune in! - -### Rewards are going to: -* help small projects to get more stable liquidity -* increase META token stable TVL -* stimulate bridging tokens from other chains and attract new users to Algorand -* deepen liquidity for wrapped assets and RWAs - - -### Project TVL < $10k - -We'll announce the period, when projects with small liquidity may apply for TRD and create a pool. -Due diligence by hand. After the period we'll split the rewards between the pools. - -Projects will have a choice of tokens to pair into liquidity. Stable assets which are listed below. - -### Deep META - -We've been building deep liquidity for META, pairing with different assets, to make token as robust as possible. -Also pairing with wrapped tokens to be less dependent from ALGO. - -We'll incentivize META pools in pair with: ALGO, USDC, gALGO, GOLD$, WETH, WBTC, xUSD, COOP. - -### Degen acquisition - -We have planned a few promo campaigns using bridged tokens like BOBO and DEGEN. -To attract users from other blockchains and make them create Algorand wallet. - -To stay on Algorand after claiming the bait rewards degens need a reason, another bait. -We'll create staking and farming pools with DEGEN, BOBO and others allocating big reward amounts so that APRs would be sweet and degen just couldn't leave the page. - -Of course, this will also encourage Algorand people to just bridge those tokens to Algorand, which is actually also good. - -### Wrap-wrap - -Wrapped assets still don't have many LP pairs and it should be improved. We'll incentivize the pools paired with tokens listed above. - -Probably this section will get the smallest part of the rewards, because it will mostly reward people, who would stake anyway, so it's not useful much. -### Distribution - -The precise percent distribution between pools is to be calculated. - -### Conclusion - -Cometa has it's own area of responsibility — helping projects to grow and just improving the liquidity across the board. -So we shouldn't incentivize much already popular tokens, it's a bit wasteful. We're better use non-trivial ideas to bring value. - -What do you think about our plans? Please share any thoughts! diff --git a/api/background.py b/api/background.py index 41672287..2035df36 100644 --- a/api/background.py +++ b/api/background.py @@ -26,7 +26,6 @@ update_asset_price, ) from flex.data.lp_registry import get_lp_token_definitions -from flex.migrations import migrate_background from flex.sync_pools import sync_pools_loop spawn = multiprocessing.get_context("spawn") @@ -251,9 +250,6 @@ def run_background(): async def tasks(): task_list = [] - if settings.migrate: - task_list.append(asyncio.create_task(safe_background_task(migrate_background(), "migrate_background"))) - if settings.update_contract_caches: task_list.append(asyncio.create_task(safe_background_task(update_contracts_worker(), "update_contracts"))) diff --git a/api/db_model.py b/api/db_model.py index f7b15f0b..43c83f5c 100644 --- a/api/db_model.py +++ b/api/db_model.py @@ -2,5 +2,5 @@ class ContractType(str, Enum): - FARM = 'farm' - DISTRIBUTION = 'distribution' + FARM = "farm" + DISTRIBUTION = "distribution" diff --git a/api/migrations.py b/api/migrations.py deleted file mode 100644 index 1ea6fa5a..00000000 --- a/api/migrations.py +++ /dev/null @@ -1,94 +0,0 @@ -import logging -from datetime import datetime - -from blockchain.node import get_current_round -from blockchain.util import date_from_block -from core.db.contracts import get_all_pool_contracts, update_contract_with -from core.db.model import ContractInfo -from core.decorators import safe_async_method -from core.util import parse_bignum - -logger = logging.getLogger(__name__) - - -def update_contract_start_end_dates(contract: ContractInfo) -> ContractInfo | None: - start_time = datetime.now() - current_block = get_current_round() - - metadata = contract.metadata - if metadata is None: - logger.warning(f'Contract has no metadata: {contract}') - return None - - cache = metadata.get('cache') - logger.info(f'Updating pool {contract.id}...') - - end_block = metadata.get('end_block') - if end_block is None: - end_block = parse_bignum(cache['initial']['endBlock']) - metadata['end_block'] = end_block - logger.info(f'Updated end block for {contract.id} to {end_block}') - - begin_block = metadata.get('begin_block') - if begin_block is None and cache is not None: - begin_block = parse_bignum(cache['initial']['beginBlock']) - metadata['begin_block'] = begin_block - logger.info(f'Updated begin block for {contract.id} to {begin_block}') - - if contract.end_date is None: - contract.end_date = date_from_block(end_block, current_block, start_time) - metadata['end_date'] = contract.end_date - logger.info(f'Updated end date for {contract.id} to {contract.end_date}') - - if contract.begin_date is None: - contract.begin_date = date_from_block(begin_block, current_block, start_time) - metadata['begin_date'] = contract.begin_date - logger.info(f'Updated begin date for {contract.id} to {contract.begin_date}') - - if metadata.get('lock_length_blocks') is None and cache is not None: - lock_length_blocks = parse_bignum(cache['initial']['lockLengthBlocks']) - metadata['lock_length_blocks'] = lock_length_blocks - logger.info(f'Updated lock length blocks for {contract.id} to {lock_length_blocks}') - - update_contract_with( - contract_id=contract.id, - metadata=metadata, - begin_date=contract.begin_date, - end_date=contract.end_date - ) - - logger.info(f'Pool {contract.id} updated: {contract.begin_date} - {contract.end_date}.') - - return contract - - -@safe_async_method -async def update_pool_start_end_dates() -> None: - start_time = datetime.now() - current_block = get_current_round() - logger.info(f'Migrating pools info. Current block: {current_block}, start time: {start_time}.') - - all_contracts = get_all_pool_contracts() - logger.info(f'Found {len(all_contracts)} contracts.') - - for contract in all_contracts: - try: - if contract.begin_date is not None and contract.end_date is not None: - logger.info(f'Skipping pool {contract.id}...') - continue - - contract = update_contract_start_end_dates(contract) - - if contract is not None: - update_contract_with( - contract_id=contract.id, - metadata=contract.metadata, - begin_date=contract.begin_date, - end_date=contract.end_date - ) - - logger.info(f'Pool {contract.id} updated: {contract.begin_date} - {contract.end_date}.') - - except Exception as e: - logger.error(f'Failed to update pool {contract.id}: {e}', exc_info=True) - continue diff --git a/api/nft_lottery.py b/api/nft_lottery.py index 785b5402..a010dcd8 100644 --- a/api/nft_lottery.py +++ b/api/nft_lottery.py @@ -735,7 +735,7 @@ def draw_id(lottery: NftLottery) -> int | None: while True: if not lottery.available_nfts: return None - # TODO: refactor to get balance once + # Re-check inventory because another draw may have consumed the NFT. res = random.choice(lottery.available_nfts) data = indexer_client.lookup_account_assets(address=cometa_public_key, asset_id=res) assets = data.get("assets", []) diff --git a/api/notifications.py b/api/notifications.py index 33aa35bc..54d0717e 100644 --- a/api/notifications.py +++ b/api/notifications.py @@ -18,12 +18,7 @@ async def notify_telegram_chat(chat_id: int, text: str): - return await bot.send_message( - chat_id=chat_id, - text=text, - parse_mode=ParseMode.HTML, - disable_web_page_preview=True - ) + return await bot.send_message(chat_id=chat_id, text=text, parse_mode=ParseMode.HTML, disable_web_page_preview=True) async def notify_cometa_telegram_channel(text: str): @@ -37,26 +32,22 @@ def notify_discord_webhook(text: str): _ = webhook.execute() -async def announce_stake( - duration: timedelta, - lock_duration: timedelta, - metadata: dict -) -> None: - stake_token_id = int(metadata['stake_token_id']) - reward_token_id = int(metadata['reward_token_id']) +async def announce_stake(duration: timedelta, lock_duration: timedelta, metadata: dict) -> None: + stake_token_id = int(metadata["stake_token_id"]) + reward_token_id = int(metadata["reward_token_id"]) stake_token = get_asset(stake_token_id) reward_token = get_asset(reward_token_id) - stake_token_link = f'https://vestige.fi/asset/{stake_token_id}' - reward_token_link = f'https://vestige.fi/asset/{reward_token_id}' + stake_token_link = f"https://vestige.fi/asset/{stake_token_id}" + reward_token_link = f"https://vestige.fi/asset/{reward_token_id}" - stake_token_name = stake_token['params']['unit-name'] - reward_token_name = reward_token['params']['unit-name'] - token_buy_link = 'https://app.cometa.farm/swap' + stake_token_name = stake_token["params"]["unit-name"] + reward_token_name = reward_token["params"]["unit-name"] + token_buy_link = "https://app.cometa.farm/swap" try: - lock_str = '' if lock_duration.days == 0 else f'🔒 {lock_duration.days} days\n' + lock_str = "" if lock_duration.days == 0 else f"🔒 {lock_duration.days} days\n" telegram_text = f''' 💸 New pool {stake_token_name} → {reward_token_name} @@ -73,8 +64,8 @@ async def announce_stake( logger.exception(e) try: - lock_str = '' if lock_duration.days == 0 else f'🔒 **{lock_duration.days} days**\n' - discord_text = f''' + lock_str = "" if lock_duration.days == 0 else f"🔒 **{lock_duration.days} days**\n" + discord_text = f""" 💸 New pool **{stake_token_name} → {reward_token_name}** ⏳ **{duration.days} days** @@ -87,44 +78,40 @@ async def announce_stake( Enjoy staking ❤️ https://app.cometa.farm/ - ''' + """ notify_discord_webhook(discord_text) except Exception as e: logger.exception(e) -async def announce_farm( - duration: timedelta, - lock_duration: timedelta, - metadata: dict -) -> None: - if metadata.get('asset1_id') is None: +async def announce_farm(duration: timedelta, lock_duration: timedelta, metadata: dict) -> None: + if metadata.get("asset1_id") is None: return await announce_stake(duration, lock_duration, metadata) - lp_token_id = int(metadata['stake_token_id']) - asset1_id = int(metadata['asset1_id']) - asset2_id = int(metadata['asset2_id']) - reward_token_id = int(metadata['reward_token_id']) + lp_token_id = int(metadata["stake_token_id"]) + asset1_id = int(metadata["asset1_id"]) + asset2_id = int(metadata["asset2_id"]) + reward_token_id = int(metadata["reward_token_id"]) lp_token = get_asset(lp_token_id) asset1 = get_asset(asset1_id) asset2 = get_asset(asset2_id) reward_token = get_asset(reward_token_id) - asset1_link = f'https://vestige.fi/asset/{asset1_id}' - asset2_link = f'https://vestige.fi/asset/{asset2_id}' - reward_link = f'https://vestige.fi/asset/{reward_token_id}' - lp_pool_address = lp_token['params']['reserve'] - get_lp_link = f'https://app.tinyman.org/#/pool/{lp_pool_address}/add-liquidity' + asset1_link = f"https://vestige.fi/asset/{asset1_id}" + asset2_link = f"https://vestige.fi/asset/{asset2_id}" + reward_link = f"https://vestige.fi/asset/{reward_token_id}" + lp_pool_address = lp_token["params"]["reserve"] + get_lp_link = f"https://app.tinyman.org/#/pool/{lp_pool_address}/add-liquidity" - asset1_name = asset1['params']['unit-name'] - asset2_name = asset2['params']['unit-name'] - reward_token_name = reward_token['params']['unit-name'] + asset1_name = asset1["params"]["unit-name"] + asset2_name = asset2["params"]["unit-name"] + reward_token_name = reward_token["params"]["unit-name"] try: - lock_str = '' if lock_duration.days == 0 else f'🔒 {lock_duration.days} days\n' - asset1_str = f'{asset1_name}' if asset1_id != 0 else 'ALGO' - asset2_str = f'{asset2_name}' if asset2_id != 0 else 'ALGO' + lock_str = "" if lock_duration.days == 0 else f"🔒 {lock_duration.days} days\n" + asset1_str = f'{asset1_name}' if asset1_id != 0 else "ALGO" + asset2_str = f'{asset2_name}' if asset2_id != 0 else "ALGO" telegram_text = f''' 💸 New pool {asset1_str}/{asset2_str} → {reward_token_name} @@ -139,15 +126,15 @@ async def announce_farm( logger.exception(e) try: - lock_str = '' if lock_duration.days == 0 else f'🔒 **{lock_duration.days} days**\n' - assets_info_str = '' + lock_str = "" if lock_duration.days == 0 else f"🔒 **{lock_duration.days} days**\n" + assets_info_str = "" if asset1_id != 0: - assets_info_str += f'ℹ️ **${asset1_name}** *{asset1_link}*\n' + assets_info_str += f"ℹ️ **${asset1_name}** *{asset1_link}*\n" if asset2_id != 0: - assets_info_str += f'ℹ️ **${asset2_name}** *{asset2_link}*\n' + assets_info_str += f"ℹ️ **${asset2_name}** *{asset2_link}*\n" if reward_token_id != asset1_id and reward_token_id != asset2_id: - assets_info_str += f'ℹ️ **${reward_token_name}** *{reward_link}*\n' - discord_text = f''' + assets_info_str += f"ℹ️ **${reward_token_name}** *{reward_link}*\n" + discord_text = f""" 💸 New pool **{asset1_name}/{asset2_name} → {reward_token_name}** ⏳ **{duration.days} days** @@ -156,26 +143,22 @@ async def announce_farm( ✅ Get LP tokens: *{get_lp_link}* Enjoy farming ❤️ https://app.cometa.farm/ - ''' + """ notify_discord_webhook(discord_text) except Exception as e: logger.exception(e) -async def announce_distribution( - duration: timedelta, - lock_duration: timedelta, - metadata: dict -) -> None: - stake_token_id = metadata['stake_token_id'] +async def announce_distribution(duration: timedelta, lock_duration: timedelta, metadata: dict) -> None: + stake_token_id = metadata["stake_token_id"] stake_token = get_asset(int(stake_token_id)) - stake_token_name = stake_token['params']['unit-name'] + stake_token_name = stake_token["params"]["unit-name"] - token_link = f'https://vestige.fi/asset/{stake_token_id}' - token_buy_link = 'https://app.cometa.farm/swap' + token_link = f"https://vestige.fi/asset/{stake_token_id}" + token_buy_link = "https://app.cometa.farm/swap" try: - lock_str = '' if lock_duration.days == 0 else f'🔒 {lock_duration.days} days\n' + lock_str = "" if lock_duration.days == 0 else f"🔒 {lock_duration.days} days\n" token_str = f'{stake_token_name}' telegram_text = f''' 💸 New pool {token_str} → {token_str} @@ -191,8 +174,8 @@ async def announce_distribution( logger.exception(e) try: - lock_str = '' if lock_duration.days == 0 else f'🔒 **{lock_duration.days} days**\n' - discord_text = f''' + lock_str = "" if lock_duration.days == 0 else f"🔒 **{lock_duration.days} days**\n" + discord_text = f""" 💸 New pool **{stake_token_name} → {stake_token_name}** ⏳ **{duration.days} days** @@ -204,20 +187,14 @@ async def announce_distribution( Enjoy staking ❤️ https://app.cometa.farm/ - ''' + """ notify_discord_webhook(discord_text) except Exception as e: logger.exception(e) -# TODO: show additional algo rewards -# TODO: show projected APY async def notify_new_pool( - begin_block: int, - end_block: int, - lock_length_blocks: int, - type: str, - metadata: Optional[dict] = None + begin_block: int, end_block: int, lock_length_blocks: int, type: str, metadata: Optional[dict] = None ): try: duration = duration_from_block_count(end_block - begin_block + 1) @@ -230,6 +207,6 @@ async def notify_new_pool( elif type == ContractType.DISTRIBUTION: return await announce_distribution(duration, lock_duration, metadata) else: - raise Exception(f'Unknown contract type: {type}') + raise Exception(f"Unknown contract type: {type}") except Exception as e: logger.exception(e) diff --git a/api/pool_snapshot.py b/api/pool_snapshot.py deleted file mode 100644 index eed64e5d..00000000 --- a/api/pool_snapshot.py +++ /dev/null @@ -1,121 +0,0 @@ -import json -import logging -from collections import defaultdict -from typing import Optional - -from algosdk.v2client import indexer - -from blockchain.node import get_current_round -from env import settings - -indexer_client = indexer.IndexerClient(indexer_token=settings.algod_token, indexer_address=settings.algo_indexer_address) - -ASSET_TRANSFER_TX = 'asset-transfer-transaction' -APPLICATION_CALL_TX = 'application-transaction' -PAYMENT_TX = 'payment-transaction' - -logger = logging.getLogger(__name__) - -def get_pool_wallet(pool_id: int) -> Optional[str]: - data = indexer_client.application_logs(application_id=pool_id, limit=10) - log_data = data.get('log-data') - if log_data is None or len(log_data) == 0: - return None - - txid = log_data[0]['txid'] - data = indexer_client.transaction(txid=txid) - transaction = data.get('transaction') - if transaction is None: - return None - - return transaction['inner-txns'][0]['sender'] - - -def get_pool_snapshot(pool_id: int, max_round: Optional[int] = None, watch_address: Optional[str] = None): - pool_wallet = get_pool_wallet(pool_id) - if pool_wallet is None: - pool_wallet = get_pool_wallet(pool_id) - if pool_wallet is None: - return {'error': 'Pool not found, bro.'} - logger.info(f'Pool wallet: {pool_wallet}') - - if max_round is None: - max_round = get_current_round() - - next_token = None - all_txns = [] - balances = defaultdict(lambda: 0) - - total_staked = 0 - - if watch_address is not None: - logger.info(f'Watching address {watch_address}') - - while True: - data = indexer_client.search_transactions_by_address(address=pool_wallet, - next_page=next_token) - txns = data['transactions'] - logger.info(f'New txns, cnt = {len(txns)}') - - for tx in txns: - logger.info(f'Processing tx: {tx}') - if ASSET_TRANSFER_TX in tx: - if max_round is not None and tx['confirmed-round'] > max_round: - continue - - sender = tx['sender'] - amount = tx[ASSET_TRANSFER_TX]['amount'] - if sender == watch_address: - logger.info(f'{balances[sender]} + {amount} = {balances[sender] + amount}') - balances[sender] += amount - logger.info(f'Total staked + {amount} = {total_staked + amount}') - total_staked += amount - elif APPLICATION_CALL_TX in tx: - inner_txns = tx['inner-txns'] - is_claim = False - for inner_tx in inner_txns: - if PAYMENT_TX in inner_tx: - is_claim = True - if is_claim: - continue - for inner_tx in inner_txns: - if ASSET_TRANSFER_TX in inner_tx: - if inner_tx['confirmed-round'] > max_round: - continue - - receiver = inner_tx[ASSET_TRANSFER_TX]['receiver'] - amount = inner_tx[ASSET_TRANSFER_TX]['amount'] - if receiver == watch_address: - logger.info(f'{balances[receiver]} - {amount} = {balances[receiver] - amount}') - balances[receiver] -= amount - logger.info(f'Total staked - {amount} = {total_staked - amount}') - total_staked -= amount - - logger.info(f'{len(txns)} txns processed!') - logger.info(f'Currently {len(balances)} balances\n') - - all_txns.extend(txns) - if 'next-token' in data: - next_token = data['next-token'] - else: - break - - res_filename = f'pool_{pool_id}_round_{max_round}.json' - with open(res_filename, 'w') as write_file: - json.dump(balances, write_file, indent=4, sort_keys=True) - - logger.info(f'{len(all_txns)} processed!') - logger.info(f'{len(balances)} wallets are written to "{res_filename}"!') - - total_microtokens = 0 - for k, v in balances.items(): - total_microtokens += v - - logger.info(f'In total {total_microtokens} microtokens') - - return balances - - -if __name__ == '__main__': - snapshot = get_pool_snapshot(1010907359, watch_address='GRHZVZ7IGCD75RDJWTWJ4ZJOIYG3YGPHKRXJ7IN2W77AIMFQ57V7GBDVIA') - print(json.dumps(snapshot, indent=4, sort_keys=True)) diff --git a/api/stats.py b/api/stats.py index cc50b312..4243154a 100644 --- a/api/stats.py +++ b/api/stats.py @@ -3,12 +3,12 @@ from dataclasses import dataclass from typing import Optional -from cachetools import cached, TTLCache, FIFOCache +from cachetools import FIFOCache, TTLCache, cached from dataclasses_json import dataclass_json from core.db.mongodb import get_db_collection from core.tinychart import get_asset_price -from dexes.tinyman import init_tinyman_client, get_pool_info +from dexes.tinyman import get_pool_info, init_tinyman_client from env import settings @@ -21,8 +21,8 @@ class CometaSnapshot: distribution_tvl: float | None = None -tiny_client = init_tinyman_client(settings.algod_address) -snapshots = get_db_collection(settings.db_name, 'snapshot') +tiny_client = init_tinyman_client() +snapshots = get_db_collection(settings.db_name, "snapshot") logger = logging.getLogger(__name__) @@ -30,10 +30,7 @@ class CometaSnapshot: def save_snapshot(farm_tvl: float, distribution_tvl: float, staking_tvl: float) -> CometaSnapshot: cur_time = time.time() snapshot = CometaSnapshot( - farm_tvl=farm_tvl, - distribution_tvl=distribution_tvl, - timestamp=cur_time, - staking_tvl=staking_tvl + farm_tvl=farm_tvl, distribution_tvl=distribution_tvl, timestamp=cur_time, staking_tvl=staking_tvl ) snapshots.insert_one(snapshot.to_dict()) return snapshot @@ -65,13 +62,8 @@ def get_asset_info(asset_id: int) -> dict: def get_tvl() -> dict: snapshot = get_last_snapshot() if snapshot is None: - return {'farm': 0, 'distribution': 0, 'staking': 0, 'total': 0} + return {"farm": 0, "distribution": 0, "staking": 0, "total": 0} farm = snapshot.farm_tvl or 0 distribution = snapshot.distribution_tvl or 0 staking = snapshot.staking_tvl or 0 - return { - 'farm': farm, - 'distribution': distribution, - 'staking': staking, - 'total': farm + distribution + staking - } + return {"farm": farm, "distribution": distribution, "staking": staking, "total": farm + distribution + staking} diff --git a/api/swaps.py b/api/swaps.py index b1fb2f0b..1a408379 100644 --- a/api/swaps.py +++ b/api/swaps.py @@ -1,11 +1,7 @@ from dataclasses import dataclass -from typing import Union from dataclasses_json import dataclass_json -from core.db.db_manager import DbManager -from env import settings - @dataclass_json @dataclass @@ -14,18 +10,5 @@ class SwapInfo: wallet: str asset1_id: int asset2_id: int - asset1_amount: Union[int, float] - asset2_amount: Union[int, float] - - -swaps = DbManager(settings.db_name, 'swaps', '_id', SwapInfo) - - -def validate_swap(swap: SwapInfo) -> None: - # TODO: check algoindexer for swap OR THROW - pass - - -def record_swap(swap: SwapInfo) -> SwapInfo: - validate_swap(swap) - return swaps.create(swap) + asset1_amount: int | float + asset2_amount: int | float diff --git a/app.py b/app.py index f38cc54e..35714804 100644 --- a/app.py +++ b/app.py @@ -25,27 +25,36 @@ from blockchain.util import date_from_block from core.auth import require_password from core.db.contracts import ( + ContractIdentityConflictError, ContractInfo, + ContractWriteResult, + ensure_contract_id_index, get_contract, get_contracts_by_type, - insert_contract, + get_or_create_contract, invalidate_contracts_cache, update_contract, ) from core.db.model import UserPool from core.util import parse_bignum, strip_version from env import settings +from flex.application.pool_registration import ( + PoolIdentityConflictError, + create_pool_from_contract, + ensure_pool_identity_slot, +) from flex.blockchain.contract_state import ( ContractStateDecodeError, ContractStateFetchError, fetch_contract_views, ) from flex.data.asset_prices import get_asset_price_not_cached -from flex.data.pool_state import get_or_create_pool_state, update_pool_state +from flex.data.pool_state import ( + PoolStateIdentityConflictError, + create_pool_state_from_contract, +) from flex.data.pool_state_priced import calculate_user_pool_state_cost from flex.db.indexes import ensure_database_indexes -from flex.migrations import migrate_before_start -from flex.migrations.contracts import create_pool_from_contract from flex.providers.vestige import get_dex_tag_by_name from flex.sync_pools import get_sync_user_state_by_address @@ -222,9 +231,9 @@ def parse_cache(cache: Optional[dict]) -> dict: return {} -async def create_contract(contract_info: AddContract, new_metadata: dict) -> ContractInfo: +async def create_contract(contract_info: AddContract, new_metadata: dict) -> ContractWriteResult: return await create_contract_with( - type=contract_info.type, + type=contract_info.type.value, id=contract_info.id, version=contract_info.version, description=contract_info.description, @@ -232,56 +241,67 @@ async def create_contract(contract_info: AddContract, new_metadata: dict) -> Con ) -async def create_contract_with(type: str, id: int, version: str, description: str, metadata: dict) -> ContractInfo: +async def create_contract_with( + type: str, + id: int, + version: str, + description: str | None, + metadata: dict, +) -> ContractWriteResult: try: - cache = metadata.get("cache") + canonical_metadata = dict(metadata) + cache = canonical_metadata.get("cache") metadata_fields = parse_cache(cache) current_date = datetime.now() - if "dex" in metadata: + if "dex" in canonical_metadata: try: - metadata["dex"] = get_dex_tag_by_name(metadata["dex"]) + canonical_metadata["dex"] = get_dex_tag_by_name(canonical_metadata["dex"]) except Exception as e: - logger.warning(f"Could not get DEX tag for {metadata['dex']}: {e}") + logger.warning(f"Could not get DEX tag for {canonical_metadata['dex']}: {e}") + + for field_name in ("begin_block", "end_block", "lock_length_blocks"): + if field_name in metadata_fields: + canonical_metadata[field_name] = metadata_fields[field_name] contract = ContractInfo( type=type, id=id, version=version, - description=description, + description=description or "", deployed_timestamp=current_date.timestamp(), deployed_date=current_date, begin_date=metadata_fields.get("begin_date"), end_date=metadata_fields.get("end_date"), - metadata=metadata, + metadata=canonical_metadata, ) - insert_contract(contract) - invalidate_contracts_cache() + # Reject an opposite-kind orphan before anchoring the saga in the + # network-scoped legacy contract collection. + ensure_pool_identity_slot(contract) + write_result = get_or_create_contract(contract) + if write_result.created: + invalidate_contracts_cache() - try: - pool_info = await create_pool_from_contract(contract) - if pool_info is not None: - logger.info(f"Created pool info for contract {id}: {pool_info.id}") + # Contract and pool records live in separate databases. Treat this as a + # retry-safe saga: a failed pool write fails the request, while the next + # identical request resumes from the canonical contract record. + pool_info = await create_pool_from_contract(write_result.contract) + logger.info(f"Ensured pool info exists for contract {id}: {pool_info.id}") - try: - pool_state = await get_or_create_pool_state(pool_info.id) - await update_pool_state(pool_state) - logger.info(f"Updated pool state for pool {pool_info.id}") - except Exception as e: - logger.error(f"Error updating pool state: {e}", exc_info=True) + # Registration owns identity creation only. The legacy staking event + # projection is intentionally disabled; replaying it in a concurrent + # request path could apply a transfer before its marker is persisted. + pool_state = await create_pool_state_from_contract(write_result.contract) + logger.info(f"Ensured pool state identity exists for pool {pool_state.pool_id}") - try: - await get_asset_price_not_cached(pool_info.stake_token.id) - await get_asset_price_not_cached(pool_info.reward_token.id) - logger.info( - f"Fetched asset prices for tokens {pool_info.stake_token.id} and {pool_info.reward_token.id}" - ) - except Exception as e: - logger.error(f"Error fetching asset prices: {e}", exc_info=True) + try: + await get_asset_price_not_cached(pool_info.stake_token.id) + await get_asset_price_not_cached(pool_info.reward_token.id) + logger.info(f"Fetched asset prices for tokens {pool_info.stake_token.id} and {pool_info.reward_token.id}") except Exception as e: - logger.error(f"Error creating pool from contract {id}: {e}", exc_info=True) + logger.error(f"Error fetching asset prices: {e}", exc_info=True) - return contract + return write_result except Exception as e: logger.error(f"Fatal error creating contract {id}: {e}", exc_info=True) raise @@ -324,9 +344,6 @@ async def _fetch_contract_view(contract_id: int, contract_type: str, version: st async def register_contract(contract: AddContract) -> ContractInfo: logger.info(f"Registering a new contract {contract}") - if get_contract(contract.id) is not None: - raise HTTPException(status_code=409, detail="Contract already exists") - cache_metadata = {} if contract.type in ("farm", "distribution"): @@ -350,18 +367,27 @@ async def register_contract(contract: AddContract) -> ContractInfo: metadata = {**contract.metadata, **cache_metadata} if contract.metadata is not None else cache_metadata logger.info(f"Registering a contract with metadata:\n{metadata}") - rich_contract = await create_contract(contract, metadata) - try: - await notify_new_pool( - begin_block=rich_contract.metadata["begin_block"], - end_block=rich_contract.metadata["end_block"], - lock_length_blocks=rich_contract.metadata["lock_length_blocks"], - type=contract.type, - metadata=contract.metadata, - ) - except Exception as e: - logger.error(f"Error notifying about new pool: {e}", exc_info=True) + registration = await create_contract(contract, metadata) + except ( + ContractIdentityConflictError, + PoolIdentityConflictError, + PoolStateIdentityConflictError, + ) as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + rich_contract = registration.contract + + if registration.created: + try: + await notify_new_pool( + begin_block=rich_contract.metadata["begin_block"], + end_block=rich_contract.metadata["end_block"], + lock_length_blocks=rich_contract.metadata["lock_length_blocks"], + type=contract.type, + metadata=contract.metadata, + ) + except Exception as e: + logger.error(f"Error notifying about new pool: {e}", exc_info=True) return rich_contract @@ -574,20 +600,12 @@ def setup_logging(): def init_app(): - """Initialize the application with migrations if needed""" - if settings.migrate: - logger.info("Running database migrations...") - try: - migrate_before_start() - logger.info("Database migrations completed successfully") - except Exception as e: - logger.error(f"Error during database migrations: {e}", exc_info=True) - raise - + """Initialize database invariants and legacy contract metadata.""" # These indexes enforce idempotent event and price projections. try: from flex import db as flex_db + ensure_contract_id_index() ensure_database_indexes(flex_db) except Exception as e: logger.error(f"Error during database index setup: {e}", exc_info=True) diff --git a/blockchain/assets.py b/blockchain/assets.py index 8fc058cc..da776eda 100644 --- a/blockchain/assets.py +++ b/blockchain/assets.py @@ -1,6 +1,5 @@ from env import settings - MICROALGOS_IN_ALGO = 1000000 MAINNET_USDC_ASA_ID = 31566704 @@ -15,4 +14,3 @@ ALGO_ASA_ID = 0 USDC_ASA_ID = MAINNET_USDC_ASA_ID if settings.is_mainnet() else TESTNET_USDC_ASA_ID META_ASA_ID = MAINNET_META_ASA_ID if settings.is_mainnet() else TESTNET_META_ASA_ID - diff --git a/blockchain/indexer.py b/blockchain/indexer.py index 1d0f07ae..422b377b 100644 --- a/blockchain/indexer.py +++ b/blockchain/indexer.py @@ -16,7 +16,7 @@ headers={"User-Agent": "py-algorand-sdk", "x-algo-api-token": settings.algod_token}, ) -# TODO: INFO NOT FULL, handle get_asset(0) better +# Synthetic native-ALGO metadata matching the Indexer asset response shape. ALGO_ASSET_INFO = { "created-at-round": 3317341, "deleted": False, diff --git a/blockchain/nfts.py b/blockchain/nfts.py index 318d2d79..3163ca38 100644 --- a/blockchain/nfts.py +++ b/blockchain/nfts.py @@ -15,6 +15,4 @@ class NftInfo: def get_nft_info(asa_id: int) -> NftInfo: asset = get_asset(asa_id) - return NftInfo(asa_id=asa_id, - name=asset['params']['name'], - image_url=asset['params']['url']) + return NftInfo(asa_id=asa_id, name=asset["params"]["name"], image_url=asset["params"]["url"]) diff --git a/blockchain/node.py b/blockchain/node.py index 1db0a848..adbabd85 100644 --- a/blockchain/node.py +++ b/blockchain/node.py @@ -1,8 +1,8 @@ import logging -from algosdk.v2client.algod import AlgodClient from algosdk.error import AlgodHTTPError -from cachetools import cached, TTLCache +from algosdk.v2client.algod import AlgodClient +from cachetools import TTLCache, cached from env import settings @@ -10,11 +10,11 @@ def init_algod_client() -> AlgodClient: - return AlgodClient(settings.algod_token, settings.algod_address, - headers={ - 'User-Agent': 'py-algorand-sdk', - 'x-algo-api-token': settings.algod_token - }) + return AlgodClient( + settings.algod_token, + settings.algod_address, + headers={"User-Agent": "py-algorand-sdk", "x-algo-api-token": settings.algod_token}, + ) algod_client = init_algod_client() @@ -24,14 +24,14 @@ def init_algod_client() -> AlgodClient: def get_current_round(): try: data = algod_client.status() - current_round = data['last-round'] + current_round = data["last-round"] get_current_round._last_known_round = current_round return current_round except AlgodHTTPError as e: logger.error(f"Failed to get current round from Algod: {e}") - fallback = getattr(get_current_round, '_last_known_round', 0) + fallback = getattr(get_current_round, "_last_known_round", 0) logger.warning(f"Using fallback round: {fallback}") return fallback except Exception as e: logger.error(f"Unexpected error getting current round: {e}", exc_info=True) - return getattr(get_current_round, '_last_known_round', 0) + return getattr(get_current_round, "_last_known_round", 0) diff --git a/blockchain/util.py b/blockchain/util.py index 45289a60..3632236f 100644 --- a/blockchain/util.py +++ b/blockchain/util.py @@ -1,4 +1,4 @@ -from datetime import timedelta, datetime +from datetime import UTC, datetime, timedelta from blockchain.indexer import indexer_client from env import settings @@ -11,7 +11,7 @@ def duration_from_block_count(blocks: int) -> timedelta: def duration_from_blocks(begin_block: int, end_block: int) -> timedelta: if end_block < begin_block: - raise ValueError(f'End block {end_block} is less than begin block {begin_block}.') + raise ValueError(f"End block {end_block} is less than begin block {begin_block}.") return duration_from_block_count(end_block - begin_block + 1) @@ -21,5 +21,5 @@ def date_from_block(round_num: int, current_round_num: int, current_date: dateti return current_date + pool_time_remains round_info = indexer_client.block_info(round_num=round_num, header_only=True) - timestamp = round_info['timestamp'] - return datetime.fromtimestamp(timestamp) + timestamp = round_info["timestamp"] + return datetime.fromtimestamp(timestamp, UTC).replace(tzinfo=None) diff --git a/bot/background.py b/bot/background.py index fa35dde9..57258a7b 100644 --- a/bot/background.py +++ b/bot/background.py @@ -9,7 +9,7 @@ from core.db.new_pools import new_pools from core.decorators import repeat_every, safe_async_method -spawn = multiprocessing.get_context('spawn') +spawn = multiprocessing.get_context("spawn") logger = logging.getLogger(__name__) @@ -31,12 +31,12 @@ async def notify_new_pools(): for user in all_users: await notify_user_new_pool(user, pool) new_pools.remove(pool) - logger.info(f'Notified users about new pool: {pool}') + logger.info(f"Notified users about new pool: {pool}") @repeat_every(bot_settings.user_pools_cache_ttl_seconds) async def notify_all(): - logger.info('Notifying...') + logger.info("Notifying...") if bot_settings.notify_users: await notify_users() @@ -45,9 +45,7 @@ async def notify_all(): def run_background(): async def tasks(): - await asyncio.gather( - notify_all() - ) + await asyncio.gather(notify_all()) asyncio.run(tasks()) @@ -56,10 +54,9 @@ async def tasks(): def start_bg_tasks(): proc = spawn.Process(target=run_background) proc.start() - logger.info(f'STARTED BG TASKS: {proc}') + logger.info(f"STARTED BG TASKS: {proc}") try: yield proc finally: proc.terminate() proc.join() - diff --git a/bot/db/users.py b/bot/db/users.py index ff10f901..6a971c2b 100644 --- a/bot/db/users.py +++ b/bot/db/users.py @@ -4,7 +4,7 @@ from core.db.db_manager import DbManager from env import settings -bot_users = DbManager[BotUser](settings.db_name, 'bot_users', 'telegram_id', BotUser) +bot_users = DbManager[BotUser](settings.db_name, "bot_users", "telegram_id", BotUser) def create_user(algo_address: str, telegram_id: int) -> BotUser: @@ -17,11 +17,11 @@ def get_user(args: dict) -> Optional[BotUser]: def get_user_by_address(address: str) -> BotUser: - return get_user({'algo_address': address}) + return get_user({"algo_address": address}) def get_user_by_tg(tg_id: int) -> BotUser: - return get_user({'telegram_id': tg_id}) + return get_user({"telegram_id": tg_id}) def update_user(user: BotUser) -> BotUser: diff --git a/bot/env.py b/bot/env.py index 9b59bf00..ea64c326 100644 --- a/bot/env.py +++ b/bot/env.py @@ -2,9 +2,9 @@ from pydantic_settings import BaseSettings, SettingsConfigDict -FEEDBACK_COMMAND = 'feedback' -SUPPORT_COMMAND = 'support' -MESSAGE_ALL_COMMAND = 'message_all' +FEEDBACK_COMMAND = "feedback" +SUPPORT_COMMAND = "support" +MESSAGE_ALL_COMMAND = "message_all" class Settings(BaseSettings): @@ -26,13 +26,13 @@ class Settings(BaseSettings): user_pools_cache_ttl_seconds: int = 300 logs_dir: str - logging_level: str = 'INFO' + logging_level: str = "INFO" telegram_admin_ids: list[int] - db_name: str = 'COMETA_BOT' + db_name: str = "COMETA_BOT" - model_config = SettingsConfigDict(env_file='bot/.env', env_file_encoding='utf-8') + model_config = SettingsConfigDict(env_file="bot/.env", env_file_encoding="utf-8") @property def remind_again_delay(self): diff --git a/bot/formatting.py b/bot/formatting.py index b234c829..d43739bb 100644 --- a/bot/formatting.py +++ b/bot/formatting.py @@ -1,4 +1,4 @@ -from bot.utils import usd_format, seconds_format +from bot.utils import seconds_format, usd_format from core.db.model import UserPool @@ -8,15 +8,19 @@ def calculate_apy(apr: float, periods: int) -> float: def format_user_pool(pool: UserPool) -> str: if pool.is_ended(): - return f'💸 {pool.name}.\n' \ - f'Stake = ${usd_format(pool.staked_usd)}, rewards = ${usd_format(pool.reward_usd)}.\n' \ - f'It ended {seconds_format(pool.ended_duration)} ago :(\n' + return ( + f"💸 {pool.name}.\n" + f"Stake = ${usd_format(pool.staked_usd)}, rewards = ${usd_format(pool.reward_usd)}.\n" + f"It ended {seconds_format(pool.ended_duration)} ago :(\n" + ) - text = f'💸 {pool.name}, {usd_format(pool.current_apr)}% APR.\n' \ - f'Stake = ${usd_format(pool.staked_usd)}, rewards = ${usd_format(pool.reward_usd)}.\n' + text = ( + f"💸 {pool.name}, {usd_format(pool.current_apr)}% APR.\n" + f"Stake = ${usd_format(pool.staked_usd)}, rewards = ${usd_format(pool.reward_usd)}.\n" + ) if pool.needs_compound(): apy = calculate_apy(pool.current_apr / 100, 365) * 100 - text += f'Daily compound gives you {usd_format(apy)}% APY!\n' + text += f"Daily compound gives you {usd_format(apy)}% APY!\n" return text diff --git a/bot/notifier.py b/bot/notifier.py index 083b9b93..74a11fc8 100644 --- a/bot/notifier.py +++ b/bot/notifier.py @@ -19,22 +19,21 @@ async def notify_user(user: BotUser): pools = await get_address_pools(user.algo_address) - text = f'🤖 {Phrases.greet()}️\n' + text = f"🤖 {Phrases.greet()}️\n" compound_pools = filter_compoundable_pools(pools) if compound_pools: - text += '\n\n✅ Need compounding:\n\n' - text += '\n'.join([format_user_pool(pool) for pool in compound_pools]) + text += "\n\n✅ Need compounding:\n\n" + text += "\n".join([format_user_pool(pool) for pool in compound_pools]) ended_pools = filter_ended_pools(pools) if ended_pools: - text += '\n\n❌ Need withdraw:\n\n' - text += '\n'.join([format_user_pool(pool) for pool in ended_pools]) + text += "\n\n❌ Need withdraw:\n\n" + text += "\n".join([format_user_pool(pool) for pool in ended_pools]) if ended_pools or compound_pools: - text += '\n\nIt is the time.\n\nhttps://app.cometa.farm/\n' + text += "\n\nIt is the time.\n\nhttps://app.cometa.farm/\n" - # TODO: save all notifications to DB logger.debug(text) await app_context.bot.send_message(text=text, chat_id=user.telegram_id, parse_mode=ParseMode.HTML) @@ -45,10 +44,10 @@ async def notify_user(user: BotUser): @safe_async_method async def notify_user_new_pool(user: BotUser, pool: NewPoolInfo): - text = f'🤖 {Phrases.greet()}️\n\n' - text += f'🎉 Good news! New {pool.type} pool has started on Cometa!\n\n' - text += f'{pool.name}\n\n' - text += f'Be one of the first to grab juicy APR😏\n\n' - text += 'https://app.cometa.farm/' + text = f"🤖 {Phrases.greet()}️\n\n" + text += f"🎉 Good news! New {pool.type} pool has started on Cometa!\n\n" + text += f"{pool.name}\n\n" + text += "Be one of the first to grab juicy APR😏\n\n" + text += "https://app.cometa.farm/" await app_context.bot.send_message(text=text, chat_id=user.telegram_id, parse_mode=ParseMode.HTML) diff --git a/bot/phrase_manager.py b/bot/phrase_manager.py index 09a69ae8..dbc5d078 100644 --- a/bot/phrase_manager.py +++ b/bot/phrase_manager.py @@ -4,19 +4,23 @@ class Phrases: @classmethod def greet(cls) -> str: - return random.choice([ - 'Hey, beautiful!❤', - 'Hey, what\'s up?)', - 'How you doing?😏', - 'What a great day, huh?)', - 'Bip-bop! Beep, bop!🤖' - ]) + return random.choice( + [ + "Hey, beautiful!❤", + "Hey, what's up?)", + "How you doing?😏", + "What a great day, huh?)", + "Bip-bop! Beep, bop!🤖", + ] + ) @classmethod def check_pools(cls) -> str: - return random.choice([ - 'Let\'s see what you have😏', - 'Are you farming hard? Let\'s find out!', - 'So, what do we have here...', - 'Farming hard, huh?', - ]) + return random.choice( + [ + "Let's see what you have😏", + "Are you farming hard? Let's find out!", + "So, what do we have here...", + "Farming hard, huh?", + ] + ) diff --git a/bot/utils.py b/bot/utils.py index 20db0122..20991608 100644 --- a/bot/utils.py +++ b/bot/utils.py @@ -2,12 +2,12 @@ def seconds_format(s: float): seconds = int(s) periods = [ - ('год', 'года', 60*60*24*365), - ('месяц', 'месяца', 60*60*24*30), - ('день', 'дней', 60*60*24), - ('час', 'часа', 60*60), - ('минута', 'минут', 60), - ('секунда', 'секунд', 1) + ("год", "года", 60 * 60 * 24 * 365), + ("месяц", "месяца", 60 * 60 * 24 * 30), + ("день", "дней", 60 * 60 * 24), + ("час", "часа", 60 * 60), + ("минута", "минут", 60), + ("секунда", "секунд", 1), ] strings = [] @@ -15,10 +15,7 @@ def seconds_format(s: float): if seconds > period_seconds: period_value, seconds = divmod(seconds, period_seconds) name = periods_name if period_value > 1 else period_name - return f'{period_value} {name}' - - # TODO: remove return to have more details - # strings.append(f'{period_value} {name}') + return f"{period_value} {name}" return ", ".join(strings) diff --git a/core/cometa.py b/core/cometa.py index 19d71c28..41d8a9ea 100644 --- a/core/cometa.py +++ b/core/cometa.py @@ -12,8 +12,8 @@ from core.util import BLOCKS_IN_A_YEAR, blocks_to_seconds, parse_bignum from flex.blockchain.contract_state import fetch_contract_local_views -# ffs -BIG_NUM = 1000000000000000000 +# Reach reward-per-token values use an 18-decimal fixed-point scale. +REWARD_PER_TOKEN_SCALE = 10**18 logger = logging.getLogger(__name__) @@ -48,7 +48,8 @@ def get_pool_state(contract: ContractInfo, is_mainnet: bool = True) -> PoolState additional["asset1_id"] = asset1_id additional["asset2_id"] = asset2_id - total_tokens = total_microtokens / (10**6) # TODO: fix not all lp tokens have 6 decimals + # Deployed legacy Reach LP contracts use six-decimal pool tokens. + total_tokens = total_microtokens / (10**6) lp_price = get_lp_price(asset1_id, asset2_id) if is_mainnet else 0 total_cost = total_tokens * lp_price if is_mainnet else 1 else: @@ -130,9 +131,10 @@ def recalculate_reward( last_block_with_rewards = min(current_block, pool.end_block) reward_blocks_passed = last_block_with_rewards - pool.last_update_block reward_per_token_stored_new = ( - pool.reward_per_token_stored + reward_blocks_passed * pool.reward_per_block * BIG_NUM // pool.total_staked + pool.reward_per_token_stored + + reward_blocks_passed * pool.reward_per_block * REWARD_PER_TOKEN_SCALE // pool.total_staked ) - reward_to_pay_now = staked * (reward_per_token_stored_new - reward_per_token_paid) // BIG_NUM + reward_to_pay_now = staked * (reward_per_token_stored_new - reward_per_token_paid) // REWARD_PER_TOKEN_SCALE return reward + reward_to_pay_now diff --git a/core/db/cometa_users.py b/core/db/cometa_users.py index a5aa52c2..7bf81ddf 100644 --- a/core/db/cometa_users.py +++ b/core/db/cometa_users.py @@ -3,13 +3,13 @@ from core.cometa import fetch_user_pools from core.db.db_manager import DbManager -from core.db.model import UserPool, CometaUser +from core.db.model import CometaUser, UserPool from core.decorators import safe_async_method from env import settings logger = logging.getLogger(__name__) -cometa_users = DbManager[CometaUser](settings.db_name, 'cometa_users', 'address', CometaUser) +cometa_users = DbManager[CometaUser](settings.db_name, "cometa_users", "address", CometaUser) @safe_async_method @@ -21,7 +21,7 @@ async def update_user_pools(user: CometaUser, is_mainnet: bool = True) -> list[U cometa_users.update(user) return user_pools except Exception as e: - logger.error(f'Error updating user pools for {user.address}: {e}', exc_info=True) + logger.error(f"Error updating user pools for {user.address}: {e}", exc_info=True) return [] diff --git a/core/db/contracts.py b/core/db/contracts.py index 48c401da..11a52575 100644 --- a/core/db/contracts.py +++ b/core/db/contracts.py @@ -1,39 +1,102 @@ -import time -from datetime import datetime, timedelta, timezone +from dataclasses import dataclass +from datetime import datetime, timedelta from typing import Optional -from cachetools import cached, TTLCache +from cachetools import TTLCache, cached +from pymongo.collection import Collection +from pymongo.errors import DuplicateKeyError from core.db.model import ContractInfo from core.db.mongodb import get_db_collection from env import settings -collection = get_db_collection(settings.db_name, 'contract') +collection = get_db_collection(settings.db_name, "contract") _contracts_cache = TTLCache(maxsize=16, ttl=settings.contracts_cache_ttl) -def add_contract(type: str, id: int, version: str, description: str, metadata: Optional[dict]) -> str: - cur_time = time.time() - contract = ContractInfo(type, id, version, cur_time, description, metadata) - res = collection.insert_one(contract.to_dict()) - return str(res.inserted_id) - - -def insert_contract(contract: ContractInfo) -> str: - res = collection.insert_one(contract.to_dict()) - return str(res.inserted_id) +class ContractIdentityConflictError(ValueError): + """A contract ID is already bound to another declared identity.""" + + +@dataclass(frozen=True) +class ContractWriteResult: + contract: ContractInfo + created: bool + + +def _contract_identity(contract: ContractInfo) -> tuple[str, str]: + contract_type = getattr(contract.type, "value", contract.type) + return str(contract_type), contract.version.removeprefix("^") + + +def ensure_contract_id_index( + *, + target_collection: Collection | None = None, +) -> None: + """Fail closed on duplicate contract IDs, then enforce uniqueness.""" + + target = target_collection if target_collection is not None else collection + duplicate = next( + iter( + target.aggregate( + [ + {"$group": {"_id": "$id", "count": {"$sum": 1}}}, + {"$match": {"count": {"$gt": 1}}}, + {"$limit": 1}, + ], + allowDiskUse=True, + ) + ), + None, + ) + if duplicate is not None: + raise RuntimeError( + f"contract contains duplicate immutable ID {duplicate['_id']!r}; reconcile it before registration" + ) + target.create_index("id", unique=True, name="contract_id_unique") + + +def get_or_create_contract( + contract: ContractInfo, + *, + target_collection: Collection | None = None, +) -> ContractWriteResult: + """Persist one canonical contract and recover safely from concurrent retries.""" + + target = target_collection if target_collection is not None else collection + try: + result = target.update_one( + {"id": contract.id}, + {"$setOnInsert": contract.to_dict()}, + upsert=True, + ) + created = result.upserted_id is not None + except DuplicateKeyError: + # A competing upsert won after this request evaluated its selector. + created = False + + stored_document = target.find_one({"id": contract.id}) + if stored_document is None: + raise RuntimeError(f"contract {contract.id} disappeared after atomic upsert") + + stored = ContractInfo.from_dict(stored_document) + if _contract_identity(stored) != _contract_identity(contract): + raise ContractIdentityConflictError( + f"contract {contract.id} is already registered with a different type or version" + ) + return ContractWriteResult(contract=stored, created=created) def update_contract(id: int, description: Optional[str] = None, metadata: Optional[dict] = None) -> bool: upd_dict = {} if description is not None: - upd_dict['description'] = description + upd_dict["description"] = description if metadata is not None: - upd_dict['metadata'] = metadata - + upd_dict["metadata"] = metadata + if len(upd_dict) > 0: - res = collection.update_one({'id': id}, {'$set': upd_dict}) + res = collection.update_one({"id": id}, {"$set": upd_dict}) return res.acknowledged else: return False @@ -41,7 +104,7 @@ def update_contract(id: int, description: Optional[str] = None, metadata: Option def update_contract_with(contract_id: int, **kwargs) -> bool: if len(kwargs) > 0: - res = collection.update_one({'id': contract_id}, {'$set': kwargs}) + res = collection.update_one({"id": contract_id}, {"$set": kwargs}) return res.acknowledged else: return False @@ -52,14 +115,14 @@ def get_contracts(args: dict) -> list[ContractInfo]: def get_all_pool_contracts() -> list[ContractInfo]: - return get_contracts({'type': {'$in': ['farm', 'distribution']}}) + return get_contracts({"type": {"$in": ["farm", "distribution"]}}) @cached(cache=_contracts_cache) def get_contracts_by_type(type: Optional[str]) -> list[ContractInfo]: if type is None: - return get_contracts({'type': {'$in': ['distribution', 'farm']}}) - return get_contracts({'type': type}) + return get_contracts({"type": {"$in": ["distribution", "farm"]}}) + return get_contracts({"type": type}) def get_active_contracts(type: str) -> list[ContractInfo]: @@ -82,15 +145,15 @@ def invalidate_contracts_cache(): def get_contract(contract_id: int) -> Optional[ContractInfo]: - res = collection.find_one({'id': contract_id}) + res = collection.find_one({"id": contract_id}) return ContractInfo.from_dict(res) if res else res def remove_contract(contract_id: int) -> int: - res = collection.delete_many({'id': contract_id}) + res = collection.delete_many({"id": contract_id}) return res.deleted_count def remove_contracts(type: str) -> int: - res = collection.delete_many({'type': type}) + res = collection.delete_many({"type": type}) return res.deleted_count diff --git a/core/db/db_manager.py b/core/db/db_manager.py index 4adfc475..7541a203 100644 --- a/core/db/db_manager.py +++ b/core/db/db_manager.py @@ -1,13 +1,13 @@ from dataclasses import dataclass from functools import cached_property -from typing import TypeVar, Generic, Any +from typing import Any, Generic, TypeVar from core.db.mongodb import get_db_collection -T = TypeVar('T') +T = TypeVar("T") -# TODO: migrate all Cometa DB collections to CollectionManager +# Compatibility repository for legacy collections. @dataclass class DbManager(Generic[T]): db_name: str @@ -39,7 +39,7 @@ def get_all(self) -> list[T]: def update(self, item: T) -> T: item_dict = item.to_dict() - self.collection.update_one({self.primary_key: item_dict.get(self.primary_key)}, {'$set': item_dict}) + self.collection.update_one({self.primary_key: item_dict.get(self.primary_key)}, {"$set": item_dict}) return item def remove(self, item: T): diff --git a/core/db/migrations/separate_user_info.py b/core/db/migrations/separate_user_info.py deleted file mode 100644 index d3d64dff..00000000 --- a/core/db/migrations/separate_user_info.py +++ /dev/null @@ -1,38 +0,0 @@ -import logging -from datetime import datetime - -from bot.db.model import BotUser -from core.db.contracts import get_all_pool_contracts, update_contract_with -from core.db.model import CometaUser -from core.db.mongodb import get_db_collection -from env import settings - -logger = logging.getLogger(__name__) - - -def migrate_users(): - logger.info('Migrating users...') - - users = get_db_collection('COMETA_BOT', 'users') - cometa_users = get_db_collection(settings.db_name, 'cometa_users') - bot_users = get_db_collection(settings.db_name, 'bot_users') - - cnt = 0 - for user in users.find(): - cometa_users.insert_one(CometaUser(user['algo_address'], user['pools']).to_dict()) - bot_users.insert_one(BotUser(user['algo_address'], user['telegram_id']).to_dict()) - cnt += 1 - - logger.info(f'Migrated {cnt} users') - - -def migrate_contract_dates(): - contracts = get_all_pool_contracts() - for contract in contracts: - print(f'Before:\n{contract.format_str()}\n') - contract.begin_date = datetime.fromisoformat(contract.begin_date) - contract.end_date = datetime.fromisoformat(contract.end_date) - contract.metadata['begin_date'] = contract.begin_date - contract.metadata['end_date'] = contract.end_date - update_contract_with(contract.id, metadata=contract.metadata, begin_date=contract.begin_date, end_date=contract.end_date) - print(f'After:\n{contract.format_str()}\n') diff --git a/core/db/model.py b/core/db/model.py index f7bfaaba..4b4adcf8 100644 --- a/core/db/model.py +++ b/core/db/model.py @@ -38,7 +38,8 @@ class UserPool: reward_usd: float lock_timestamp: int ended_duration: Optional[float] - staked_token_id: Optional[int] = None # TODO: remove Optional when all UserPools are migrated + # Legacy UserPool documents may not contain this field. + staked_token_id: Optional[int] = None reward_token_id: Optional[int] = None staked_tokens: Optional[float] = None reward_tokens: Optional[float] = None @@ -60,9 +61,9 @@ class CometaUser: class PoolType(str, Enum): - FARM = 'farm' - DISTRIBUTION = 'distribution' - STAKING = 'staking' + FARM = "farm" + DISTRIBUTION = "distribution" + STAKING = "staking" def __str__(self): return self.value @@ -97,15 +98,15 @@ class PoolState: class PoolStatus(str, Enum): - LIVE = 'live' - ENDED = 'ended' - UPCOMING = 'upcoming' + LIVE = "live" + ENDED = "ended" + UPCOMING = "upcoming" def __str__(self): return self.value @classmethod - def from_current_block(cls, current_block: int, start_block: int, end_block: int) -> 'PoolStatus': + def from_current_block(cls, current_block: int, start_block: int, end_block: int) -> "PoolStatus": if current_block < start_block: return PoolStatus.UPCOMING if current_block > end_block: diff --git a/core/db/new_pools.py b/core/db/new_pools.py index 48588b55..e8f142a7 100644 --- a/core/db/new_pools.py +++ b/core/db/new_pools.py @@ -14,4 +14,4 @@ class NewPoolInfo: type: str -new_pools = DbManager[NewPoolInfo](settings.db_name, 'new_pools', 'id', NewPoolInfo) +new_pools = DbManager[NewPoolInfo](settings.db_name, "new_pools", "id", NewPoolInfo) diff --git a/core/db/pools.py b/core/db/pools.py index ddf396e4..5da3efa4 100644 --- a/core/db/pools.py +++ b/core/db/pools.py @@ -2,5 +2,4 @@ from core.db.model import PoolInfo from env import settings - -pools_db = DbManager[PoolInfo](settings.db_name, 'pools', 'id', PoolInfo) +pools_db = DbManager[PoolInfo](settings.db_name, "pools", "id", PoolInfo) diff --git a/core/decorators.py b/core/decorators.py index 65e0e8ba..39622c3e 100644 --- a/core/decorators.py +++ b/core/decorators.py @@ -4,6 +4,7 @@ logger = logging.getLogger(__name__) + def safe_async_method(fn): @wraps(fn) async def wrapper(*args, **kwargs): @@ -11,11 +12,12 @@ async def wrapper(*args, **kwargs): return await fn(*args, **kwargs) except Exception as e: logger.error( - 'Error in `%s`: %s', + "Error in `%s`: %s", fn.__qualname__, e, exc_info=True, ) + return wrapper @@ -26,5 +28,7 @@ async def wrapper(*args, **kwargs): while True: await fn(*args, **kwargs) await asyncio.sleep(seconds) + return wrapper + return decorator diff --git a/core/tinychart.py b/core/tinychart.py index b013f7dd..0211b78e 100644 --- a/core/tinychart.py +++ b/core/tinychart.py @@ -2,12 +2,12 @@ from dataclasses import dataclass import httpx -from cachetools import cached, TTLCache +from cachetools import TTLCache, cached from blockchain.assets import MICROALGOS_IN_ALGO from env import settings -BASE_URL = 'https://api.vestigelabs.org' +BASE_URL = "https://api.vestigelabs.org" USDC_ASSET_ID = 31566704 logger = logging.getLogger(__name__) @@ -30,17 +30,17 @@ class Price: usd: float microalgo: int - def multiply(self, mul: float) -> 'Price': + def multiply(self, mul: float) -> "Price": return Price(self.usd * mul, int(self.microalgo * mul)) @cached(cache=TTLCache(maxsize=1, ttl=settings.algo_price_ttl)) def get_algo_price() -> float: - url = f'{BASE_URL}/assets/price?asset_ids=0&denominating_asset_id={USDC_ASSET_ID}' + url = f"{BASE_URL}/assets/price?asset_ids=0&denominating_asset_id={USDC_ASSET_ID}" data = _get_client().get(url).json() if isinstance(data, list) and len(data) > 0: - return data[0]['price'] - logger.error(f'Failed to get ALGO price from Vestige: {data}') + return data[0]["price"] + logger.error(f"Failed to get ALGO price from Vestige: {data}") return 0.0 @@ -48,13 +48,13 @@ def get_algo_price() -> float: def get_asset_price(asset_id: int) -> float: if asset_id == 0: return get_algo_price() - url = f'{BASE_URL}/assets/price?asset_ids={asset_id}&denominating_asset_id={USDC_ASSET_ID}' - logger.debug(f'Getting price for asset {asset_id} from {url}') + url = f"{BASE_URL}/assets/price?asset_ids={asset_id}&denominating_asset_id={USDC_ASSET_ID}" + logger.debug(f"Getting price for asset {asset_id} from {url}") data = _get_client().get(url).json() - logger.debug(f'Response: {data}') + logger.debug(f"Response: {data}") if isinstance(data, list) and len(data) > 0: - return data[0]['price'] - logger.error(f'No price data for asset {asset_id}: {data}') + return data[0]["price"] + logger.error(f"No price data for asset {asset_id}: {data}") return 0.0 diff --git a/dexes/humble.py b/dexes/humble.py index 93a51d3c..2a56cee9 100644 --- a/dexes/humble.py +++ b/dexes/humble.py @@ -6,7 +6,7 @@ from core.db.mongodb import get_db_collection from env import settings -humble_pools = get_db_collection(settings.db_name, 'humblePools') +humble_pools = get_db_collection(settings.db_name, "humblePools") @dataclass_json @@ -27,7 +27,7 @@ class HumblePool: def get_pool_by_id(pool_id: int) -> Optional[HumblePool]: - res = humble_pools.find_one({'poolAddress': pool_id }) + res = humble_pools.find_one({"poolAddress": pool_id}) return HumblePool.from_dict(res) if res else None @@ -36,4 +36,4 @@ def get_pools(args: dict) -> List[HumblePool]: def get_pools_by_assets(assetA: int, assetB: int) -> List[HumblePool]: - return get_pools({'tokenAId': assetA, 'tokenBId': assetB}) + return get_pools({"tokenAId": assetA, "tokenBId": assetB}) diff --git a/dexes/tinyman.py b/dexes/tinyman.py index 75898059..bb3a7e61 100644 --- a/dexes/tinyman.py +++ b/dexes/tinyman.py @@ -2,56 +2,37 @@ import logging import urllib.request from dataclasses import dataclass -from typing import Optional -from algosdk import account, encoding, mnemonic -from algosdk.transaction import ApplicationOptInTxn, AssetOptInTxn, PaymentTxn from algosdk.v2client.algod import AlgodClient from cachetools import TTLCache, cached from tinyman.assets import Asset -from tinyman.utils import TransactionGroup -from tinyman.v2.client import TinymanV2MainnetClient, TinymanV2TestnetClient, TinymanV2Client +from tinyman.v2.client import TinymanV2Client, TinymanV2MainnetClient, TinymanV2TestnetClient -from blockchain.assets import ALGO_ASA_ID, USDC_ASA_ID from blockchain.node import init_algod_client from env import settings -ASSETS_PATH = 'https://asa-list.tinyman.org/assets.json' - -TXNS_FIELD = 'txns' -SIGNED_TXNS_FIELD = 'signed_txns' -TX_ID_FIELD = 'tx_id' - - -private_key = mnemonic.to_private_key(settings.algo_mnemonic) -public_key = account.address_from_private_key(private_key) +ASSETS_PATH = "https://asa-list.tinyman.org/assets.json" logger = logging.getLogger(__name__) -def tinyman_from_algod(algod_client: AlgodClient, address: Optional[str] = public_key) -> TinymanV2Client: +def tinyman_from_algod(algod_client: AlgodClient, address: str | None = None) -> TinymanV2Client: + """Build a read-only Tinyman client for the configured Algorand network.""" + if settings.is_mainnet(): return TinymanV2MainnetClient(algod_client=algod_client, user_address=address) - else: - return TinymanV2TestnetClient(algod_client=algod_client, user_address=address) + return TinymanV2TestnetClient(algod_client=algod_client, user_address=address) -def init_tinyman_client(address: Optional[str] = public_key) -> TinymanV2Client: - algod_client = init_algod_client() - return tinyman_from_algod(algod_client, address) +def init_tinyman_client(address: str | None = None) -> TinymanV2Client: + return tinyman_from_algod(init_algod_client(), address) def get_amount(micros: int, asset: Asset) -> float: - decimals = 10 ** asset.decimals - return micros / decimals - - -def get_micros(amount: float, asset: Asset) -> int: - decimals = 10 ** asset.decimals - return int(amount * decimals) + return micros / (10**asset.decimals) -@dataclass +@dataclass(frozen=True) class PoolInfo: name: str asset1_reserve: float @@ -60,350 +41,39 @@ class PoolInfo: def get_pool_info(client: TinymanV2Client, asset1_id: int, asset2_id: int) -> PoolInfo: - # class are cached inside TinymanClient asset1 = client.fetch_asset(asset1_id) asset2 = client.fetch_asset(asset2_id) - pool = client.fetch_pool(asset1, asset2) - logger.debug(f'Found pool for assets {asset1_id} and {asset2_id}: {pool}') + logger.debug("Found Tinyman pool for assets %s and %s", asset1_id, asset2_id) if pool.asset_1_reserves is None or pool.asset_2_reserves is None or pool.issued_pool_tokens is None: - raise ValueError(f'For assests {asset1_id} and {asset2_id} pool is empty:\n{pool}') + raise ValueError(f"Tinyman pool for assets {asset1_id} and {asset2_id} is empty") - asset_1_reserve = get_amount(pool.asset_1_reserves, pool.asset_1) - asset_2_reserve = get_amount(pool.asset_2_reserves, pool.asset_2) + asset1_reserve = get_amount(pool.asset_1_reserves, pool.asset_1) + asset2_reserve = get_amount(pool.asset_2_reserves, pool.asset_2) total_lp_tokens = get_amount(pool.issued_pool_tokens, pool.pool_token_asset) - # Because Tinyman SDK swap them inside Pool + # Tinyman orders pool assets by ID rather than by the caller's argument order. if asset1_id < asset2_id: - tmp = asset_1_reserve - asset_1_reserve = asset_2_reserve - asset_2_reserve = tmp - - return PoolInfo(pool.pool_token_asset.name, asset_1_reserve, asset_2_reserve, total_lp_tokens) - - -def get_price(client: TinymanV2Client, asset_id: int) -> float: - if asset_id == ALGO_ASA_ID: - return 1 - ALGO = client.fetch_asset(ALGO_ASA_ID) - asset = client.fetch_asset(asset_id) - pool = client.fetch_pool(asset, ALGO) - return pool.asset1_price - - -def get_asset_swap_pool(client, asset1_id, asset2_id, asset1_amount, slippage: float): - asset1 = client.fetch_asset(asset1_id) - asset2 = client.fetch_asset(asset2_id) - pool = client.fetch_pool(asset1, asset2) - - quote = pool.fetch_fixed_input_swap_quote(asset1(get_micros(asset1_amount, asset1)), slippage=slippage) - - return pool, quote - - -def get_asset_swap_cost(client, asset1_id, asset2_id, asset1_amount, slippage: float): - asset1 = client.fetch_asset(asset1_id) - asset2 = client.fetch_asset(asset2_id) - pool, quote = get_asset_swap_pool(client, asset1_id, asset2_id, asset1_amount, slippage) - - decimal1 = 10 ** asset1.decimals - decimal2 = 10 ** asset2.decimals + asset1_reserve, asset2_reserve = asset2_reserve, asset1_reserve - price_per_token = quote.price * decimal1 / decimal2 - res_tokens = price_per_token * asset1_amount - - return float(res_tokens) - - -def get_optin_transactions(client, asset_id, optin_client=True): - optin_txns = [] - suggested_params = client.algod.suggested_params() - if optin_client and not client.is_opted_in(): - txn = ApplicationOptInTxn( - sender=client.user_address, - sp=suggested_params, - index=client.validator_app_id, - ) - optin_txns.append(txn) - - if asset_id > 0 and not client.asset_is_opted_in(asset_id): - txn = AssetOptInTxn( - sender=client.user_address, - sp=suggested_params, - index=asset_id, - ) - optin_txns.append(txn) - - transaction_group = TransactionGroup(optin_txns) - tx_id = '' - if len(transaction_group.transactions) > 0: - tx_id = transaction_group.transactions[0].get_txid() - - return encode_transactions(transaction_group.transactions), tx_id - - -def check_optin(client: TinymanV2Client, asset_id: int, user_address: str): - if not client.is_opted_in(user_address): - print('Account not opted into app, opting in now..') - transaction_group = client.prepare_asset_optin_transactions(asset_id=asset_id, user_address=user_address) - transaction_group.sign_with_private_key(public_key, private_key) - result = client.submit(transaction_group, wait=True) - - if not client.asset_is_opted_in(asset_id, user_address): - print(f'Account not opted into asset {asset_id}, opting in now..') - transaction_group = client.prepare_asset_optin_transactions(asset_id, user_address) - transaction_group.sign_with_private_key(public_key, private_key) - result = client.submit(transaction_group, wait=True) - - -def encode_transactions(transactions): - encode_trans = [] - for txn in transactions: - if txn: - txn = encoding.msgpack_encode(txn) - encode_trans.append(txn) - else: - encode_trans.append([]) - return encode_trans - - -def get_swap_asset_transactions(client, asset1_id, asset2_id, asset1_amount, slippage: float): - pool, quote = get_asset_swap_pool(client, asset1_id, asset2_id, asset1_amount, slippage) - transaction_group = pool.prepare_swap_transactions_from_quote(quote) - - tx_id = transaction_group.transactions[0].get_txid() - - encoded_transactions = encode_transactions(transaction_group.transactions) - encoded_signed_transactions = encode_transactions(transaction_group.signed_transactions) - - return encoded_transactions, encoded_signed_transactions, tx_id - - -def get_fee_transaction(client, address, fee): - receiver = 'METAZZXDNBTZSI5PQORZD3Z7GMDXGJBZ3ZZXWS43KARTPGV4ZDOIWQPIF4' - - suggested_params = client.algod.suggested_params() - fee = int(fee * 10 ** 6) - - return PaymentTxn( - sender=address, - sp=suggested_params, - receiver=receiver, - amt=fee, - note='fee', + return PoolInfo( + name=pool.pool_token_asset.name, + asset1_reserve=asset1_reserve, + asset2_reserve=asset2_reserve, + total_lp_tokens=total_lp_tokens, ) -def get_swap_data(client, token1_id, token2_id, token1_amount, slippage: float): - asset1 = client.fetch_asset(token1_id) - asset2 = client.fetch_asset(token2_id) - - best_tokens, best_path = 0, [] - best_path.append({ - 'asset_id': token1_id, - 'unit_name': asset1.unit_name, - 'amount': token1_amount - }) - - # SWAP TOKEN1-TOKEN2 - try: - direct_tokens = get_asset_swap_cost(client, token1_id, token2_id, token1_amount, slippage) - best_tokens = direct_tokens - except: - direct_tokens = 0 - - # SWAP TOKEN1-ALGO-TOKEN2 - try: - algos = get_asset_swap_cost(client, token1_id, ALGO_ASA_ID, token1_amount, slippage) - # transactions commissions - algos -= 0.002 * 2 - res = get_asset_swap_cost(client, ALGO_ASA_ID, token2_id, algos, slippage) - if res > best_tokens: - best_tokens = res - best_path.append({ - 'asset_id': ALGO_ASA_ID, - 'unit_name': 'ALGO', - 'amount': algos - }) - except: - pass - - best_path.append({ - 'asset_id': token2_id, - 'unit_name': asset2.unit_name, - 'amount': best_tokens - }) - - # SWAP DIFF IN USDC - try: - algos_diff = get_asset_swap_cost(client, token2_id, ALGO_ASA_ID, max(0, best_tokens - direct_tokens), slippage) - usdc_diff = get_asset_swap_cost(client, ALGO_ASA_ID, USDC_ASA_ID, algos_diff, slippage) - except: - usdc_diff = 0 - - return { - 'best_swap': best_tokens, - 'best_path': best_path, - 'direct_swap': direct_tokens, - 'usdc_diff': usdc_diff, - } - - -def zap(client: TinymanV2Client, user_address: str, asset_id: int, microalgos: int) -> dict: - ALGO = client.fetch_asset(ALGO_ASA_ID) - asset2 = client.fetch_asset(asset_id) - pool = client.fetch_pool(ALGO, asset2) - - check_optin(client, asset_id, user_address) - - half = microalgos // 2 - # TODO: set slippage - quote = pool.fetch_fixed_input_swap_quote(ALGO(half), slippage=0.01) - print(quote) - print(f'price={quote.price}, amount_in={quote.amount_in}, amount_out={quote.amount_out}') - - transaction_group = pool.prepare_swap_transactions_from_quote(quote) - transaction_group.sign_with_private_key(public_key, private_key) - swap_tx = client.submit(transaction_group, wait=True) - print(f'Swapped with: {swap_tx}') - - check_optin(client, pool.liquidity_asset.id, user_address) - - quote = pool.fetch_mint_quote(ALGO(half), slippage=0.01) - print(quote) - transaction_group = pool.prepare_mint_transactions_from_quote(quote) - transaction_group.sign_with_private_key(public_key, private_key) - add_liquidity_tx = client.submit(transaction_group, wait=True) - print(f'Added liquidity with: {add_liquidity_tx}') - - info = pool.fetch_pool_position() - share = info['share'] * 100 - print(f'Pool Tokens: {info[pool.liquidity_asset]}') - print(f'Assets: {info[asset2]}, {info[ALGO]}') - print(f'Share of pool: {share:.3f}%') - - return {'added_lp_tokens': info[pool.liquidity_asset]} - - -def get_swap_transactions(client, asset1_id, asset2_id, asset1_amount, slippage: float): - transactions = [] - tx_id = '' - optin_transactions, tx_id = get_optin_transactions(client, asset2_id) - if len(optin_transactions) > 0: - transactions.append({ - TXNS_FIELD: optin_transactions, - SIGNED_TXNS_FIELD: ['' for _ in range(len(optin_transactions))], - TX_ID_FIELD: tx_id - }) - - best_tokens_swap = get_swap_data(client, asset1_id, asset2_id, asset1_amount, slippage) - for num, token in enumerate(best_tokens_swap['best_path'][:-1]): - cur_asset_id = token['asset_id'] - cur_asset_amount = token['amount'] - next_asset_id = best_tokens_swap['best_path'][num + 1]['asset_id'] - - # if we swap through algo then pay commission - # if cur_asset_id == 0 and len(best_tokens_swap['best_path']) > 2: - # algo_amount = cur_asset_amount - # TODO: fix calculation (Y - X) * 10% * A / Y - # fee_amount = algo_amount * 0.01 - # cur_asset_amount -= fee_amount - # fee_txn = get_fee_transaction(client, address, fee_amount) - # encoded_fee_txn = encode_transactions([fee_txn]) - # transactions.append({ - # TXNS_FIELD: encoded_fee_txn, - # SIGNED_TXNS_FIELD: [[]] - # }) - - swap_transactions, swap_signed_transactions, tx_id = get_swap_asset_transactions( - client, cur_asset_id, next_asset_id, cur_asset_amount, slippage) - transactions.append({ - TXNS_FIELD: swap_transactions, - SIGNED_TXNS_FIELD: swap_signed_transactions, - TX_ID_FIELD: tx_id - }) - - return { - 'transactions': transactions, - 'tx_id': tx_id - } - - -def get_zap_pool(client, asset1_id, asset2_id, asset1_amount, slippage: float): - asset1 = client.fetch_asset(asset1_id) - asset2 = client.fetch_asset(asset2_id) - pool = client.fetch_pool(asset1, asset2) - quote = pool.fetch_mint_quote(asset1(get_micros(asset1_amount, asset1)), slippage=slippage) - - return asset1, asset2, pool, quote - - -def get_zap_data(client, asset1_id, asset2_id, asset1_amount, swap_half, slippage: float): - asset1_amount = asset1_amount / 2 if swap_half else asset1_amount - asset1, asset2, pool, quote = get_zap_pool(client, asset1_id, asset2_id, asset1_amount, slippage) - pool_lp_id = pool.liquidity_asset.id - - asset2_amount = get_amount(quote.amounts_in[asset2].amount, asset2) - lp_amount = get_amount(quote.liquidity_asset_amount.amount, quote.liquidity_asset_amount.asset) - print(asset2_amount, lp_amount) - - return { - 'asset1_amount': asset1_amount, - 'asset2_amount': asset2_amount, - 'lp_amount': lp_amount, - 'pool_lp_id': pool_lp_id - } - - -def get_zap_transactions(client, asset1_id, asset2_id, asset1_amount, swap_half, slippage: float): - asset1_amount = asset1_amount / 2 if swap_half else asset1_amount - asset2_amount = get_swap_data(client, asset1_id, asset2_id, asset1_amount, slippage)['best_swap'] - # TODO: easy fix - asset2_amount *= (1 - slippage - 0.01) - asset1, asset2, pool, quote = get_zap_pool(client, asset2_id, asset1_id, asset2_amount, slippage) - pool_lp_id = pool.liquidity_asset.id - - transactions = [] - - if swap_half: - swap_transactions = get_swap_transactions(client, asset1_id, asset2_id, asset1_amount, slippage) - transactions = swap_transactions['transactions'] - - optin_transactions, tx_id = get_optin_transactions(client, pool_lp_id, False) - if len(optin_transactions) > 0: - transactions.append({ - TXNS_FIELD: optin_transactions, - SIGNED_TXNS_FIELD: ['' for _ in range(len(optin_transactions))], - TX_ID_FIELD: tx_id - }) - - transaction_group = pool.prepare_mint_transactions_from_quote(quote) - - tx_id = transaction_group.transactions[0].get_txid() - encoded_transactions = encode_transactions(transaction_group.transactions) - encoded_signed_transactions = encode_transactions(transaction_group.signed_transactions) - - transactions.append({ - TXNS_FIELD: encoded_transactions, - SIGNED_TXNS_FIELD: encoded_signed_transactions, - TX_ID_FIELD: tx_id - }) - - return { - 'transactions': transactions, - 'tx_id': tx_id - } - - -# TODO: move asset infos to DB @cached(cache=TTLCache(maxsize=1, ttl=settings.asset_prices_ttl)) -def get_all_assets(): - with urllib.request.urlopen(ASSETS_PATH, timeout=30) as url: - return json.loads(url.read().decode()) +def get_all_assets() -> dict[str, dict]: + with urllib.request.urlopen(ASSETS_PATH, timeout=30) as response: + payload = json.loads(response.read().decode()) + if not isinstance(payload, dict): + raise ValueError("Tinyman asset registry must be a JSON object") + return payload -def get_asset_info(asset_id: int) -> Optional[dict]: +def get_asset_info(asset_id: int) -> dict | None: return get_all_assets().get(str(asset_id)) - diff --git a/docker-compose.yml b/docker-compose.yml index 9149235b..11d7adad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,7 @@ services: mongodb: # Set MONGODB_IMAGE to the exact version already validated against the # existing data directory before changing it in production. - image: "${MONGODB_IMAGE:-mongo}" + image: "${MONGODB_IMAGE:-mongo:7.0.28@sha256:4510cf3d7050003e958745adb25d2deb3fb907430716162d9cc1a92eda2a6047}" restart: unless-stopped command: mongod --port ${MONGODB_PORT:-27017} volumes: @@ -51,7 +51,7 @@ services: start_period: 20s algod: - image: "${ALGOD_IMAGE:-algorand/algod:latest}" + image: "${ALGOD_IMAGE:-algorand/algod@sha256:9c667451da575abcf325c631bd24f845f051122e689628eba1c83daa3ebd3cbc}" restart: unless-stopped environment: NETWORK: "${ALGO_NETWORK:-mainnet}" diff --git a/docs/architecture/lp-projection.md b/docs/architecture/lp-projection.md index 42ea0e05..d7b8b6d2 100644 --- a/docs/architecture/lp-projection.md +++ b/docs/architecture/lp-projection.md @@ -13,7 +13,7 @@ Each scoped transfer has a fixed-width cursor: ``` Events are sorted by that cursor. One `find_one_and_update` atomically applies -the Decimal128 balance delta and advances `last_event_order`. The audit marker +the Decimal128 balance delta and advances `last_event_order`. The event marker is inserted afterward. If the process dies in that gap, replay sees the cursor, repairs the missing marker, and does not apply the delta again. @@ -58,7 +58,7 @@ same snapshot round require reconciliation. Snapshots update only integer ledger fields and cursors: they never calculate or publish a price. Duplicate holdings and out-of-range values are rejected at the Indexer adapter; issued supply is range-checked before subtraction. Refresh builds an immutable -candidate state, and the repository repeats validation before issuing its +proposed state, and the repository repeats validation before issuing its single-document CAS, so a failed snapshot cannot partially mutate in-memory or persisted balances. @@ -68,8 +68,8 @@ can all be present. The ledger projector therefore has no price-publishing dependency. Public LP prices come from the separately validated `asset_prices` read model with source and freshness checks. The raw-balance publisher has been removed; startup deletes its identifiable legacy rows and read paths -independently reject them. A future DEX adapter must derive economic reserves -from verified protocol state before it can become a price source. +independently reject them. Only DEX adapters that derive economic reserves from +verified protocol state may publish an LP price. ## Worker ordering diff --git a/docs/architecture/native-reach-state.md b/docs/architecture/native-reach-state.md index fe6941ff..da338d78 100644 --- a/docs/architecture/native-reach-state.md +++ b/docs/architecture/native-reach-state.md @@ -39,6 +39,7 @@ the API and frontend contract remain unchanged. Add a layout only after recording the canonical program digest, schema, page count, state size, and offsets. Cover global and local decoding, identity -rejection, malformed payloads, failure isolation, and concurrency bounds. Run -`make quality`, then compare at least one mainnet application with the previous -decoder before enabling the version. +rejection, malformed payloads, failure isolation, and concurrency bounds with +committed deterministic fixtures. Run `make quality`, then verify at least one +mainnet application against its authoritative Algorand state before enabling +the version. diff --git a/docs/architecture/outbound-asset-transfers.md b/docs/architecture/outbound-asset-transfers.md index 59a29c16..fe56a38e 100644 --- a/docs/architecture/outbound-asset-transfers.md +++ b/docs/architecture/outbound-asset-transfers.md @@ -83,7 +83,5 @@ The exact draw is marked claimed with a conditional update only after confirmation. Multiple workers may race safely: they resolve to the same persisted transaction and cannot update another draw. -Lottery inventory itself is still a separate authority boundary. A future -re-enabled lottery must reserve each one-of-one NFT atomically before exposing -the feature; the public lottery routes remain disabled until that control and -product reconciliation are reviewed. +Lottery inventory is a separate authority boundary. Public lottery routes are +disabled because one-of-one NFT inventory is not yet reserved atomically. diff --git a/docs/audit/01-audit-architecture-financial-2026-07-19.md b/docs/audit/01-audit-architecture-financial-2026-07-19.md deleted file mode 100644 index 9fff7df6..00000000 --- a/docs/audit/01-audit-architecture-financial-2026-07-19.md +++ /dev/null @@ -1,175 +0,0 @@ -# Мультиагентный аудит архитектуры и финансовой корректности - -**Дата:** 2026-07-19 -**База:** состояние `main` до ветки `audit/financial-correctness`; история -репозитория затем была очищена, поэтому старые SHA намеренно не используются. - -## Резюме - -Аудит проводился как review production fintech backend, а не косметическая -подготовка профиля. Независимые потоки проверяли выплаты, MongoDB concurrency, -LP accounting, price provenance, Algorand boundaries, supply chain, Git history, -CI и публичную документацию. Повторяющиеся находки перепроверялись тестами и -исправлялись только после воспроизведения. - -Ключевой результат: денежные операции используют integer base units, -неизменяемые business IDs, persisted signed intents и одно-документные CAS. -Непроверенные источники цены и staking projection остаются fail-closed. - -| Граница | Контроль | -| --- | --- | -| Выплата | exact allocation, durable intent, bounded signer fee, on-chain reconciliation | -| Staking draw | CAS entitlement, одна generation, crash-repair того же draw ID | -| LP ledger | strict uint64 input, ordered cursor, marker repair, fenced round lease | -| Цена | provenance + freshness + clock-skew guard; raw pool balances запрещены | -| Проверка | Python 3.12/3.14, 372 fast tests, 26 real-Mongo tests, 83.19% focused coverage | - -MongoDB гарантирует атомарность одной операции над одним документом; поэтому -денежные инварианты размещены внутри одного CAS aggregate, а не blind -read-modify-write. Междокументная одновременная видимость потребует replica-set -transactions. См. [atomicity](https://www.mongodb.com/docs/manual/core/write-operations-atomicity/) -и [transactions](https://www.mongodb.com/docs/manual/core/transactions/). - -## Ранжированные находки - -### 1. Critical — секрет оставался в достижимой Git history — исправлено в репозитории - -**Почему:** удаление файла из HEAD не отзывает значение и не удаляет старый -blob; его можно восстановить из любого достижимого commit. - -**Исправление:** чувствительные исторические пути удалены из всех публикуемых -refs через `git-filter-repo`; old-object и fresh-clone проверки входят в -процедуру публикации. CI сканирует все публикуемые refs digest-pinned -TruffleHog (`.github/workflows/ci.yml:84`). Ротация ранее использованного -credential остаётся обязательным внешним действием владельца. - -### 2. Critical — double-pay window, float allocation и signer fee — исправлено - -**Почему:** crash после broadcast до Mongo update допускал повторную отправку; -float shares не сохраняли бюджет; доверенный Algod мог предложить чрезмерную -комиссию. - -**Исправление:** immutable signed intent сохраняется до первого broadcast и -reconciled по txid (`flex/application/asset_transfers.py:227`). Airdrop заранее -фиксирует полный manifest, а largest-remainder allocation сохраняет каждую base -unit. Confirmed intent и complete manifest терминальны. Gateway до подписи и -повторно перед broadcast проверяет genesis, signature, lease и configured fee -floor/ceiling (`flex/blockchain/asset_transfers.py:50`). Algorand minimum fee описан в -[официальной документации](https://dev.algorand.co/concepts/transactions/fees/). - -### 3. Critical — staking lottery создавала две независимые liabilities — исправлено - -**Почему:** прежний `read recent draws → insert` позволял двум workers создать -разные draw IDs; idempotency выплаты не объединяет разные business operations. - -**Исправление:** один entitlement на `(lottery_name, wallet)` атомарно меняет -`next_eligible_at`, `generation` и active draw. Crash recovery продолжает тот же -draw и replay prize selection (`api/nft_lottery.py:275`). Гонка 32 вызовов и crash -между entitlement/draw writes проверены на настоящем MongoDB. One-of-one NFT -inventory ещё требует отдельной атомарной reservation, поэтому публичные -lottery routes остаются disabled. - -### 4. Critical — LP replay мог потерять или повторить баланс — исправлено - -**Почему:** blind RMW, stale snapshots и истёкший worker нарушали exactly-once -projection. - -**Исправление:** per-state cursor CAS, marker-last repair и re-read после -конкурентного marker (`flex/db/lp_projection.py:52`). Round commit fenced -неистёкшим lease. Fee pool-sender — отдельное replay-safe ALGO событие; -token-token operational ALGO не смешивается с economic reserves. Indexer -amounts, IDs, rounds, duplicates и snapshots проверяются до negation и Mongo -write (`flex/sync_pools.py:78`, `flex/data/lp_states.py:113`). - -### 5. Critical — raw account balance мог манипулировать LP price — исправлено - -**Почему:** donation, minimum-balance funding или protocol excess не являются -экономическими reserves DEX. - -**Исправление:** LP projector стал ledger-only; raw-balance publisher удалён. -Startup очищает обе legacy provenance signatures, а readers независимо их -отклоняют. LP registry прекращает весь refresh, если не классифицирован хотя бы -один farm stake token. Provider observation за пределами clock-skew budget -отклоняется до записи; уже сохранённое далёкое future value считается invalid -и заменяется корректной котировкой. Новый источник допускается только после -DEX-specific app state verification. Canonical supply доверяется лишь с -`total_supply_source=indexer`. - -### 6. High — shared browser key не является пользовательской авторизацией — открыто - -**Почему:** один `X-API-Key` не доказывает wallet ownership и не разделяет роли -(`core/auth.py:8`, `app.py:323`). - -**Конкретный фикс:** wallet-signature challenge с nonce, expiry и replay table; -server-to-server maintenance role с отдельным secret и audit log. До этого -shared key считается compatibility/rate-control механизмом. - -### 7. High — blocking persistence остаётся в async routes — открыто - -**Почему:** sync PyMongo/provider calls в event loop увеличивают tail latency -всех запросов при деградации Mongo или DEX (`app.py:418`, `app.py:529`). - -**Конкретный фикс:** async repository ports с timeout/cancellation; переходно — -`asyncio.to_thread` вокруг целого repository call и saturation test. - -### 8. High — legacy staking classifier не доказывает transaction group — mitigated - -**Почему:** соседний transfer без проверки app ID, selector и group order не -доказывает stake. - -**Текущий контроль:** `SYNC_STAKING_POOLS=false`, а включение отклоняется. -Следующий фикс — типизированный parser полной Algorand group и adversarial -fixtures. Это отдельно от исправленного lottery entitlement. - -### 9. High — mocks не доказывали Mongo/BSON invariants — исправлено - -**Почему:** fake collections не воспроизводят Decimal128 comparison, unique -index races, `$inc` promotion и `find_one_and_update`. - -**Исправление:** digest-pinned MongoDB CI job проверяет 26 integration scenarios: -CAS replay, marker repair, uint64 max, legacy int64 promotion, terminal payout -races, uniqueness, staking entitlement и lease fencing -(`.github/workflows/ci.yml:106`). Stable `python` context агрегирует Python -matrix, container configuration/image scan, Mongo и all-ref secret scan -(`.github/workflows/ci.yml:133`). - -### 10. Medium — runtime/container proof остаётся неполным — открыто - -**Почему:** production smoke заменяет entrypoint shell-командой, а stateful -Mongo/Algod image rollout требует отдельной data-format rehearsal. - -**Конкретный фикс:** disposable stack должен запустить штатный entrypoint, -дождаться health, проверить `/status`, SIGTERM и logs. Production digests -фиксировать только после backup/restore и downgrade rehearsal. - -## Milestones - -1. **M0 — money safety (готово):** findings 2–5; точные суммы, terminal states, - fee/network policy, CAS entitlements, strict uint64 boundaries. -2. **M1 — proof and history (готово в коде):** findings 1 и 9; Python - 3.12/3.14, real Mongo, immutable scanner, clean-history procedure. -3. **M2 — authority (следующий):** finding 6, atomic contract registration, - transactional outbox и one-of-one NFT inventory. -4. **M3 — runtime (следующий):** findings 7, 8 и 10; затем SLO, metrics и - controlled deploy rehearsal. - -Python 3.14.6 проверяется как forward-compatibility gate и является текущим -maintenance release ([Python.org](https://www.python.org/downloads/release/python-3146/)); -production-equivalent environment остаётся на Python 3.12. - -## Воспроизводимость - -```bash -make sync -make quality - -# Только disposable MongoDB, никогда production: -MONGODB_TEST_URI=mongodb://127.0.0.1:27017 \ - pipenv run pytest tests/integration -m integration -v -``` - -Отдельные чистые environments Python 3.12.8 и 3.14.6 дали одинаковый результат: -372 passed, 26 integration skipped. Все 26 integration tests прошли на -standalone MongoDB отдельно. Форматирование не включалось в findings: его -обеспечивает Ruff; review приоритизировал correctness, security, data integrity -и recovery semantics. diff --git a/env.py b/env.py index 60dde2cc..5602030d 100644 --- a/env.py +++ b/env.py @@ -25,7 +25,6 @@ class Settings(BaseSettings): server_port: int workers_num: int api_password: str - migrate: bool = False mongodb_host: str mongodb_port: int @@ -103,9 +102,8 @@ class Settings(BaseSettings): farm_creation_fee: int farm_flat_algo_creation_fee: int - # Accepted temporarily so existing deployments can roll forward before - # removing obsolete integration keys from their .env files. These values - # are intentionally unused. + # Accepted as no-op compatibility keys for existing deployment env files. + migrate: bool = False enable_js: bool = False reach_no_warn: bool = False reach_connector_mode: str = "ALGO" diff --git a/flex/__init__.py b/flex/__init__.py index 1bb593d1..aaf15f78 100644 --- a/flex/__init__.py +++ b/flex/__init__.py @@ -6,7 +6,7 @@ # Load the namespace package once so later ``flex.db.*`` imports do not replace # the public ``db`` dependency below on the parent package. -import_module(f'{__name__}.db') +import_module(f"{__name__}.db") class _LazyCometaDatabase: diff --git a/flex/application/pool_registration.py b/flex/application/pool_registration.py new file mode 100644 index 00000000..3b54b296 --- /dev/null +++ b/flex/application/pool_registration.py @@ -0,0 +1,192 @@ +import logging +from datetime import UTC, datetime + +from blockchain.node import get_current_round +from blockchain.util import date_from_block +from core.db.model import ContractInfo +from core.util import parse_bignum +from flex import db +from flex.blockchain.info import get_app_address +from flex.data.assets import get_asset_info +from flex.db.model.pools import CometaPool, FarmingPool, StakingPool + +logger = logging.getLogger(__name__) + + +class PoolIdentityConflictError(ValueError): + """A pool ID is already bound to incompatible immutable metadata.""" + + +def _is_farming_contract(contract: ContractInfo) -> bool: + return contract.type == "farm" and "dex" in (contract.metadata or {}) + + +def ensure_pool_identity_slot(contract: ContractInfo) -> None: + """Reject an ID already present in the opposite pool collection.""" + + opposite = db.staking_pools if _is_farming_contract(contract) else db.farming_pools + if opposite.exists(id=contract.id): + raise PoolIdentityConflictError(f"pool {contract.id} is already registered as the opposite pool type") + + +def _pool_identity(pool: CometaPool) -> tuple[object, ...]: + common = ( + type(pool), + pool.id, + pool.address, + pool.stake_token.id, + pool.reward_token.id, + pool.reward_amount_micros, + pool.algo_reward_amount_micros, + pool.begin_block, + pool.end_block, + pool.lock_length_blocks, + ) + if isinstance(pool, FarmingPool): + return ( + *common, + pool.first_token.id, + pool.second_token.id, + pool.dex_name, + ) + return common + + +def _validate_persisted_pool(candidate: CometaPool, stored: CometaPool) -> CometaPool: + if _pool_identity(candidate) != _pool_identity(stored): + raise PoolIdentityConflictError(f"pool {candidate.id} has incompatible persisted identity fields") + return stored + + +async def staking_pool_from_contract_info(contract_info: ContractInfo, distribution: bool = False) -> StakingPool: + # Distribution contracts historically used the same token for stake and + # reward; existing on-chain contracts retain that compatibility rule. + if distribution: + stake_token_id = contract_info.metadata.get("stake_token_id") + if stake_token_id is None: + stake_token_id = parse_bignum(contract_info.metadata["cache"]["initial"]["token"]) + stake_token_id = int(stake_token_id) + + reward_token_id = contract_info.metadata.get("reward_token_id") + if reward_token_id is None: + reward_token_id = parse_bignum(contract_info.metadata["cache"]["initial"]["token"]) + reward_token_id = int(reward_token_id) + else: + stake_token_id = contract_info.metadata.get("stake_token_id") + if stake_token_id is None: + stake_token_id = parse_bignum(contract_info.metadata["cache"]["initial"]["stakeToken"]) + stake_token_id = int(stake_token_id) + + reward_token_id = contract_info.metadata.get("reward_token_id") + if reward_token_id is None: + reward_token_id = parse_bignum(contract_info.metadata["cache"]["initial"]["rewardToken"]) + reward_token_id = int(reward_token_id) + + reward_amount_micros = parse_bignum(contract_info.metadata["cache"]["initial"]["totalRewardAmount"]) + algo_reward_amount_micros = parse_bignum(contract_info.metadata["cache"]["initial"]["totalAlgoRewardAmount"]) + + begin_block = contract_info.metadata["begin_block"] + end_block = contract_info.metadata["end_block"] + begin_date = contract_info.begin_date + end_date = contract_info.end_date + + if begin_date is None: + start_time = datetime.now(UTC).replace(tzinfo=None) + current_block = get_current_round() + begin_date = date_from_block(begin_block, current_block, start_time) + end_date = date_from_block(end_block, current_block, start_time) + + return StakingPool( + id=contract_info.id, + description=contract_info.description, + address=(await get_app_address(contract_info.id)), + stake_token=await get_asset_info(stake_token_id), + reward_token=await get_asset_info(reward_token_id), + reward_amount_micros=reward_amount_micros, + algo_reward_amount_micros=algo_reward_amount_micros, + begin_block=begin_block, + end_block=end_block, + lock_length_blocks=contract_info.metadata["lock_length_blocks"], + deploy_date=datetime.fromtimestamp(contract_info.deployed_timestamp, UTC).replace(tzinfo=None), + begin_date=begin_date, + end_date=end_date, + ) + + +async def farming_pool_from_contract_info(contract_info: ContractInfo) -> FarmingPool: + # Legacy metadata used two field naming conventions. + first_token_id = contract_info.metadata.get("asset1_id") + if first_token_id is None: + first_token_id = contract_info.metadata["asset_1_id"] + second_token_id = contract_info.metadata.get("asset2_id") + if second_token_id is None: + second_token_id = contract_info.metadata["asset_2_id"] + first_token_id = int(first_token_id) + second_token_id = int(second_token_id) + + lp_token_id = contract_info.metadata.get("stake_token_id") + if lp_token_id is None: + lp_token_id = parse_bignum(contract_info.metadata["cache"]["initial"]["stakeToken"]) + lp_token_id = int(lp_token_id) + + reward_token_id = contract_info.metadata.get("reward_token_id") + if reward_token_id is None: + reward_token_id = parse_bignum(contract_info.metadata["cache"]["initial"]["rewardToken"]) + reward_token_id = int(reward_token_id) + + reward_amount_micros = parse_bignum(contract_info.metadata["cache"]["initial"]["totalRewardAmount"]) + algo_reward_amount_micros = parse_bignum(contract_info.metadata["cache"]["initial"]["totalAlgoRewardAmount"]) + + begin_block = contract_info.metadata["begin_block"] + end_block = contract_info.metadata["end_block"] + + begin_date = contract_info.begin_date + end_date = contract_info.end_date + if begin_date is None: + start_time = datetime.now(UTC).replace(tzinfo=None) + current_block = get_current_round() + begin_date = date_from_block(begin_block, current_block, start_time) + end_date = date_from_block(end_block, current_block, start_time) + + lp_token_info = await get_asset_info(lp_token_id) + + return FarmingPool( + id=contract_info.id, + dex_name=contract_info.metadata["dex"], + description=contract_info.description, + address=(await get_app_address(contract_info.id)), + first_token=await get_asset_info(first_token_id), + second_token=await get_asset_info(second_token_id), + stake_token=lp_token_info, + reward_token=await get_asset_info(reward_token_id), + reward_amount_micros=reward_amount_micros, + algo_reward_amount_micros=algo_reward_amount_micros, + begin_block=begin_block, + end_block=end_block, + lock_length_blocks=contract_info.metadata["lock_length_blocks"], + deploy_date=datetime.fromtimestamp(contract_info.deployed_timestamp, UTC).replace(tzinfo=None), + begin_date=begin_date, + end_date=end_date, + ) + + +async def create_pool_from_contract(contract: ContractInfo) -> CometaPool: + logger.debug(f"Creating Pool from {contract.type} contract {contract.id}") + ensure_pool_identity_slot(contract) + + if contract.type == "distribution": + pool = await staking_pool_from_contract_info(contract, distribution=True) + stored = db.staking_pools.get_or_create(pool) + return _validate_persisted_pool(pool, stored) + + # contract.type == 'farm' + + if "dex" in contract.metadata: + # Legacy farm contracts with DEX metadata represent LP staking. + pool = await farming_pool_from_contract_info(contract) + stored = db.farming_pools.get_or_create(pool) + return _validate_persisted_pool(pool, stored) + + pool = await staking_pool_from_contract_info(contract) + stored = db.staking_pools.get_or_create(pool) + return _validate_persisted_pool(pool, stored) diff --git a/flex/blockchain/base.py b/flex/blockchain/base.py index 25f0cfea..22f30871 100644 --- a/flex/blockchain/base.py +++ b/flex/blockchain/base.py @@ -1,6 +1,6 @@ from datetime import timedelta -from algosdk import mnemonic, account +from algosdk import account, mnemonic from algosdk.v2client.algod import AlgodClient from algosdk.v2client.indexer import IndexerClient @@ -12,18 +12,12 @@ indexer_client: IndexerClient = IndexerClient( indexer_token=settings.algod_token, indexer_address=settings.algo_indexer_address, - headers={ - 'User-Agent': 'py-algorand-sdk', - 'x-algo-api-token': settings.algod_token - } + headers={"User-Agent": "py-algorand-sdk", "x-algo-api-token": settings.algod_token}, ) algod_client: AlgodClient = AlgodClient( algod_token=settings.algod_token, algod_address=settings.algod_address, - headers={ - 'User-Agent': 'py-algorand-sdk', - 'x-algo-api-token': settings.algod_token - } + headers={"User-Agent": "py-algorand-sdk", "x-algo-api-token": settings.algod_token}, ) cometa_private_key = mnemonic.to_private_key(settings.algo_mnemonic) diff --git a/flex/data/lp_states.py b/flex/data/lp_states.py index 908825a2..c7fc0a3b 100644 --- a/flex/data/lp_states.py +++ b/flex/data/lp_states.py @@ -37,7 +37,7 @@ async def create_lp_state_by_lp_token_id(lp_token_id: int) -> LpState: async def create_lp_state_by_lp_token(lp_token: LpToken) -> LpState: - # TODO: remove after test + # Canonical LP identity stores native ALGO as the second asset. if lp_token.asset1_id == 0: raise MetaError(f"ALGO LP token asset1 ID = 0, not id2: {lp_token}") @@ -51,7 +51,7 @@ async def create_lp_state_by_lp_token(lp_token: LpToken) -> LpState: asset2_reserve_micros = balances[lp_token.asset2_id] lp_token_reserve_micros = balances[lp_token.id] - # TODO: add field total_supply_micros to LP token + # Supply is Indexer-provenanced and refreshed independently of pool metadata. lp_token_total_supply_micros = await get_asset_total_supply(lp_token.id) try: total_supply_micros = require_algorand_uint64( @@ -308,9 +308,6 @@ async def get_lp_state_by_lp_token_id(lp_token_id: int) -> LpState: lp_state = db.lp_states.get_one(token_id=lp_token_id) if lp_state is None: lp_state = await create_lp_state_by_lp_token_id(lp_token_id) - # TODO: add setting: do update or not - # elif (await get_current_round()) - lp_state.last_updated_round > settings.lp_state_ttl_rounds: - # lp_state = await update_lp_state(lp_state) return lp_state diff --git a/flex/data/lp_tokens.py b/flex/data/lp_tokens.py index a042fa77..af9e81c3 100644 --- a/flex/data/lp_tokens.py +++ b/flex/data/lp_tokens.py @@ -52,7 +52,6 @@ async def lp_token_from_tinyman_pool(tinyman_pool: TinymanPoolInfo) -> LpToken: async def fetch_lp_token_strong(lp_token_id: int, asset1_id: int, asset2_id: int, dex_provider: str) -> LpToken: - # TODO: refactor if dex_provider == DexProvider.PACT: pact_pool = await get_pact_pool_info(asset1_id, asset2_id, lp_token_id) if pact_pool is not None: diff --git a/flex/data/pool_state.py b/flex/data/pool_state.py index 52b9916c..fbbccd69 100644 --- a/flex/data/pool_state.py +++ b/flex/data/pool_state.py @@ -16,6 +16,19 @@ logger = logging.getLogger(__name__) +class PoolStateIdentityConflictError(ValueError): + """A pool state business key is bound to another immutable identity.""" + + +def _pool_state_identity(state: PoolState) -> tuple[object, ...]: + return ( + state.pool_id, + state.type, + state.address, + state.stake_token.id, + ) + + def get_pool_type_from_contract(contract: ContractInfo) -> PoolType: if contract.type == "distribution": return PoolType.STAKING @@ -39,20 +52,23 @@ async def create_pool_state_from_contract(contract: ContractInfo) -> PoolState: stake_token=pool.stake_token, ) - db.pool_states.create(pool_state) - logger.info(f"Created new pool state: address = {pool_state.address}") - return pool_state + stored = db.pool_states.get_or_create_by(pool_state, pool_id=pool_state.pool_id) + if _pool_state_identity(stored) != _pool_state_identity(pool_state): + raise PoolStateIdentityConflictError( + f"pool state {pool_state.pool_id} has incompatible persisted identity fields" + ) + logger.info(f"Ensured pool state exists: address = {stored.address}") + return stored @cached(ttl=30) async def get_or_create_pool_state(pool_id: int) -> PoolState: - pool_state = db.pool_states.get_one(pool_id=pool_id) - if pool_state is None: - contract_info = get_contract(pool_id) - if contract_info is None: - raise MetaError(f"Pool {pool_id} contract not found") - pool_state = await create_pool_state_from_contract(contract_info) - return pool_state + contract_info = get_contract(pool_id) + if contract_info is None: + raise MetaError(f"Pool {pool_id} contract not found") + # Always rebuild the immutable candidate. The atomic upsert then validates + # an existing state instead of trusting a matching business key alone. + return await create_pool_state_from_contract(contract_info) def get_user_state_pool(user_state: UserState, pool_state: PoolState) -> UserPoolState: @@ -97,7 +113,7 @@ async def update_pool_states_with_transactions( pool_state_by_id = {pool_state.pool_id: pool_state for pool_state in pool_states or []} user_state_by_address = {} - # TODO: remove after migration DIRTY HACKS + # Reset migrations may discover users before their first replayed event. if reset_pool_states: all_user_addresses = {user_state.address for user_state in db.user_states.get_all()} tx_addresses = {tx.user_address for tx in transactions} @@ -227,8 +243,9 @@ async def update_pool_state_by_id(pool_id: int) -> PoolState: @cached(ttl=30) async def get_or_create_user_state(address: str) -> UserState: - user_state = db.user_states.get_one(address=address) - if user_state is None: - user_state = db.user_states.create(UserState(address=address)) - logger.info(f"Created new user state: address = {user_state.address}") + user_state = db.user_states.get_or_create_by( + UserState(address=address), + address=address, + ) + logger.debug(f"Ensured user state exists: address = {user_state.address}") return user_state diff --git a/flex/data/pool_state_priced.py b/flex/data/pool_state_priced.py index b26ab5b6..ad22db0e 100644 --- a/flex/data/pool_state_priced.py +++ b/flex/data/pool_state_priced.py @@ -4,15 +4,15 @@ from flex import db from flex.blockchain.base import BLOCKS_IN_A_YEAR from flex.data.pools import get_pool_info_by_id -from flex.db.model.pool_states import UserState, PoolState +from flex.db.model.pool_states import PoolState, UserState from flex.db.model.pools import PoolType +from flex.db.model.priced import PoolStateCost, UserCost, UserPoolCost from flex.providers.tinyman import get_tinyman_pool_info -from flex.providers.vestige import get_asset_price_usd, get_algo_price_usd -from flex.db.model.priced import UserCost, PoolStateCost, UserPoolCost +from flex.providers.vestige import get_algo_price_usd, get_asset_price_usd from flex.util import build_key_str -@cached(ttl=settings.asset_prices_ttl, namespace='lp_price', key_builder=build_key_str) +@cached(ttl=settings.asset_prices_ttl, namespace="lp_price", key_builder=build_key_str) async def get_lp_price_usd(asset1_id: int, asset2_id: int) -> float | None: tinyman_pool = await get_tinyman_pool_info(asset1_id, asset2_id) if tinyman_pool.total_lp_tokens == 0: @@ -28,7 +28,7 @@ async def get_lp_price_usd(asset1_id: int, asset2_id: int) -> float | None: async def calculate_lp_token_price_usd(lp_token_id: int) -> float: lp_token = db.lp_tokens.get_one(id=lp_token_id) if lp_token is None: - raise ValueError(f'LP token {lp_token_id} not recorded in DB') + raise ValueError(f"LP token {lp_token_id} not recorded in DB") return await get_lp_price_usd(lp_token.asset1_id, lp_token.asset2_id) @@ -45,14 +45,12 @@ async def calculate_pool_state_cost(pool_state: PoolState) -> PoolStateCost: rewards_usd = pool_info.reward_amount * reward_token_price_usd algo_rewards_usd = pool_info.algo_reward_amount * (await get_algo_price_usd()) total_rewards_usd = rewards_usd + algo_rewards_usd - current_apr = total_rewards_usd / staked_usd * 100 * BLOCKS_IN_A_YEAR / pool_info.length_blocks if staked_usd > 0 else 0 - - return PoolStateCost( - info=pool_state.to_info(), - staked_usd=staked_usd, - current_apr=current_apr + current_apr = ( + total_rewards_usd / staked_usd * 100 * BLOCKS_IN_A_YEAR / pool_info.length_blocks if staked_usd > 0 else 0 ) + return PoolStateCost(info=pool_state.to_info(), staked_usd=staked_usd, current_apr=current_apr) + async def calculate_user_pool_state_cost(user_state: UserState) -> UserCost: user_cost = UserCost(address=user_state.address) @@ -64,8 +62,5 @@ async def calculate_user_pool_state_cost(user_state: UserState) -> UserCost: stake_token_price = await get_asset_price_usd(pool_info.stake_token.id) staked_usd = stake_token_price * user_pool_state.staked_amount user_cost.total_staked_usd += staked_usd - user_cost.pools_by_id[pool_info.id] = UserPoolCost( - pool_info=pool_info, - staked_usd=staked_usd - ) + user_cost.pools_by_id[pool_info.id] = UserPoolCost(pool_info=pool_info, staked_usd=staked_usd) return user_cost diff --git a/flex/data/pools.py b/flex/data/pools.py index 9176a8cc..19754a11 100644 --- a/flex/data/pools.py +++ b/flex/data/pools.py @@ -9,8 +9,8 @@ logger = logging.getLogger(__name__) -# TODO: replace the two-collection lookup with a unified pool repository. -@cached(namespace='pool_info_by_id', key_builder=build_key_str) # pool info is almost never updated +# Pool types remain in separate legacy collections. +@cached(namespace="pool_info_by_id", key_builder=build_key_str) # pool info is almost never updated async def get_pool_info_by_id(pool_id: int) -> PoolInfo: pool = db.staking_pools.get_by_primary_key(pool_id, throw_ex=False) if pool is not None: @@ -20,7 +20,7 @@ async def get_pool_info_by_id(pool_id: int) -> PoolInfo: if pool is not None: return pool.to_info() - raise ValueError(f'Pool {pool_id} not found') + raise ValueError(f"Pool {pool_id} not found") async def get_pools_by_query(query_dict: dict, pool_type: PoolType = PoolType.ANY) -> list[PoolInfo]: @@ -29,7 +29,9 @@ async def get_pools_by_query(query_dict: dict, pool_type: PoolType = PoolType.AN elif pool_type == PoolType.STAKING: return [pool.to_info() for pool in db.staking_pools.get_many(**query_dict)] else: - return [pool.to_info() for pool in db.staking_pools.get_many(**query_dict)] + [pool.to_info() for pool in db.farming_pools.get_many(**query_dict)] + return [pool.to_info() for pool in db.staking_pools.get_many(**query_dict)] + [ + pool.to_info() for pool in db.farming_pools.get_many(**query_dict) + ] async def get_all_pools() -> list[PoolInfo]: diff --git a/flex/data/stats.py b/flex/data/stats.py index eca934bc..81981075 100644 --- a/flex/data/stats.py +++ b/flex/data/stats.py @@ -24,7 +24,7 @@ async def calculate_total_tvl_usd_for_type(type: PoolType) -> float: if pool_token_price_usd is None: pool_token_price_usd = (await get_asset_price(pool.stake_token.id)).price_usd if pool.total_staked < 0: - logger.warning(f'Negative total_staked in pool {pool.pool_id}: {pool.total_staked}') + logger.warning(f"Negative total_staked in pool {pool.pool_id}: {pool.total_staked}") total_usd += pool.total_staked * pool_token_price_usd return total_usd @@ -32,8 +32,4 @@ async def calculate_total_tvl_usd_for_type(type: PoolType) -> float: async def calculate_total_tvl_usd() -> dict: farm_tvl = await calculate_total_tvl_usd_for_type(PoolType.FARMING) stake_tvl = await calculate_total_tvl_usd_for_type(PoolType.STAKING) - return { - 'farming': farm_tvl, - 'staking': stake_tvl, - 'total': farm_tvl + stake_tvl - } + return {"farming": farm_tvl, "staking": stake_tvl, "total": farm_tvl + stake_tvl} diff --git a/flex/db/classes/collection_manager.py b/flex/db/classes/collection_manager.py index 50ff028d..8db594f6 100644 --- a/flex/db/classes/collection_manager.py +++ b/flex/db/classes/collection_manager.py @@ -53,7 +53,17 @@ def get_by_primary_key(self, val: Any, throw_ex: bool = True) -> EntityT | None: def get_or_create(self, item: EntityT) -> EntityT: """Use one upsert operation; concurrent uniqueness requires a primary-key index.""" - query = self.elem_type.encode_query({self.primary_key_name: item.primary_key}) + return self.get_or_create_by( + item, + **{self.primary_key_name: item.primary_key}, + ) + + def get_or_create_by(self, item: EntityT, **identity: Any) -> EntityT: + """Atomically create an entity using a uniquely indexed business identity.""" + + if not identity: + raise ValueError("get_or_create_by requires at least one identity field") + query = self.elem_type.encode_query(identity) try: document = self.mongodb_collection.find_one_and_update( query, @@ -66,7 +76,7 @@ def get_or_create(self, item: EntityT) -> EntityT: if document is None: raise if document is None: - raise DbError(code=500, message=f"Failed to read {self.name} after upsert") + raise DbError(code=500, message=f"Failed to read {self.name} after business-key upsert") return self.item_from_dict(document) def get_or_create_with(self, **kwargs) -> EntityT: diff --git a/flex/db/cometa_database.py b/flex/db/cometa_database.py index a8299a81..72fae0d9 100644 --- a/flex/db/cometa_database.py +++ b/flex/db/cometa_database.py @@ -2,10 +2,10 @@ from flex.db.classes.database import EntitiesDatabase from flex.db.model.airdrop import AirdropManifest -from flex.db.model.blockchain import LpToken, Asset, PoolTransaction, SyncState, SyncBlock +from flex.db.model.blockchain import Asset, LpToken, PoolTransaction, SyncBlock, SyncState from flex.db.model.liquidity_pools import LpState, LpTransaction -from flex.db.model.pool_states import UserState, PoolState -from flex.db.model.pools import StakingPool, FarmingPool +from flex.db.model.pool_states import PoolState, UserState +from flex.db.model.pools import FarmingPool, StakingPool from flex.db.model.priced import AirdropReward, AssetPrice from flex.db.model.transfers import AssetTransferIntent diff --git a/flex/db/indexes.py b/flex/db/indexes.py index 7286e690..a0e32845 100644 --- a/flex/db/indexes.py +++ b/flex/db/indexes.py @@ -13,18 +13,22 @@ ("assets", False), ("asset_prices", True), ("asset_transfer_intents", False), + ("farming_pools", False), ("pool_transactions", False), ("lp_transactions", False), ("lp_tokens", False), + ("staking_pools", False), ) _HOT_INDEXES = ( ("lp_states", "token_id", "token_id_unique"), ("lp_states", "address", "address_unique"), - ("pool_states", "pool_id", "pool_id_idx"), - ("user_states", "address", "address_idx"), + ("pool_states", "pool_id", "pool_id_unique"), + ("user_states", "address", "address_unique"), ) +_UNIQUE_HOT_COLLECTIONS = frozenset({"lp_states", "pool_states", "user_states"}) + def delete_unverified_legacy_lp_prices(database: CometaDatabase) -> int: """Remove reconstructable prices produced from unauthenticated pool balances.""" @@ -182,6 +186,22 @@ def ensure_airdrop_indexes(database: CometaDatabase) -> None: ) +def ensure_pool_identity_is_disjoint(database: CometaDatabase) -> None: + """Reject an application ID assigned to both staking and farming pools.""" + + farming_ids = database.farming_pools.mongodb_collection.distinct("id") + if not farming_ids: + return + conflict = database.staking_pools.mongodb_collection.find_one( + {"id": {"$in": farming_ids}}, + projection={"_id": 0, "id": 1}, + ) + if conflict is not None: + raise RuntimeError( + f"pool ID {conflict['id']!r} exists in both staking_pools and farming_pools; reconcile it before startup" + ) + + def ensure_sync_state_singleton(database: CometaDatabase) -> None: """Migrate one legacy random-ID cursor and reject competing checkpoints.""" @@ -210,6 +230,7 @@ def ensure_database_indexes(database: CometaDatabase) -> dict[str, int]: # row cannot displace a safe provider-backed observation with the same ID. delete_unverified_legacy_lp_prices(database) ensure_airdrop_indexes(database) + ensure_pool_identity_is_disjoint(database) removed_by_collection: dict[str, int] = {"airdrop_manifests": 0} for collection_name, can_deduplicate in _UNIQUE_ID_POLICIES: @@ -230,7 +251,7 @@ def ensure_database_indexes(database: CometaDatabase) -> dict[str, int]: for manager_name, field_name, index_name in _HOT_INDEXES: manager = getattr(database, manager_name) - if manager_name == "lp_states": + if manager_name in _UNIQUE_HOT_COLLECTIONS: create_unique_field_index_fail_closed( manager.mongodb_collection, collection_name=manager_name, diff --git a/flex/db/model/liquidity_pools.py b/flex/db/model/liquidity_pools.py index 29ebde95..477c263e 100644 --- a/flex/db/model/liquidity_pools.py +++ b/flex/db/model/liquidity_pools.py @@ -94,7 +94,7 @@ class LpState( decoder=decode_bson_uint64, ) ) - # TODO: could not fit into MongoDB ??? (64 bits) + # Financial uint64 values use the model's BSON-safe codecs. asset1_reserve_micros: int = field( metadata=config( encoder=encode_bson_integer, diff --git a/flex/db/model/pool_states.py b/flex/db/model/pool_states.py index 0512bd95..6daef63a 100644 --- a/flex/db/model/pool_states.py +++ b/flex/db/model/pool_states.py @@ -3,8 +3,8 @@ from dataclasses_json import dataclass_json -from flex.db.classes.base_entity import BaseEntity, EntityT -from flex.db.model.blockchain import TxInfo, AssetInfo +from flex.db.classes.base_entity import BaseEntity +from flex.db.model.blockchain import AssetInfo, TxInfo from flex.db.model.pools import PoolType from flex.db.util import get_uuid from flex.util import format_timedelta @@ -29,7 +29,7 @@ class PoolStateInfo: @dataclass_json @dataclass -class PoolState(BaseEntity['PoolState']): +class PoolState(BaseEntity["PoolState"]): pool_id: int type: PoolType stake_token: AssetInfo @@ -61,7 +61,7 @@ def last_tx_round(self) -> int | None: def total_staked(self) -> float: return self.stake_token.micros_to_amount(self.total_staked_micros) - def to_info(self, now: datetime| None = None) -> PoolStateInfo: + def to_info(self, now: datetime | None = None) -> PoolStateInfo: now = now or datetime.now() return PoolStateInfo( pool_id=self.pool_id, @@ -73,9 +73,10 @@ def to_info(self, now: datetime| None = None) -> PoolStateInfo: total_staked=self.total_staked, updated_round=self.last_tx_round, since_update=now - self.updated, - staked_micros_by_address=self.staked_micros_by_address + staked_micros_by_address=self.staked_micros_by_address, ) + @dataclass_json @dataclass class UserPoolStateInfo: @@ -121,7 +122,7 @@ class UserStateInfo: @dataclass_json @dataclass -class UserState(BaseEntity['UserState']): +class UserState(BaseEntity["UserState"]): address: str pool_by_address: dict[str, UserPoolState] = field(default_factory=dict) last_tx: TxInfo | None = None @@ -135,6 +136,5 @@ def to_info(self, now: datetime | None = None) -> UserStateInfo: address=self.address, pools={pool.pool_id: pool.to_info() for pool in self.pool_by_address.values()}, updated_round=self.last_tx.confirmed_round if self.last_tx else None, - since_update=format_timedelta(now - self.updated) + since_update=format_timedelta(now - self.updated), ) - diff --git a/flex/db/redis.py b/flex/db/redis.py deleted file mode 100644 index 9a81f424..00000000 --- a/flex/db/redis.py +++ /dev/null @@ -1,22 +0,0 @@ -import logging -from typing import Callable, Any - -from env import settings - -from aiocache import Cache - - -logger = logging.getLogger(__name__) - -# cache = Cache(Cache.REDIS, endpoint=settings.redis_host, port=settings.redis_port, namespace="main") -# -# -# async def global_cache_get(key: str, cls, fetch_func: Callable[[], Any], ttl: int = 120) -> Any: -# cached_data = await cache.get(key) -# if cached_data: -# return cls.from_dict(cached_data) if isinstance(cached_data, dict) else cached_data -# -# data = await fetch_func() -# if data: -# await cache.set(key, data.to_dict() if hasattr(data, 'to_dict') else data, ttl=ttl) # Cache the data -# return data \ No newline at end of file diff --git a/flex/db/util.py b/flex/db/util.py index f99b3912..d4cafe4f 100644 --- a/flex/db/util.py +++ b/flex/db/util.py @@ -4,7 +4,7 @@ def string_to_snake_case(s: str) -> str: - return re.sub(r'(? str: diff --git a/flex/migrations/__init__.py b/flex/migrations/__init__.py deleted file mode 100644 index 95ca5d48..00000000 --- a/flex/migrations/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -import logging - -from env import settings -from flex.migrations.assets_set_logo_url import assets_set_logo_url_from_tinyman_info -from flex.migrations.lp_upgrades import upgrade_lp_models, set_lp_state_price_algo -from flex.migrations.pool_state_resync import remove_previous_pool_states, apply_creation_txns -from flex.migrations.reset_api_data import remove_all_new_models - -logger = logging.getLogger(__name__) - - -def migrate_before_start() -> None: - if not settings.migrate: - return - - logger.info('Migrating sync...') - remove_previous_pool_states() - logger.info('DONE sync migration.') - - -async def migrate_background() -> None: - if not settings.migrate: - return - - logger.info('Migrating ASYNC...') - # await apply_creation_txns() - logger.info('DONE ASYNC migration.') diff --git a/flex/migrations/assets_set_logo_url.py b/flex/migrations/assets_set_logo_url.py deleted file mode 100644 index 508fe431..00000000 --- a/flex/migrations/assets_set_logo_url.py +++ /dev/null @@ -1,28 +0,0 @@ -import logging - -from env import settings -from flex import db -from flex.db.model.blockchain import Asset -from flex.providers.tinyman import get_tinyman_assets_details - - -logger = logging.getLogger(__name__) - - -def assets_set_logo_url_from_tinyman_info() -> list[Asset]: - assets = db.assets.get_many(logo_url=None) - if len(assets) == 0: - logger.info('All assets have logo_url set.') - return [] - - logger.info(f'Setting logo_url for {len(assets)} assets from tinyman info.') - - tinyman_infos = get_tinyman_assets_details() - for asset in assets: - info = tinyman_infos.get(asset.id) - if info is not None: - asset.logo_url = info.logo_svg_url - else: - asset.logo_url = settings.asset_default_logo_svg_url - db.assets.update(asset) - return assets diff --git a/flex/migrations/contracts.py b/flex/migrations/contracts.py deleted file mode 100644 index d04df2c4..00000000 --- a/flex/migrations/contracts.py +++ /dev/null @@ -1,203 +0,0 @@ -import logging -from datetime import datetime - -from blockchain.node import get_current_round -from blockchain.util import date_from_block -from core.db.contracts import get_contracts_by_type -from core.db.model import ContractInfo -from core.util import parse_bignum -from flex import db -from flex.data.assets import get_asset_info -from flex.blockchain.info import get_app_address -from flex.db.model.pools import StakingPool, FarmingPool, CometaPool - -logger = logging.getLogger(__name__) - - -async def staking_pool_from_contract_info(contract_info: ContractInfo, distribution: bool = False) -> StakingPool: - # Distribution contracts historically used the same token for stake and - # reward; keep that compatibility rule explicit until old contracts migrate. - if distribution: - stake_token_id = contract_info.metadata.get('stake_token_id') - if stake_token_id is None: - stake_token_id = parse_bignum(contract_info.metadata['cache']['initial']['token']) - stake_token_id = int(stake_token_id) - - reward_token_id = contract_info.metadata.get('reward_token_id') - if reward_token_id is None: - reward_token_id = parse_bignum(contract_info.metadata['cache']['initial']['token']) - reward_token_id = int(reward_token_id) - else: - stake_token_id = contract_info.metadata.get('stake_token_id') - if stake_token_id is None: - stake_token_id = parse_bignum(contract_info.metadata['cache']['initial']['stakeToken']) - stake_token_id = int(stake_token_id) - - reward_token_id = contract_info.metadata.get('reward_token_id') - if reward_token_id is None: - reward_token_id = parse_bignum(contract_info.metadata['cache']['initial']['rewardToken']) - reward_token_id = int(reward_token_id) - - reward_amount_micros = parse_bignum(contract_info.metadata['cache']['initial']['totalRewardAmount']) - algo_reward_amount_micros = parse_bignum(contract_info.metadata['cache']['initial']['totalAlgoRewardAmount']) - - begin_block = contract_info.metadata['begin_block'] - end_block = contract_info.metadata['end_block'] - begin_date = contract_info.begin_date - end_date = contract_info.end_date - - if begin_date is None: - start_time = datetime.now() - current_block = get_current_round() - begin_date = date_from_block(begin_block, current_block, start_time) - end_date = date_from_block(end_block, current_block, start_time) - - return StakingPool( - id=contract_info.id, - description=contract_info.description, - address=(await get_app_address(contract_info.id)), - - stake_token=await get_asset_info(stake_token_id), - reward_token=await get_asset_info(reward_token_id), - - reward_amount_micros=reward_amount_micros, - algo_reward_amount_micros=algo_reward_amount_micros, - - begin_block=begin_block, - end_block=end_block, - lock_length_blocks=contract_info.metadata['lock_length_blocks'], - - deploy_date=datetime.fromtimestamp(contract_info.deployed_timestamp), - begin_date=begin_date, - end_date=end_date, - ) - - -async def farming_pool_from_contract_info(contract_info: ContractInfo) -> FarmingPool: - # TODO: migrations could be so beautiful and elegant - first_token_id = contract_info.metadata.get('asset1_id') - if first_token_id is None: - first_token_id = contract_info.metadata['asset_1_id'] - second_token_id = contract_info.metadata.get('asset2_id') - if second_token_id is None: - second_token_id = contract_info.metadata['asset_2_id'] - first_token_id = int(first_token_id) - second_token_id = int(second_token_id) - - lp_token_id = contract_info.metadata.get('stake_token_id') - if lp_token_id is None: - lp_token_id = parse_bignum(contract_info.metadata['cache']['initial']['stakeToken']) - lp_token_id = int(lp_token_id) - - reward_token_id = contract_info.metadata.get('reward_token_id') - if reward_token_id is None: - reward_token_id = parse_bignum(contract_info.metadata['cache']['initial']['rewardToken']) - reward_token_id = int(reward_token_id) - - reward_amount_micros = parse_bignum(contract_info.metadata['cache']['initial']['totalRewardAmount']) - algo_reward_amount_micros = parse_bignum(contract_info.metadata['cache']['initial']['totalAlgoRewardAmount']) - - begin_block = contract_info.metadata['begin_block'] - end_block = contract_info.metadata['end_block'] - - begin_date = contract_info.begin_date - end_date = contract_info.end_date - if begin_date is None: - start_time = datetime.now() - current_block = await get_current_round() - begin_date = date_from_block(begin_block, current_block, start_time) - end_date = date_from_block(end_block, current_block, start_time) - - lp_token_info = await get_asset_info(lp_token_id) - - return FarmingPool( - id=contract_info.id, - dex_name=contract_info.metadata['dex'], - description=contract_info.description, - address=(await get_app_address(contract_info.id)), - - first_token=await get_asset_info(first_token_id), - second_token=await get_asset_info(second_token_id), - stake_token=lp_token_info, - reward_token=await get_asset_info(reward_token_id), - - reward_amount_micros=reward_amount_micros, - algo_reward_amount_micros=algo_reward_amount_micros, - - begin_block=begin_block, - end_block=end_block, - lock_length_blocks=contract_info.metadata['lock_length_blocks'], - - deploy_date=datetime.fromtimestamp(contract_info.deployed_timestamp), - begin_date=begin_date, - end_date=end_date, - ) - - -async def create_pool_from_contract(contract: ContractInfo) -> CometaPool | None: - logger.debug(f'Creating Pool from {contract.type} contract {contract.id}') - - if contract.type == 'distribution': - if db.staking_pools.exists(id=contract.id): - logger.info(f'Staking pool {contract.id} already exists in DB') - return None - - pool = await staking_pool_from_contract_info(contract, distribution=True) - db.staking_pools.create(pool) - return pool - - # contract.type == 'farm' - - if 'dex' in contract.metadata: - # ancient system: 'farm' can be staking A -> B. Do not bother refactoring if lol - if db.farming_pools.exists(id=contract.id): - logger.info(f'Farming pool {contract.id} already exists in DB') - return None - - pool = await farming_pool_from_contract_info(contract) - db.farming_pools.create(pool) - return pool - - if db.staking_pools.exists(id=contract.id): - logger.info(f'Staking pool {contract.id} already exists in DB') - return None - - pool = await staking_pool_from_contract_info(contract) - db.staking_pools.create(pool) - return pool - - -async def all_contracts_to_pools() -> dict: - created_pools = [] - - distribution_contracts = get_contracts_by_type('distribution') - logger.info(f'Migrating {len(distribution_contracts)} distribution contracts to Pools DB\n') - failed_contract_ids = [] - for contract in distribution_contracts: - try: - pool = await create_pool_from_contract(contract) - if pool is not None: - created_pools.append(pool) - except Exception as e: - logger.error(f'Failed to process contract {contract.id}: {e}\n{contract}\n', exc_info=True) - failed_contract_ids.append(contract.id) - - farm_contracts = get_contracts_by_type('farm') - logger.info(f'Migrating {len(farm_contracts)} farm contracts to Pools DB\n') - for contract in farm_contracts: - try: - pool = await create_pool_from_contract(contract) - if pool is not None: - created_pools.append(pool) - except Exception as e: - logger.error(f'Failed to process contract {contract.id}: {e}\n{contract}\n', exc_info=True) - failed_contract_ids.append(contract.id) - - logger.info(f'Created {len(created_pools)} pools') - logger.info(f'Failed to process contracts: {failed_contract_ids}') - - return { - 'created_count': len(created_pools), - 'failed_contract_ids': failed_contract_ids, - 'created_pools': created_pools, - } diff --git a/flex/migrations/fix_dex_providers.py b/flex/migrations/fix_dex_providers.py deleted file mode 100644 index 84520afe..00000000 --- a/flex/migrations/fix_dex_providers.py +++ /dev/null @@ -1,36 +0,0 @@ -import logging - -from flex import db -from flex.providers.vestige import get_dex_tag_by_name, is_valid_dex_provider - -logger = logging.getLogger(__name__) - - -def fix_dex_names() -> None: - logger.info("Fixing dex names.") - - lp_tokens = db.lp_tokens.get_all() - for lp_token in lp_tokens: - if not is_valid_dex_provider(lp_token.dex_provider): - logger.info(f"Invalid DEX in LP token:\n{lp_token.pretty_str()}") - lp_token.dex_provider = get_dex_tag_by_name(lp_token.dex_provider) - db.lp_tokens.update(lp_token) - - farming_pools = db.farming_pools.get_all() - for farming_pool in farming_pools: - if not is_valid_dex_provider(farming_pool.dex_name): - logger.info(f"Invalid DEX in farming pool:\n{farming_pool.pretty_str()}") - farming_pool.dex_name = get_dex_tag_by_name(farming_pool.dex_name) - db.farming_pools.update(farming_pool) - - lp_states = db.lp_states.get_all() - for lp_state in lp_states: - if not is_valid_dex_provider(lp_state.dex_provider): - logger.info(f"Invalid DEX in LP state:\n{lp_state.pretty_str()}") - lp_state.dex_provider = get_dex_tag_by_name(lp_state.dex_provider) - db.lp_states.update_with( - lp_state, - dex_provider=lp_state.dex_provider, - ) - - logger.info("All DEX names are valid now!") diff --git a/flex/migrations/lp_upgrades.py b/flex/migrations/lp_upgrades.py deleted file mode 100644 index 72fd4de2..00000000 --- a/flex/migrations/lp_upgrades.py +++ /dev/null @@ -1,30 +0,0 @@ -import logging - -from flex import db -from flex.data.lp_states import update_lp_state - -logger = logging.getLogger(__name__) - - -def upgrade_lp_models(): - removed_lp_states = db.lp_states.remove_by() - logger.info(f'Removed {removed_lp_states} LP states') - - -async def set_lp_state_price_algo(): - all_lp_states = db.lp_states.get_all() - logger.info(f'Setting {len(all_lp_states)} LP states a price algo') - - updated_lp_states = [] - for lp_state in all_lp_states: - try: - if lp_state.token_price_algo is not None: - continue - - logger.debug(f'Updating LP state {lp_state.id}') - lp_state = await update_lp_state(lp_state) - updated_lp_states.append(lp_state) - except Exception as e: - logger.error(f'Failed to update LP state {lp_state.id}: {e}', exc_info=True) - - logger.info(f'Updated {len(updated_lp_states)} LP states') diff --git a/flex/migrations/migrate_assets.py b/flex/migrations/migrate_assets.py deleted file mode 100644 index 51703b8f..00000000 --- a/flex/migrations/migrate_assets.py +++ /dev/null @@ -1,16 +0,0 @@ -import logging - -from flex import db - - -logger = logging.getLogger(__name__) - - -def asset_add_reserve() -> None: - logger.info('Adding reserve to assets') - - removed_cnt = db.assets.remove_all() - logger.info(f'Removed {removed_cnt} assets') - - # asset_ids = load_all_assets_data() - # logger.info(f'Loaded {len(asset_ids)} assets') diff --git a/flex/migrations/pool_state_resync.py b/flex/migrations/pool_state_resync.py deleted file mode 100644 index de14c1bf..00000000 --- a/flex/migrations/pool_state_resync.py +++ /dev/null @@ -1,65 +0,0 @@ -import logging - -from flex import db -from flex.data.pool_state import apply_creation_tx, get_or_create_user_state -from flex.data.pools import get_all_pools - -logger = logging.getLogger(__name__) - - -def remove_previous_pool_states(): - removed_pool_states = db.pool_states.remove_all() - logger.info(f'Removed {removed_pool_states} LP states') - - removed_user_states = db.user_states.remove_all() - logger.info(f'Removed {removed_user_states} User states') - - removed_pool_transactions = db.pool_transactions.remove_all() - logger.info(f'Removed {removed_pool_transactions} Pool transactions') - - -async def apply_creation_txns(): - logger.info('Applying creation txns...') - - pool_states = db.pool_states.get_all() - logger.info(f'Found {len(pool_states)} pool states') - - all_pools = await get_all_pools() - pool_info_by_id = {pool.id: pool for pool in all_pools} - - updated_pool_states = [] - for pool_state in pool_states: - try: - pool_info = pool_info_by_id.get(pool_state.pool_id) - if pool_info is None: - logger.error(f'Pool {pool_state.pool_id} not found') - continue - - if pool_info.reward_token.id != pool_state.stake_token.id: - logger.debug(f'Pool {pool_state.pool_id} is not distribution pool') - continue - - pool_state = db.pool_states.get_one(pool_id=pool_state.pool_id) - if pool_state.stake_amount_reduced_by_rewards: - logger.info(f'Already reduced the rewards {pool_state.pool_id}') - continue - - pool_txns = db.pool_transactions.get_many_by_query(query_dict={'pool_id': pool_state.pool_id}, limit=1) - if not pool_txns: - logger.error(f'Pool {pool_state.pool_id} not found') - continue - - pool_txn = pool_txns[0] - - logger.info(f'Applying creation txn for pool {pool_state.pool_id} — {pool_state.stake_token.name}: {pool_txn}') - - user_state = await get_or_create_user_state(pool_txn.user_address) - pool_state = await apply_creation_tx(pool_state, user_state, pool_txn) - db.pool_states.update(pool_state) - updated_pool_states.append(pool_state) - except Exception as e: - logger.error(f'Failed to apply creation txn for pool {pool_state.pool_id}: {e}', exc_info=True) - - logger.info(f'Updated {len(updated_pool_states)} pool states') - - return updated_pool_states diff --git a/flex/migrations/reset_api_data.py b/flex/migrations/reset_api_data.py deleted file mode 100644 index 8b2c706c..00000000 --- a/flex/migrations/reset_api_data.py +++ /dev/null @@ -1,19 +0,0 @@ -import logging - -from flex import db - -logger = logging.getLogger(__name__) - - -def remove_all_new_models(): - removed_lp_states = db.lp_states.remove_all() - logger.info(f'Removed {removed_lp_states} LP states') - - # removed_lp_tokens = db.lp_tokens.clear() - # logger.info(f'Removed {removed_lp_tokens} LP tokens') - # - # removed_assets = db.assets.clear() - # logger.info(f'Removed {removed_assets} assets') - # - # removed_asset_prices = db.asset_prices.clear() - # logger.info(f'Removed {removed_asset_prices} asset prices') diff --git a/flex/providers/tinyman.py b/flex/providers/tinyman.py index 22c6c039..3ec9d6dd 100644 --- a/flex/providers/tinyman.py +++ b/flex/providers/tinyman.py @@ -3,7 +3,7 @@ from datetime import datetime import requests -from cachetools import cached, TTLCache +from cachetools import TTLCache, cached from dataclasses_json import dataclass_json from tinyman.assets import Asset from tinyman.v2.client import TinymanV2MainnetClient, TinymanV2TestnetClient @@ -41,7 +41,7 @@ class TinymanPoolInfo: def get_amount(micros: int, asset: Asset) -> float: - decimals = 10 ** asset.decimals + decimals = 10**asset.decimals return micros / decimals @@ -51,6 +51,7 @@ async def get_tinyman_pool_info(asset1_id: int, asset2_id: int) -> TinymanPoolIn asset1_id, asset2_id = asset2_id, asset1_id import asyncio + loop = asyncio.get_running_loop() # Tinyman SDK methods do blocking HTTP — run in executor to avoid blocking the event loop @@ -59,9 +60,9 @@ async def get_tinyman_pool_info(asset1_id: int, asset2_id: int) -> TinymanPoolIn pool = await loop.run_in_executor(None, tinyman_client.fetch_pool, asset1, asset2) - logger.debug(f'Found pool for assets {asset1_id} and {asset2_id}: {pool}') + logger.debug(f"Found pool for assets {asset1_id} and {asset2_id}: {pool}") if pool.asset_1_reserves is None or pool.asset_2_reserves is None or pool.issued_pool_tokens is None: - raise ValueError(f'Tinyman pool for assets {asset1_id} and {asset2_id} is empty: {pool}') + raise ValueError(f"Tinyman pool for assets {asset1_id} and {asset2_id} is empty: {pool}") asset1_reserve = get_amount(pool.asset_1_reserves, pool.asset_1) asset2_reserve = get_amount(pool.asset_2_reserves, pool.asset_2) @@ -78,7 +79,7 @@ async def get_tinyman_pool_info(asset1_id: int, asset2_id: int) -> TinymanPoolIn asset1_reserve_micros=pool.asset_1_reserves, asset2_reserve_micros=pool.asset_2_reserves, total_lp_tokens_micros=pool.issued_pool_tokens, - address=pool.address + address=pool.address, ) @@ -87,7 +88,7 @@ async def fetch_algo_tinyman_pool_by_asset_id(asset_id: int) -> TinymanPoolInfo pool = await get_tinyman_pool_info(asset_id, 0) return pool except ValueError as e: - logger.error(f'Failed to get pool for asset {asset_id}: {e}') + logger.error(f"Failed to get pool for asset {asset_id}: {e}") return None @@ -105,19 +106,19 @@ class TinymanAssetInfo: @cached(cache=TTLCache(maxsize=1, ttl=60 * 60 * 24)) def get_tinyman_assets_details() -> dict[int, TinymanAssetInfo]: - url = 'https://asa-list.tinyman.org/assets.json' + url = "https://asa-list.tinyman.org/assets.json" response = requests.get(url, timeout=30) data = response.json() assets_info = [] for asa_id, asa_data in data.items(): asset_details = TinymanAssetInfo( id=int(asa_id), - name=asa_data['name'], - unit_name=asa_data['unit_name'], - decimals=asa_data['decimals'], - total_amount=asa_data['total_amount'], - logo_png_url=asa_data['logo']['png'], - logo_svg_url=asa_data['logo']['svg'] + name=asa_data["name"], + unit_name=asa_data["unit_name"], + decimals=asa_data["decimals"], + total_amount=asa_data["total_amount"], + logo_png_url=asa_data["logo"]["png"], + logo_svg_url=asa_data["logo"]["svg"], ) assets_info.append(asset_details) return {asset.id: asset for asset in assets_info} diff --git a/flex/sync_pools.py b/flex/sync_pools.py index 1d132073..99775d88 100644 --- a/flex/sync_pools.py +++ b/flex/sync_pools.py @@ -237,7 +237,7 @@ async def process_pool_transactions(txns: list[dict]) -> list[PoolTransaction]: return pool_transactions -async def update_pools(txns: list[dict]) -> [PoolState]: +async def update_pools(txns: list[dict]) -> list[PoolState]: pool_transactions = await process_pool_transactions(txns) return await update_pool_states_with_transactions(pool_transactions) @@ -276,8 +276,8 @@ def _snapshot_checkpoint_round( return checkpoint -async def catch_up_the_sync_manually(sync_state: SyncState, current_round: int) -> SyncState: - logger.info("\n\nMANUAL roll rock and roll BABE.\n") +async def reconcile_sync_checkpoint(sync_state: SyncState, current_round: int) -> SyncState: + logger.info("Reconciling LP snapshots before event sync") logger.info( f"Last sync round = {sync_state.last_round}, sync lag = {sync_state.rounds_since_updated(current_round)} rounds.\n" ) @@ -289,7 +289,7 @@ async def catch_up_the_sync_manually(sync_state: SyncState, current_round: int) ) current_round = await get_current_round() - logger.info(f"\n\nAnother, shorter loop, starting from round {current_round}\n") + logger.info("Starting event sync from reconciled round %s", current_round) if settings.sync_liquidity_pools: logger.info("\n\nSyncing LP states from authoritative account snapshots.\n") _ = await create_lp_states_from_all_pools() @@ -311,7 +311,7 @@ async def catch_up_the_sync_manually(sync_state: SyncState, current_round: int) expected_last_round=sync_state.last_round, round_number=checkpoint_round, ) - logger.info(f"\n\nALL synced up to round {checkpoint_round}.\n") + logger.info("Snapshot reconciliation complete through round %s", checkpoint_round) return sync_state @@ -333,7 +333,7 @@ async def sync_pools_loop(): previous_round = sync_state.last_round try: - sync_state = await catch_up_the_sync_manually(sync_state, current_round) + sync_state = await reconcile_sync_checkpoint(sync_state, current_round) except SyncCoordinatorError: # A competing worker may have completed the same authoritative cutover. refreshed = await get_sync_state() @@ -440,8 +440,4 @@ async def get_sync_pool_state_by_id(pool_id: int) -> PoolState: async def get_sync_user_state_by_address(user_address: str) -> UserState: - user_state = db.user_states.get_one(address=user_address) - # TODO: uncomment - # if is_sync_delayed(): - # user_state = await update_user_state(user_state) - return user_state + return db.user_states.get_one(address=user_address) diff --git a/flex/tdr_stats.py b/flex/tdr_stats.py deleted file mode 100644 index 042865dd..00000000 --- a/flex/tdr_stats.py +++ /dev/null @@ -1,157 +0,0 @@ -import asyncio -import json - -from flex import db -from flex.blockchain.base import indexer_client -from flex.data.asset_prices import get_asset_price -from flex.data.pools import get_pools_by_query -from flex.providers.vestige import vestige_full_asset_price_not_cached -from flex.sync_pools import find_transfer_transactions, process_pool_transactions - -# 01.02 — 30.04 - -DATE_0_BLOCK = 35720000 - -DATE_1_BLOCK = 37530000 -DATE_1_ALGO_PRICE = 0.251 - -DATE_2_BLOCK = 38496000 - - -async def calculate_pool_tvl(pool_address: str) -> int: - total_tvl = 0 - next_token = None - while True: - data = indexer_client.search_transactions_by_address( - address=pool_address, - next_page=next_token - ) - txns = data['transactions'] - print(f'Pool {pool_address}: processing {len(txns)} txns...') - - raw_txns = find_transfer_transactions(txns) - pool_txns = await process_pool_transactions(raw_txns) - - for pool_tx in pool_txns: - if pool_tx.confirmed_round < DATE_2_BLOCK: - total_tvl += pool_tx.delta_amount_micros - - if 'next-token' in data: - next_token = data['next-token'] - else: - break - - return total_tvl - - -async def get_user_transactions(pool_address: str, start_block: int, end_block: int, pool_id: int, ind: int | None = None) -> dict: - print(f'#{ind}: {pool_address}') - address_txns = {} - next_token = None - while True: - data = indexer_client.search_transactions_by_address( - address=pool_address, - next_page=next_token, - min_round=start_block, - max_round=end_block - ) - txns = data['transactions'] - print(f'Pool {pool_id}: processing {len(txns)} txns...') - - for tx in txns: - confirmed_round = tx['confirmed-round'] - if confirmed_round < start_block or confirmed_round > end_block: - continue - txid = tx['id'] - sender = tx['sender'] - address_txns.setdefault(sender, []).append(txid) - - if 'next-token' in data: - next_token = data['next-token'] - else: - break - - print(f'#{ind}: {len(address_txns)} users') - - return { - 'txns': address_txns, - 'pool_id': pool_id - } - - -async def get_all_user_transactions(start_block: int, end_block: int) -> dict[str, dict[str, list[str]]]: - farm_pools = db.farming_pools.get_all() - pool_id_address = {p.id: p.address for p in farm_pools} - staking_pools = db.staking_pools.get_all() - pool_id_address.update({p.id: p.address for p in staking_pools}) - print(f'Getting user txns from {len(pool_id_address)} pools...') - - user_txns_cors = [] - ind = 1 - for pool_id, pool_address in pool_id_address.items(): - user_pool_txns_co = get_user_transactions(pool_address, start_block, end_block, pool_id=pool_id, ind = ind) - user_txns_cors.append(user_pool_txns_co) - ind += 1 - - user_pool_txns = await asyncio.gather(*user_txns_cors) - user_txns_by_pool_id = {} - for res in user_pool_txns: - txns_dict = res['txns'] - pool_id = res['pool_id'] - for user, txns in txns_dict.items(): - user_txns_by_pool_id.setdefault(user, {})[pool_id] = len(txns) - - return user_txns_by_pool_id - - -ALGO_FEE = 100 -TOKEN_FEE_PERCENT = 0.01 - -async def calculate_cometa_fees(start_block: int, end_block: int) -> dict: - pool_fee = {} - pool_infos = await get_pools_by_query({}) - print(f'Calculate fees: in total {len(pool_infos)} pools') - - total_algo = 0 - total_token_fee_usd = 0 - for pool in pool_infos: - if pool.begin_block > start_block and pool.begin_block < end_block: - print(f'Pool {pool.id}: {pool.reward_amount} {pool.reward_token.unit_name}') - - reward_token_usd = (await vestige_full_asset_price_not_cached(pool.reward_token.id)).usd - token_fee = pool.reward_amount * TOKEN_FEE_PERCENT - token_fee_usd = token_fee * reward_token_usd - pool_fee[pool.id] = { - 'token_fee': { - 'asa_id': pool.reward_token.id, - 'amount': token_fee, - 'amount_usd': token_fee_usd - }, - 'algo_fee': ALGO_FEE - } - total_algo += ALGO_FEE - total_token_fee_usd += token_fee_usd - - print(f'{ALGO_FEE} ALGO, {token_fee} {pool.reward_token.unit_name}, {token_fee_usd} USD\n{pool.stake_token}\n{pool.reward_token}\nreward_amount: {pool.reward_amount}\n\n') - - print(f'\n\n{len(pool_fee)} pools in period {DATE_0_BLOCK} — {DATE_1_BLOCK}: {total_algo} ALGO, {total_token_fee_usd} USD\n\n') - - return pool_fee - - -async def fetch_and_record_pool_fees() -> dict: - pool_fees = await calculate_cometa_fees(DATE_0_BLOCK, DATE_1_BLOCK) - with open(f'pool_fees_{DATE_0_BLOCK}_{DATE_1_BLOCK}.json', 'w') as f: - json.dump(pool_fees, f) - - print(f'In total {len(pool_fees)} pools created in the period {DATE_0_BLOCK} - {DATE_1_BLOCK}') - return pool_fees - - -async def fetch_and_record_user_txns() -> dict: - user_txns = await get_all_user_transactions(DATE_0_BLOCK, DATE_1_BLOCK) - with open(f'user_txns_{DATE_0_BLOCK}_{DATE_1_BLOCK}.json', 'w') as f: - json.dump(user_txns, f) - - print(f'In total {len(user_txns)} users in the period {DATE_0_BLOCK} - {DATE_1_BLOCK}') - return user_txns diff --git a/flex/txns.py b/flex/txns.py index f5339444..3d6629fc 100644 --- a/flex/txns.py +++ b/flex/txns.py @@ -5,18 +5,17 @@ from algosdk import transaction from dataclasses_json import dataclass_json -from flex.blockchain.base import algod_client, cometa_public_key, cometa_private_key +from flex.blockchain.base import algod_client, cometa_private_key, cometa_public_key from flex.db.model.blockchain import AssetInfo from flex.meta_error import MetaError from flex.util import decode_b64 - TX_WAIT_ROUNDS = 4 class TxType(str, Enum): - PAYMENT = 'payment' - ASSET_TRANSFER = 'asset-transfer' + PAYMENT = "payment" + ASSET_TRANSFER = "asset-transfer" @dataclass_json @@ -37,23 +36,23 @@ class TxInfo: def get_tx_info_with_wait(txid: str, tx_type: TxType, timeout_rounds: int = TX_WAIT_ROUNDS) -> TxInfo: try: tx_response = transaction.wait_for_confirmation(algod_client, txid, timeout_rounds) - tx_info = tx_response['txn']['txn'] - except Exception as e: - logger.error(f'Transaction with id = {txid} is not found: {e}', exc_info=True) - raise MetaError(f'Transaction with id = {txid} is not found: {e}') + tx_info = tx_response["txn"]["txn"] + except Exception as exc: + logger.error(f"Transaction with id = {txid} is not found: {exc}", exc_info=True) + raise MetaError(f"Transaction with id = {txid} is not found: {exc}") from exc - receiver = tx_info['rcv'] if tx_type == TxType.PAYMENT else tx_info['arcv'] - amount = None if tx_type == TxType.PAYMENT else tx_info['aamt'] - asa_id = 0 if tx_type == TxType.PAYMENT else tx_info['xaid'] + receiver = tx_info["rcv"] if tx_type == TxType.PAYMENT else tx_info["arcv"] + amount = None if tx_type == TxType.PAYMENT else tx_info["aamt"] + asa_id = 0 if tx_type == TxType.PAYMENT else tx_info["xaid"] return TxInfo( id=txid, - sender=tx_info['snd'], + sender=tx_info["snd"], receiver=receiver, amount=amount, asa_id=asa_id, - note=decode_b64(tx_info.get('note')), - confirmed_round=tx_response['confirmed-round'] + note=decode_b64(tx_info.get("note")), + confirmed_round=tx_response["confirmed-round"], ) @@ -65,22 +64,12 @@ def get_transfer_info_with_wait(txid: str, wait_rounds: int = TX_WAIT_ROUNDS) -> return get_tx_info_with_wait(txid, TxType.ASSET_TRANSFER, wait_rounds) -def send_asset_micros( - asset_info: AssetInfo, - address: str, - amount_micros: int, - note: str | None = None -) -> str: - logger.debug(f'Sending {asset_info.micros_to_amount(amount_micros)} {asset_info.unit_name} to {address}!') +def send_asset_micros(asset_info: AssetInfo, address: str, amount_micros: int, note: str | None = None) -> str: + logger.debug(f"Sending {asset_info.micros_to_amount(amount_micros)} {asset_info.unit_name} to {address}!") params = algod_client.suggested_params() unsigned_txn = transaction.AssetTransferTxn( - sender=cometa_public_key, - sp=params, - receiver=address, - amt=amount_micros, - index=asset_info.id, - note=note + sender=cometa_public_key, sp=params, receiver=address, amt=amount_micros, index=asset_info.id, note=note ) signed_txn = unsigned_txn.sign(cometa_private_key) txid = algod_client.send_transaction(signed_txn) @@ -89,23 +78,16 @@ def send_asset_micros( def send_asset_micros_with_wait( - asset_info: AssetInfo, - address: str, - amount_micros: int, - note: str | None = None, - wait_rounds: int = TX_WAIT_ROUNDS + asset_info: AssetInfo, address: str, amount_micros: int, note: str | None = None, wait_rounds: int = TX_WAIT_ROUNDS ) -> TxInfo: txid = send_asset_micros(asset_info, address, amount_micros, note) tx_info = get_transfer_info_with_wait(txid, wait_rounds=wait_rounds) - logger.debug(f'Sent {asset_info.micros_to_amount(amount_micros)} {asset_info.unit_name} to {address} with txid: {txid}!') + logger.debug( + f"Sent {asset_info.micros_to_amount(amount_micros)} {asset_info.unit_name} to {address} with txid: {txid}!" + ) return tx_info -def send_asset( - asset_info: AssetInfo, - address: str, - amount: float, - note: str | None = None -) -> str: +def send_asset(asset_info: AssetInfo, address: str, amount: float, note: str | None = None) -> str: amount_micros = asset_info.amount_to_micros(amount) return send_asset_micros(asset_info, address, amount_micros, note) diff --git a/flex/util.py b/flex/util.py index 45cef305..3bc431ad 100644 --- a/flex/util.py +++ b/flex/util.py @@ -7,21 +7,21 @@ def format_usd_amount(usd: float) -> str: def format_timedelta(td: timedelta) -> str: - return f'{format(td.total_seconds(), ".1f")}s' + return f"{format(td.total_seconds(), '.1f')}s" def decode_b64(str_b64: str | None) -> str | None: if str_b64 is None: return None - bytes_repr = bytes(str_b64, encoding='utf-8') + bytes_repr = bytes(str_b64, encoding="utf-8") decoded_string = base64.b64decode(bytes_repr) - return str(decoded_string, encoding='utf-8') + return str(decoded_string, encoding="utf-8") def build_key_str(func, *args, **kwargs) -> str: - fn_name = getattr(func, '__qualname__', None) or getattr(func, '__name__', str(func)) + fn_name = getattr(func, "__qualname__", None) or getattr(func, "__name__", str(func)) parts = [fn_name] + [str(a) for a in args] if kwargs: - parts += [f'{k}={v}' for k, v in sorted(kwargs.items())] - return ':'.join(parts) + parts += [f"{k}={v}" for k, v in sorted(kwargs.items())] + return ":".join(parts) diff --git a/pyproject.toml b/pyproject.toml index 6a9448cc..67e20e10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,14 +29,11 @@ quote-style = "double" [tool.mypy] files = [ "core/circuit_breaker.py", + "flex/domain", "flex/blockchain/contract_state.py", "flex/application/asset_transfers.py", "flex/db/lp_projection.py", "flex/db/sync_coordinator.py", - "flex/domain/allocation.py", - "flex/domain/lp_projection.py", - "flex/domain/pricing.py", - "flex/domain/transactions.py", "flex/providers/pact.py", ] follow_imports = "skip" diff --git a/scripts/backup_db.sh b/scripts/backup_db.sh deleted file mode 100755 index a06cd92a..00000000 --- a/scripts/backup_db.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -CONTAINER_NAME="cometa-backend_mongodb_1" -MONGODB_PORT=27017 -DUMP_DIR="/srv/db_dump" - -# Rotate backups: keep last 7 days -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -DUMP_FILE="$DUMP_DIR/db.dump.$TIMESTAMP" - -docker exec -i $CONTAINER_NAME sh -c "mongodump --archive --port $MONGODB_PORT" > "$DUMP_FILE" - -if [ $? -eq 0 ]; then - echo "Backup saved: $DUMP_FILE" - # Remove backups older than 7 days - find "$DUMP_DIR" -name "db.dump.*" -mtime +7 -delete -else - echo "BACKUP FAILED!" - rm -f "$DUMP_FILE" - exit 1 -fi diff --git a/scripts/bot/log.sh b/scripts/bot/log.sh deleted file mode 100755 index 91255a00..00000000 --- a/scripts/bot/log.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -docker logs cometa-backend_bot_1 -f diff --git a/scripts/bot/run.sh b/scripts/bot/run.sh deleted file mode 100755 index 3bb665ee..00000000 --- a/scripts/bot/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -pipenv run python telegram_bot.py diff --git a/scripts/db_shell.sh b/scripts/db_shell.sh deleted file mode 100755 index 3bd6f192..00000000 --- a/scripts/db_shell.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -CONTAINER_NAME="cometa-backend_mongodb_1" -MONGODB_PORT=27017 - -docker exec -i "$CONTAINER_NAME" sh -c "mongosh --port $MONGODB_PORT" diff --git a/scripts/log.sh b/scripts/log.sh deleted file mode 100755 index 9f399121..00000000 --- a/scripts/log.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -source .env -if [ "$ALGO_NETWORK" = "testnet" ]; -then - PREF="-testnet" -fi - -docker logs "cometa-backend${PREF}_app_1" "$@" diff --git a/scripts/redeploy.sh b/scripts/redeploy.sh deleted file mode 100755 index b0d16790..00000000 --- a/scripts/redeploy.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" -readonly PROJECT_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd -P)" -readonly STACK_DIR="${COMETA_STACK_DIR:-${PROJECT_ROOT}/..}" -readonly STACK_FILE="${STACK_DIR}/docker-compose.yml" -readonly BACKEND_SERVICE="${COMETA_BACKEND_SERVICE:-backend}" -readonly HEALTH_URL="${COMETA_HEALTH_URL:-https://api.cometa.farm/contracts}" -readonly HEALTH_ATTEMPTS="${COMETA_HEALTH_ATTEMPTS:-12}" -readonly HEALTH_DELAY_SECONDS="${COMETA_HEALTH_DELAY_SECONDS:-5}" - -trap 'printf "ERROR: redeploy failed at line %s\n" "$LINENO" >&2' ERR - -if [[ ! "${HEALTH_ATTEMPTS}" =~ ^[1-9][0-9]*$ ]]; then - printf 'ERROR: COMETA_HEALTH_ATTEMPTS must be a positive integer\n' >&2 - exit 2 -fi - -if [[ ! "${HEALTH_DELAY_SECONDS}" =~ ^[0-9]+([.][0-9]+)?$ ]]; then - printf 'ERROR: COMETA_HEALTH_DELAY_SECONDS must be a non-negative number\n' >&2 - exit 2 -fi - -if [[ ! -f "${STACK_FILE}" ]]; then - printf 'ERROR: production Compose file not found: %s\n' "${STACK_FILE}" >&2 - exit 2 -fi - -service_found=false -while IFS= read -r service; do - if [[ "${service}" == "${BACKEND_SERVICE}" ]]; then - service_found=true - break - fi -done < <(docker compose --project-directory "${STACK_DIR}" --file "${STACK_FILE}" config --services) - -if [[ "${service_found}" != true ]]; then - printf 'ERROR: Compose service %s is not defined in %s\n' "${BACKEND_SERVICE}" "${STACK_FILE}" >&2 - exit 2 -fi - -git -C "${PROJECT_ROOT}" pull --ff-only -docker compose \ - --project-directory "${STACK_DIR}" \ - --file "${STACK_FILE}" \ - up --detach --build --remove-orphans "$@" "${BACKEND_SERVICE}" - -for ((attempt = 1; attempt <= HEALTH_ATTEMPTS; attempt++)); do - if curl --fail --silent --show-error --max-time 10 "${HEALTH_URL}" >/dev/null; then - printf 'Deploy OK: %s is ready\n' "${HEALTH_URL}" - exit 0 - fi - - if ((attempt < HEALTH_ATTEMPTS)); then - printf 'Waiting for service health (%d/%d)...\n' "${attempt}" "${HEALTH_ATTEMPTS}" >&2 - sleep "${HEALTH_DELAY_SECONDS}" - fi -done - -printf 'ERROR: service did not become healthy: %s\n' "${HEALTH_URL}" >&2 -exit 1 diff --git a/scripts/restart.sh b/scripts/restart.sh deleted file mode 100755 index f8872bfb..00000000 --- a/scripts/restart.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -scripts/stop.sh && scripts/start.sh "$@" && scripts/log.sh -f diff --git a/scripts/start.sh b/scripts/start.sh deleted file mode 100755 index e48981e4..00000000 --- a/scripts/start.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -source .env -docker-compose --profile "$ALGO_NETWORK" up -d "$@" diff --git a/scripts/stop.sh b/scripts/stop.sh deleted file mode 100755 index 10e8a96a..00000000 --- a/scripts/stop.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -docker-compose down "$@" diff --git a/scripts/verify_algorand_credentials.py b/scripts/verify_algorand_credentials.py index b170dc2c..1f2d1bd2 100644 --- a/scripts/verify_algorand_credentials.py +++ b/scripts/verify_algorand_credentials.py @@ -10,40 +10,40 @@ def _required_env(name: str) -> str: value = os.getenv(name) if not value: - raise RuntimeError(f'Missing required environment variable: {name}') + raise RuntimeError(f"Missing required environment variable: {name}") return value def _safe_error(service: str, exc: Exception) -> tuple[bool, str]: - status = getattr(exc, 'status_code', None) or getattr(exc, 'code', None) - status_suffix = f' (status {status})' if isinstance(status, int) else '' - return False, f'{service} check failed: {type(exc).__name__}{status_suffix}' + status = getattr(exc, "status_code", None) or getattr(exc, "code", None) + status_suffix = f" (status {status})" if isinstance(status, int) else "" + return False, f"{service} check failed: {type(exc).__name__}{status_suffix}" def verify_algod_client() -> tuple[bool, str]: try: client = algod.AlgodClient( - algod_token=_required_env('ALGOD_TOKEN'), - algod_address=_required_env('ALGOD_ADDRESS'), - headers={'User-Agent': 'cometa-credential-check'}, + algod_token=_required_env("ALGOD_TOKEN"), + algod_address=_required_env("ALGOD_ADDRESS"), + headers={"User-Agent": "cometa-credential-check"}, ) status = client.status() return True, f"Algod is healthy at round {status['last-round']}" except Exception as exc: - return _safe_error('Algod', exc) + return _safe_error("Algod", exc) def verify_indexer_client() -> tuple[bool, str]: try: client = indexer.IndexerClient( - indexer_token=_required_env('ALGOD_TOKEN'), - indexer_address=_required_env('ALGO_INDEXER_ADDRESS'), - headers={'User-Agent': 'cometa-credential-check'}, + indexer_token=_required_env("ALGOD_TOKEN"), + indexer_address=_required_env("ALGO_INDEXER_ADDRESS"), + headers={"User-Agent": "cometa-credential-check"}, ) client.health() - return True, 'Indexer is healthy' + return True, "Indexer is healthy" except Exception as exc: - return _safe_error('Indexer', exc) + return _safe_error("Indexer", exc) def main() -> int: @@ -54,5 +54,5 @@ def main() -> int: return 0 if all(success for success, _ in checks) else 1 -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/tdr_detailed.md b/tdr_detailed.md deleted file mode 100644 index df88d26a..00000000 --- a/tdr_detailed.md +++ /dev/null @@ -1,95 +0,0 @@ -Hola Algorand! We are honoured with Targeted DeFi Rewards this time. -And we have a _master plan_ for using it the most efficiently, which we want to discuss with you. Please tune in! - -## 50% — META pools - -Cometa token is strongly connected within the whole Algorand DeFi, a variety of tokens, which provides deep liquidity to any new token. -So now we're focusing fully on stable liquidity (but still do not forget a few amazing community projects). - -With our upcoming Launchpad and a few other minor features, the token's stability will bring great value to the whole ecosystem. - -- 12% — META/ALGO Tinyman -- 8% — META/USDC Tinyman -- 6% — META/gALGO Tinyman -- 6% — META/goBTC Tinyman -- 6% — META/wETH Pact -- 6% — META/GOLD$ Pact -- 2% — META/mALGO Pact -- 2% — META/COOP Pact -- 2% — META staking - - -## 20% — Boost Bridged and RWA tokens - -Cross-chain and RWA liquidity make the ecosystem DeFi more stable and robust. -Enough of those ALGO chart dances, let's make it properly liquid. -Our team believes that the following tokens deserve much more attention than they currently have. - -We will incentiveze all the farming pools with LPs using one of the following tokens: -- GOLD$, SILVER$ -- wBTC, wETH, wAVAX, wSOL, wLINK -- goBTC, goETH -- EURS, USDT - -The rewards will be distributed among all the qualified pools for the next 90 days after the start. -- A pool gets the amount which is proportional to the total $ cost of the provided rewards. -- If the price is not stable enough, we'll take the minimum price for the recent period to calculate the cost. -- If a pool duration is shorter than 90 days, the share will be cut accordingly. - -### x2 Boost to small & new projects! - -Incentivizing already popular and big tokens makes little sense, frankly. -So instead we'll help smaller projects to grow faster! -It will bring much more relative impact for the same amount of rewards, which could be wasted on the big tokens. - -To be eligible for x2, a token has to: -- be created before 22 of April 2024 -- have stable liquidity < $20k (without messing with it beforehand, we'll analyze the charts) -- have an active Twitter - -The same rewards distribution rules apply. - -## 30% — Deeply connect Algorand and Base communities! - -**Farcaster** is a new amazing social crypto platform that has gained huge attention, mostly of the Base blockchain community. -Dope features with smart built-in DEGEN tokenomics make if _very_ active and growing fast. -We see a no-brainer way to connect Algorand and Base via Farcaster, its current state and features. - -**Killer-feature** of Farcaster is the ability to integrate any Javascript code into any post! -This is the next level stuff. Imagine doing DeFi never leaving the Twitter? - -We want to utilize it while it is still early days and _literally integrate Algorand DeFi into Farcaster posts feed_. - -One of the key Farcaster features is token DEGEN and its degen tokenomics ($800m FDV). -We're going to leverage that _hard_. -We've already bridged the token to the Algorand with the help of Messina team. - -And we've already performed **the successful test** of the plan!! - -We've developed the simplest app and posted it in Farcaster, **check it out** https://warpcast.com/cometa/0xd8c1afdd - -The app allows you to: -- create a new Algorand wallet -- claim DEGEN on Algorand wallet -- claim an NFT to Algorand wallet - -With that we made 15 Farcaster users to create Algorand wallets and understand how it works. -And it is without any funding, promotions and with the app full of bugs. -With proper funding we have 100% potential to make a really HUGE impact: -- we'll attract people from Algorand to Farcaster, to spread the word about Algorand and assimilate algofam in Farcaster, making it Algorand-native as well -- we'll get people from Base to participate in Algorand DeFi and integrate to the Algorand Twitter -- thus we'll achieve a deep symbiosis between Algorand and Base, attracting big money from Base to Algorand - -To achieve that we need: -- 8% — Algo-DEGEN liquidity pools: DEGEN, DEGEN/ALGO, DEGEN/goETH, DEGEN/META -- 8% — DEGEN rewards for completing the tasks and using the bridge app (on both sides) -- 8% — constant development of the Farcaster bridge-DeFi/NFT/DEGEN app features -- 4% — collaborations and advertisements - - -## Conclusion - -Cometa has its own area of responsibility — helping projects to grow and improve liquidity across the board. -We try non-trivial ideas, potentially achieving crazy results. Someone has to do that, Algorand has been in a shadows for too long. - -What do you think about our plans? Please share any thoughts! diff --git a/telegram_bot.py b/telegram_bot.py index 9fd44141..7bcde0dc 100644 --- a/telegram_bot.py +++ b/telegram_bot.py @@ -17,8 +17,6 @@ from bot.phrase_manager import Phrases from core.db.cometa_users import cometa_users, filter_compoundable_pools, filter_ended_pools, filter_no_action_pools -# TODO: move commands to separate files - async def start(update: Update, context: CallbackContext): await update.message.reply_html( @@ -222,7 +220,6 @@ async def show_help(update: Update, context: CallbackContext): def start_bot(): - # TODO: implement Command class app_context.application.add_handler(CommandHandler("start", start)) app_context.application.add_handler(CommandHandler("register", register)) app_context.application.add_handler(CommandHandler("change_address", change_address)) @@ -245,7 +242,6 @@ def tear_down(): logging.info("EXIT BOT\n\nBye!\n") -# TODO: set up smarter def setup_logging(): logging.basicConfig(format=LOG_FORMAT, datefmt=LOG_DATE_FORMAT, level=bot_settings.logging_level) diff --git a/tests/integration/test_mongo_contract_registration.py b/tests/integration/test_mongo_contract_registration.py new file mode 100644 index 00000000..cb5d323f --- /dev/null +++ b/tests/integration/test_mongo_contract_registration.py @@ -0,0 +1,353 @@ +import os +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from datetime import UTC, datetime +from threading import Barrier +from uuid import uuid4 + +import pytest +from pymongo import MongoClient + +import app as api_module +from core.db.contracts import ( + ensure_contract_id_index, + get_or_create_contract, +) +from core.db.model import ContractInfo +from flex.application import pool_registration +from flex.application.pool_registration import PoolIdentityConflictError +from flex.data import pool_state as pool_state_data +from flex.data.pool_state import PoolStateIdentityConflictError +from flex.db.cometa_database import CometaDatabase +from flex.db.indexes import ensure_database_indexes +from flex.db.model.blockchain import AssetInfo +from flex.db.model.pool_states import PoolState +from flex.db.model.pools import PoolType, StakingPool + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def registration_databases(): + uri = os.getenv("MONGODB_TEST_URI") + if not uri: + pytest.skip("MONGODB_TEST_URI is not configured") + + client = MongoClient( + uri, + serverSelectionTimeoutMS=2_000, + tz_aware=True, + ) + client.admin.command("ping") + suffix = uuid4().hex + legacy_database = client[f"cometa_contracts_{suffix}"] + flex_database = client[f"cometa_flex_{suffix}"] + try: + yield legacy_database, CometaDatabase(flex_database) + finally: + client.drop_database(legacy_database.name) + client.drop_database(flex_database.name) + client.close() + + +def _registration_records() -> tuple[ContractInfo, StakingPool, PoolState]: + timestamp = datetime(2026, 1, 1, tzinfo=UTC) + asset = AssetInfo( + id=7, + name="Stake", + decimals=6, + unit_name="STK", + ) + contract = ContractInfo( + type="distribution", + id=42, + version="17.0.5", + deployed_timestamp=timestamp.timestamp(), + deployed_date=timestamp, + begin_date=timestamp, + end_date=timestamp, + description="Concurrent registration", + metadata={"cache": {"initial": {}}}, + ) + pool = StakingPool( + id=contract.id, + description=contract.description, + address="POOL", + stake_token=asset, + reward_token=asset, + reward_amount_micros=1_000, + algo_reward_amount_micros=0, + begin_block=10, + end_block=20, + lock_length_blocks=0, + deploy_date=timestamp, + begin_date=timestamp, + end_date=timestamp, + ) + state = PoolState( + pool_id=contract.id, + type=PoolType.STAKING, + stake_token=asset, + address=pool.address, + ) + return contract, pool, state + + +def _persist_registration( + *, + legacy_contracts, + flex_database: CometaDatabase, + contract: ContractInfo, + pool: StakingPool, + state: PoolState, +) -> bool: + contract_write = get_or_create_contract( + contract, + target_collection=legacy_contracts, + ) + flex_database.staking_pools.get_or_create(pool) + flex_database.pool_states.get_or_create_by( + state, + pool_id=state.pool_id, + ) + return contract_write.created + + +def test_concurrent_registration_persists_one_identity_graph( + registration_databases, +) -> None: + legacy_database, flex_database = registration_databases + legacy_contracts = legacy_database["contract"] + ensure_contract_id_index(target_collection=legacy_contracts) + ensure_database_indexes(flex_database) + contract, pool, state = _registration_records() + barrier = Barrier(8) + + def register() -> bool: + barrier.wait(timeout=5) + return _persist_registration( + legacy_contracts=legacy_contracts, + flex_database=flex_database, + contract=contract, + pool=pool, + state=state, + ) + + with ThreadPoolExecutor(max_workers=8) as executor: + results = [future.result(timeout=10) for future in [executor.submit(register) for _ in range(8)]] + + assert sum(results) == 1 + assert legacy_contracts.count_documents({"id": contract.id}) == 1 + assert flex_database.staking_pools.mongodb_collection.count_documents({"id": contract.id}) == 1 + assert flex_database.pool_states.mongodb_collection.count_documents({"pool_id": contract.id}) == 1 + + +def test_contract_index_rejects_existing_duplicate_identity( + registration_databases, +) -> None: + legacy_database, _ = registration_databases + contracts = legacy_database["contract"] + contracts.insert_many([{"id": 42}, {"id": 42}]) + + with pytest.raises(RuntimeError, match="duplicate immutable ID 42"): + ensure_contract_id_index(target_collection=contracts) + + +@pytest.mark.parametrize( + ("collection_name", "field_name"), + [ + ("staking_pools", "id"), + ("farming_pools", "id"), + ("pool_states", "pool_id"), + ], +) +def test_flex_indexes_reject_existing_duplicate_identity( + registration_databases, + collection_name: str, + field_name: str, +) -> None: + _, flex_database = registration_databases + flex_database.mongodb_database[collection_name].insert_many([{field_name: 42}, {field_name: 42}]) + + with pytest.raises(RuntimeError, match="duplicate"): + ensure_database_indexes(flex_database) + + +def test_flex_indexes_reject_cross_collection_pool_identity( + registration_databases, +) -> None: + _, flex_database = registration_databases + flex_database.staking_pools.mongodb_collection.insert_one({"id": 42}) + flex_database.farming_pools.mongodb_collection.insert_one({"id": 42}) + + with pytest.raises(RuntimeError, match="exists in both"): + ensure_database_indexes(flex_database) + + +@pytest.mark.asyncio +async def test_registration_rejects_opposite_kind_orphan_without_target_write( + registration_databases, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, flex_database = registration_databases + ensure_database_indexes(flex_database) + contract, pool, _ = _registration_records() + flex_database.farming_pools.mongodb_collection.insert_one({"id": contract.id}) + + async def build_pool(_contract: ContractInfo, distribution: bool = False) -> StakingPool: + assert distribution is True + return pool + + monkeypatch.setattr(pool_registration, "db", flex_database) + monkeypatch.setattr( + pool_registration, + "staking_pool_from_contract_info", + build_pool, + ) + + with pytest.raises(PoolIdentityConflictError, match="opposite pool type"): + await pool_registration.create_pool_from_contract(contract) + + assert flex_database.staking_pools.mongodb_collection.count_documents({"id": contract.id}) == 0 + + +@pytest.mark.asyncio +async def test_registration_rejects_incompatible_existing_pool( + registration_databases, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, flex_database = registration_databases + ensure_database_indexes(flex_database) + contract, candidate, _ = _registration_records() + flex_database.staking_pools.create(replace(candidate, address="OTHER")) + + async def build_pool(_contract: ContractInfo, distribution: bool = False) -> StakingPool: + assert distribution is True + return candidate + + monkeypatch.setattr(pool_registration, "db", flex_database) + monkeypatch.setattr( + pool_registration, + "staking_pool_from_contract_info", + build_pool, + ) + + with pytest.raises(PoolIdentityConflictError, match="incompatible persisted"): + await pool_registration.create_pool_from_contract(contract) + + assert flex_database.staking_pools.mongodb_collection.count_documents({"id": contract.id}) == 1 + stored = flex_database.staking_pools.get_by_primary_key(contract.id) + assert stored.address == "OTHER" + + +@pytest.mark.asyncio +async def test_pool_state_rejects_incompatible_existing_identity( + registration_databases, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, flex_database = registration_databases + ensure_database_indexes(flex_database) + contract, pool, candidate = _registration_records() + flex_database.staking_pools.create(pool) + flex_database.pool_states.create(replace(candidate, address="OTHER")) + monkeypatch.setattr(pool_state_data, "db", flex_database) + monkeypatch.setattr(pool_state_data, "get_contract", lambda _pool_id: contract) + await pool_state_data.get_or_create_pool_state.cache.clear() + + with pytest.raises(PoolStateIdentityConflictError, match="incompatible persisted"): + await pool_state_data.get_or_create_pool_state(contract.id) + + assert flex_database.pool_states.mongodb_collection.count_documents({"pool_id": contract.id}) == 1 + stored = flex_database.pool_states.get_one(pool_id=contract.id) + assert stored.address == "OTHER" + + +@pytest.mark.asyncio +async def test_registration_saga_rejects_incompatible_existing_pool_state( + registration_databases, + monkeypatch: pytest.MonkeyPatch, +) -> None: + legacy_database, flex_database = registration_databases + legacy_contracts = legacy_database["contract"] + ensure_contract_id_index(target_collection=legacy_contracts) + ensure_database_indexes(flex_database) + contract, pool, candidate = _registration_records() + flex_database.staking_pools.create(pool) + flex_database.pool_states.create(replace(candidate, address="OTHER")) + + async def build_pool(_contract: ContractInfo, distribution: bool = False) -> StakingPool: + assert distribution is True + return pool + + monkeypatch.setattr(pool_registration, "db", flex_database) + monkeypatch.setattr( + pool_registration, + "staking_pool_from_contract_info", + build_pool, + ) + monkeypatch.setattr(pool_state_data, "db", flex_database) + monkeypatch.setattr( + api_module, + "get_or_create_contract", + lambda requested: get_or_create_contract( + requested, + target_collection=legacy_contracts, + ), + ) + monkeypatch.setattr(api_module, "invalidate_contracts_cache", lambda: None) + monkeypatch.setattr( + api_module, + "parse_cache", + lambda _cache: { + "begin_block": pool.begin_block, + "end_block": pool.end_block, + "begin_date": pool.begin_date, + "end_date": pool.end_date, + "lock_length_blocks": pool.lock_length_blocks, + }, + ) + + with pytest.raises(PoolStateIdentityConflictError, match="incompatible persisted"): + await api_module.create_contract_with( + type=contract.type, + id=contract.id, + version=contract.version, + description=contract.description, + metadata=contract.metadata, + ) + + assert legacy_contracts.count_documents({"id": contract.id}) == 1 + assert flex_database.staking_pools.mongodb_collection.count_documents({"id": contract.id}) == 1 + assert flex_database.pool_states.mongodb_collection.count_documents({"pool_id": contract.id}) == 1 + + +@pytest.mark.parametrize("failure_after", ["contract", "pool"]) +def test_registration_retry_recovers_partial_identity_graph( + registration_databases, + failure_after: str, +) -> None: + legacy_database, flex_database = registration_databases + legacy_contracts = legacy_database["contract"] + ensure_contract_id_index(target_collection=legacy_contracts) + ensure_database_indexes(flex_database) + contract, pool, state = _registration_records() + + get_or_create_contract( + contract, + target_collection=legacy_contracts, + ) + if failure_after == "pool": + flex_database.staking_pools.get_or_create(pool) + + created = _persist_registration( + legacy_contracts=legacy_contracts, + flex_database=flex_database, + contract=contract, + pool=pool, + state=state, + ) + + assert created is False + assert legacy_contracts.count_documents({"id": contract.id}) == 1 + assert flex_database.staking_pools.mongodb_collection.count_documents({"id": contract.id}) == 1 + assert flex_database.pool_states.mongodb_collection.count_documents({"pool_id": contract.id}) == 1 diff --git a/tests/unit/test_collection_manager.py b/tests/unit/test_collection_manager.py index 7cf00a1e..78d390af 100644 --- a/tests/unit/test_collection_manager.py +++ b/tests/unit/test_collection_manager.py @@ -59,6 +59,34 @@ def test_get_or_create_returns_existing_document_without_overwriting_it() -> Non assert result == existing +def test_get_or_create_by_uses_business_identity() -> None: + collection = Mock() + item = _entity() + collection.find_one_and_update.return_value = item.to_dict() + manager = CollectionManager("examples", ExampleEntity, collection) + + result = manager.get_or_create_by(item, value="new") + + assert result == item + collection.find_one_and_update.assert_called_once_with( + {"value": "new"}, + {"$setOnInsert": item.to_dict()}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + + +def test_get_or_create_by_requires_business_identity() -> None: + manager = CollectionManager("examples", ExampleEntity, Mock()) + + try: + manager.get_or_create_by(_entity()) + except ValueError as exc: + assert str(exc) == "get_or_create_by requires at least one identity field" + else: + raise AssertionError("Expected ValueError for an empty business identity") + + def test_get_or_create_with_delegates_to_upsert_path() -> None: collection = Mock() item = _entity() diff --git a/tests/unit/test_contract_repository.py b/tests/unit/test_contract_repository.py new file mode 100644 index 00000000..08d0c434 --- /dev/null +++ b/tests/unit/test_contract_repository.py @@ -0,0 +1,100 @@ +from datetime import UTC, datetime +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from pymongo.errors import DuplicateKeyError + +from core.db.contracts import ( + ContractIdentityConflictError, + ensure_contract_id_index, + get_or_create_contract, +) +from core.db.model import ContractInfo + + +def _contract(*, contract_type: str = "farm", version: str = "17.2.5") -> ContractInfo: + deployed_at = datetime(2026, 1, 1, tzinfo=UTC) + return ContractInfo( + type=contract_type, + id=42, + version=version, + deployed_timestamp=deployed_at.timestamp(), + deployed_date=deployed_at, + description="Test contract", + metadata={"cache": {"initial": {}}}, + ) + + +def test_contract_index_fails_closed_on_duplicate_identity() -> None: + collection = Mock() + collection.aggregate.return_value = [{"_id": 42, "count": 2}] + + with pytest.raises(RuntimeError, match="duplicate immutable ID 42"): + ensure_contract_id_index(target_collection=collection) + + collection.create_index.assert_not_called() + + +def test_contract_index_enforces_unique_business_id() -> None: + collection = Mock() + collection.aggregate.return_value = [] + + ensure_contract_id_index(target_collection=collection) + + collection.create_index.assert_called_once_with( + "id", + unique=True, + name="contract_id_unique", + ) + + +def test_contract_upsert_returns_canonical_existing_record() -> None: + requested = _contract() + existing = _contract() + existing.description = "Canonical description" + collection = Mock() + collection.update_one.return_value = SimpleNamespace(upserted_id=None) + collection.find_one.return_value = existing.to_dict() + + result = get_or_create_contract( + requested, + target_collection=collection, + ) + + assert result.contract.description == "Canonical description" + assert result.created is False + collection.update_one.assert_called_once_with( + {"id": requested.id}, + {"$setOnInsert": requested.to_dict()}, + upsert=True, + ) + + +def test_contract_upsert_recovers_after_concurrent_unique_insert() -> None: + contract = _contract() + collection = Mock() + collection.update_one.side_effect = DuplicateKeyError("concurrent insert") + collection.find_one.return_value = contract.to_dict() + + result = get_or_create_contract( + contract, + target_collection=collection, + ) + + assert result.contract.id == contract.id + assert result.created is False + + +def test_contract_upsert_rejects_conflicting_identity() -> None: + requested = _contract(version="17.2.5") + existing = _contract(version="17.2.4") + collection = Mock() + collection.update_one.return_value = SimpleNamespace(upserted_id=None) + collection.find_one.return_value = existing.to_dict() + + with pytest.raises(ContractIdentityConflictError, match="different type or version"): + get_or_create_contract( + requested, + target_collection=collection, + ) diff --git a/tests/unit/test_contract_state_services.py b/tests/unit/test_contract_state_services.py index 7bde6675..0add154e 100644 --- a/tests/unit/test_contract_state_services.py +++ b/tests/unit/test_contract_state_services.py @@ -10,6 +10,7 @@ import app as api_module from api import background from api.db_model import ContractType +from core.db.contracts import ContractWriteResult from env import Settings from flex.blockchain import contract_state from flex.blockchain.contract_state import ( @@ -92,11 +93,14 @@ async def create_contract(contract: object, metadata: dict[str, Any]) -> SimpleN captured["contract"] = contract captured["metadata"] = metadata return SimpleNamespace( - metadata={ - "begin_block": 1, - "end_block": 2, - "lock_length_blocks": 0, - } + contract=SimpleNamespace( + metadata={ + "begin_block": 1, + "end_block": 2, + "lock_length_blocks": 0, + } + ), + created=True, ) async def notify_new_pool(**kwargs: object) -> None: @@ -120,6 +124,151 @@ async def notify_new_pool(**kwargs: object) -> None: assert captured["notification"]["metadata"] == {"source": "test"} +@pytest.mark.asyncio +async def test_registration_retry_skips_duplicate_notification( + monkeypatch: pytest.MonkeyPatch, +) -> None: + beneficiary = f"0x{encoding.decode_address(api_module.settings.beneficiary_address).hex()}" + canonical = SimpleNamespace( + metadata={ + "begin_block": 1, + "end_block": 2, + "lock_length_blocks": 0, + } + ) + + async def fetch_view(*_args: object) -> dict[str, dict[str, Any]]: + return _view(beneficiary) + + async def create_contract(*_args: object) -> SimpleNamespace: + return SimpleNamespace(contract=canonical, created=False) + + async def unexpected_notification(**_kwargs: object) -> None: + raise AssertionError("idempotent retry must not emit a second notification") + + monkeypatch.setattr(api_module, "_fetch_contract_view", fetch_view) + monkeypatch.setattr(api_module, "create_contract", create_contract) + monkeypatch.setattr(api_module, "notify_new_pool", unexpected_notification) + + result = await api_module.register_contract( + api_module.AddContract( + type=ContractType.FARM, + id=42, + version="17.2.5", + ) + ) + + assert result is canonical + + +@pytest.mark.asyncio +async def test_contract_registration_retries_partial_pool_write( + monkeypatch: pytest.MonkeyPatch, +) -> None: + stored_contract = None + pool_attempts = 0 + invalidations = 0 + + def store_contract(contract): + nonlocal stored_contract + created = stored_contract is None + if stored_contract is None: + stored_contract = contract + return ContractWriteResult(contract=stored_contract, created=created) + + async def create_pool(contract): + nonlocal pool_attempts + pool_attempts += 1 + if pool_attempts == 1: + raise RuntimeError("injected pool write failure") + return SimpleNamespace( + id=contract.id, + stake_token=SimpleNamespace(id=7), + reward_token=SimpleNamespace(id=8), + ) + + async def create_pool_state(contract) -> SimpleNamespace: + return SimpleNamespace(pool_id=contract.id) + + async def get_price(_asset_id: int) -> None: + return None + + def invalidate() -> None: + nonlocal invalidations + invalidations += 1 + + monkeypatch.setattr( + api_module, + "parse_cache", + lambda _cache: { + "begin_block": 10, + "end_block": 20, + "begin_date": None, + "end_date": None, + "lock_length_blocks": 5, + }, + ) + monkeypatch.setattr(api_module, "get_or_create_contract", store_contract) + monkeypatch.setattr(api_module, "ensure_pool_identity_slot", lambda _contract: None) + monkeypatch.setattr(api_module, "create_pool_from_contract", create_pool) + monkeypatch.setattr(api_module, "create_pool_state_from_contract", create_pool_state) + monkeypatch.setattr(api_module, "get_asset_price_not_cached", get_price) + monkeypatch.setattr(api_module, "invalidate_contracts_cache", invalidate) + + with pytest.raises(RuntimeError, match="injected pool write failure"): + await api_module.create_contract_with( + type="farm", + id=42, + version="17.2.5", + description="Retry-safe registration", + metadata={"cache": {"initial": {}}}, + ) + + result = await api_module.create_contract_with( + type="farm", + id=42, + version="17.2.5", + description="Retry-safe registration", + metadata={"cache": {"initial": {}}}, + ) + + assert result.created is False + assert result.contract.metadata["begin_block"] == 10 + assert result.contract.metadata["end_block"] == 20 + assert result.contract.metadata["lock_length_blocks"] == 5 + assert pool_attempts == 2 + assert invalidations == 1 + + +@pytest.mark.asyncio +async def test_contract_registration_checks_pool_slot_before_legacy_write( + monkeypatch: pytest.MonkeyPatch, +) -> None: + writes = 0 + + def reject_pool_slot(_contract: object) -> None: + raise api_module.PoolIdentityConflictError("opposite pool type") + + def unexpected_write(_contract: object) -> None: + nonlocal writes + writes += 1 + + monkeypatch.setattr(api_module, "parse_cache", lambda _cache: {}) + monkeypatch.setattr(api_module, "ensure_pool_identity_slot", reject_pool_slot) + monkeypatch.setattr(api_module, "get_or_create_contract", unexpected_write) + + with pytest.raises(api_module.PoolIdentityConflictError, match="opposite pool type"): + await api_module.create_contract_with( + type="distribution", + id=42, + version="17.0.5", + description="Conflicting registration", + metadata={}, + ) + + assert writes == 0 + + @pytest.mark.asyncio async def test_registration_rejects_wrong_beneficiary( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_database_indexes.py b/tests/unit/test_database_indexes.py index e3685670..6205ea69 100644 --- a/tests/unit/test_database_indexes.py +++ b/tests/unit/test_database_indexes.py @@ -9,6 +9,7 @@ delete_unverified_legacy_lp_prices, ensure_airdrop_indexes, ensure_database_indexes, + ensure_pool_identity_is_disjoint, ensure_sync_state_singleton, ) @@ -23,6 +24,7 @@ def _database(**collections: Mock) -> SimpleNamespace: "assets", "asset_prices", "asset_transfer_intents", + "farming_pools", "pool_transactions", "lp_transactions", "lp_states", @@ -30,9 +32,12 @@ def _database(**collections: Mock) -> SimpleNamespace: "user_states", "lp_tokens", "airdrop_rewards", + "staking_pools", "sync_states", ) - return SimpleNamespace(**{name: _manager(collections.get(name, Mock())) for name in names}) + database = SimpleNamespace(**{name: _manager(collections.get(name, Mock())) for name in names}) + database.farming_pools.mongodb_collection.distinct.return_value = [] + return database def test_deduplication_keeps_newest_record_before_creating_index() -> None: @@ -100,9 +105,11 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: "assets", "asset_prices", "asset_transfer_intents", + "farming_pools", "pool_transactions", "lp_transactions", "lp_tokens", + "staking_pools", ) } for collection in unique_collections.values(): @@ -123,6 +130,8 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: database.asset_prices.mongodb_collection.delete_many.return_value = SimpleNamespace( deleted_count=1, ) + database.pool_states.mongodb_collection.aggregate.return_value = [] + database.user_states.mongodb_collection.aggregate.return_value = [] removed = ensure_database_indexes(database) @@ -131,9 +140,11 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: "assets": 0, "asset_prices": 0, "asset_transfer_intents": 0, + "farming_pools": 0, "pool_transactions": 0, "lp_transactions": 0, "lp_tokens": 0, + "staking_pools": 0, } for collection in unique_collections.values(): collection.create_index.assert_called_once_with("id", unique=True, name="id_unique") @@ -193,8 +204,16 @@ def test_database_indexes_cover_all_projection_ids_and_hot_queries() -> None: call("token_id", unique=True, name="token_id_unique"), call("address", unique=True, name="address_unique"), ] - database.pool_states.mongodb_collection.create_index.assert_called_once_with("pool_id", name="pool_id_idx") - database.user_states.mongodb_collection.create_index.assert_called_once_with("address", name="address_idx") + database.pool_states.mongodb_collection.create_index.assert_called_once_with( + "pool_id", + unique=True, + name="pool_id_unique", + ) + database.user_states.mongodb_collection.create_index.assert_called_once_with( + "address", + unique=True, + name="address_unique", + ) database.airdrop_rewards.mongodb_collection.create_index.assert_called_once_with( "operation_id", unique=True, @@ -238,6 +257,15 @@ def test_standalone_airdrop_indexes_fail_closed_on_duplicate_operations() -> Non rewards.delete_many.assert_not_called() +def test_pool_identity_cannot_span_staking_and_farming_collections() -> None: + database = _database() + database.farming_pools.mongodb_collection.distinct.return_value = [42] + database.staking_pools.mongodb_collection.find_one.return_value = {"id": 42} + + with pytest.raises(RuntimeError, match="exists in both"): + ensure_pool_identity_is_disjoint(database) + + def test_legacy_lp_price_cleanup_reports_deleted_rows() -> None: collection = Mock() collection.delete_many.return_value = SimpleNamespace(deleted_count=3) diff --git a/tests/unit/test_pool_registration.py b/tests/unit/test_pool_registration.py new file mode 100644 index 00000000..a3a6f3c7 --- /dev/null +++ b/tests/unit/test_pool_registration.py @@ -0,0 +1,62 @@ +from datetime import datetime + +import pytest + +from core.db.model import ContractInfo +from flex.application import pool_registration +from flex.db.model.blockchain import AssetInfo + + +@pytest.mark.asyncio +async def test_farming_pool_uses_synchronous_round_provider_for_missing_dates(monkeypatch): + contract = ContractInfo( + type="farm", + id=42, + version="0.1.11", + deployed_timestamp=1_700_000_000, + description="Test pool", + metadata={ + "asset1_id": 1, + "asset2_id": 2, + "stake_token_id": 3, + "reward_token_id": 4, + "dex": "tinyman", + "begin_block": 90, + "end_block": 110, + "lock_length_blocks": 5, + "cache": { + "initial": { + "totalRewardAmount": "1000", + "totalAlgoRewardAmount": "2000", + } + }, + }, + ) + observed_rounds: list[tuple[int, int]] = [] + + async def fake_asset_info(asset_id: int) -> AssetInfo: + return AssetInfo( + id=asset_id, + name=f"Asset {asset_id}", + decimals=6, + unit_name=f"A{asset_id}", + ) + + def fake_date_from_block(round_num: int, current_round: int, current_date: datetime) -> datetime: + observed_rounds.append((round_num, current_round)) + return current_date + + monkeypatch.setattr(pool_registration, "get_current_round", lambda: 100) + monkeypatch.setattr(pool_registration, "date_from_block", fake_date_from_block) + monkeypatch.setattr(pool_registration, "get_asset_info", fake_asset_info) + monkeypatch.setattr(pool_registration, "get_app_address", lambda _: _async_value("POOL")) + + pool = await pool_registration.farming_pool_from_contract_info(contract) + + assert pool.id == contract.id + assert pool.address == "POOL" + assert observed_rounds == [(90, 100), (110, 100)] + + +async def _async_value(value: str) -> str: + return value diff --git a/tests/unit/test_security_boundaries.py b/tests/unit/test_security_boundaries.py index 13fb6180..dda5e371 100644 --- a/tests/unit/test_security_boundaries.py +++ b/tests/unit/test_security_boundaries.py @@ -3,6 +3,7 @@ from fastapi import HTTPException from fastapi.testclient import TestClient +import app as app_module from app import app from core.auth import require_password from env import settings @@ -59,3 +60,24 @@ def test_wallet_assets_limit_is_bounded_by_validation(): response = client.get(f"/wallet/{address}/assets?limit=101") assert response.status_code == 422 + + +def test_legacy_migrate_flag_cannot_trigger_startup_mutation(monkeypatch): + startup_steps: list[str] = [] + + monkeypatch.setattr(settings, "migrate", True) + monkeypatch.setattr( + app_module, + "ensure_contract_id_index", + lambda: startup_steps.append("contract_index"), + ) + monkeypatch.setattr( + app_module, + "ensure_database_indexes", + lambda database: startup_steps.append("indexes"), + ) + monkeypatch.setattr(app_module, "get_contracts_by_type", lambda contract_type: []) + + app_module.init_app() + + assert startup_steps == ["contract_index", "indexes"] diff --git a/tests/unit/test_tinyman_read_adapter.py b/tests/unit/test_tinyman_read_adapter.py new file mode 100644 index 00000000..16b6d4ee --- /dev/null +++ b/tests/unit/test_tinyman_read_adapter.py @@ -0,0 +1,69 @@ +from types import SimpleNamespace + +import pytest + +from dexes import tinyman + + +def test_init_tinyman_client_has_no_implicit_signer(monkeypatch): + algod_client = object() + expected_client = object() + observed: dict[str, object] = {} + + monkeypatch.setattr(tinyman, "init_algod_client", lambda: algod_client) + + def fake_tinyman_from_algod(algod, address=None): + observed["algod"] = algod + observed["address"] = address + return expected_client + + monkeypatch.setattr(tinyman, "tinyman_from_algod", fake_tinyman_from_algod) + + assert tinyman.init_tinyman_client() is expected_client + assert observed == {"algod": algod_client, "address": None} + + +def test_get_pool_info_preserves_requested_asset_order(): + asset1 = SimpleNamespace(id=7, decimals=2) + asset2 = SimpleNamespace(id=11, decimals=3) + pool = SimpleNamespace( + asset_1_reserves=20_000, + asset_2_reserves=3_000, + issued_pool_tokens=400, + asset_1=asset2, + asset_2=asset1, + pool_token_asset=SimpleNamespace(name="TMPOOL", decimals=2), + ) + + class FakeClient: + def fetch_asset(self, asset_id): + return {7: asset1, 11: asset2}[asset_id] + + def fetch_pool(self, first_asset, second_asset): + assert (first_asset, second_asset) == (asset1, asset2) + return pool + + assert tinyman.get_pool_info(FakeClient(), 7, 11) == tinyman.PoolInfo( + name="TMPOOL", + asset1_reserve=30.0, + asset2_reserve=20.0, + total_lp_tokens=4.0, + ) + + +def test_asset_registry_rejects_non_object_payload(monkeypatch): + class FakeResponse: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + def read(self): + return b"[]" + + tinyman.get_all_assets.cache_clear() + monkeypatch.setattr(tinyman.urllib.request, "urlopen", lambda *args, **kwargs: FakeResponse()) + + with pytest.raises(ValueError, match="JSON object"): + tinyman.get_all_assets() diff --git a/upload_image.sh b/upload_image.sh deleted file mode 100755 index 19895e3c..00000000 --- a/upload_image.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -image_path="$1" - -echo "Uploading $image_path to cometa" - -scp -3 "$image_path" cometa:/var/www/media/images/