diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 6d10d669..767756fc 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -321,6 +321,28 @@ jobs: name: unit-test-coverage-xml-${{ matrix.python-version }} path: coverage-unit.xml + linter_checks: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - uses: astral-sh/setup-uv@v6 + # `uv sync` picks up `tomte` + `tox-uv` from the dev group in + # pyproject.toml, so the pins stay in one place (matches the + # `unit-tests` job at line 304). `uv run tomte tox` then resolves + # the lint envs the same way contributors do locally. + - name: Install project dependencies + run: uv sync --all-groups --frozen + - name: Code checks + run: uv run tomte tox -p -e black-check -e isort-check -e flake8 -e mypy -e pylint -e darglint + - name: Security checks + run: uv run tomte tox -p -e safety -e bandit + - name: License compatibility check + run: uv run tomte tox -e liccheck + all-checks-passed: # Single aggregate gate referenced by branch protection. When adding or # removing a mandatory job, update the `needs:` list here instead of @@ -335,6 +357,7 @@ jobs: - changes - e2e-test-migrate-to-pearl - unit-tests + - linter_checks steps: - name: Verify all dependencies succeeded or were skipped run: | diff --git a/.gitignore b/.gitignore index a0ca3675..44b1edc1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,10 @@ logs .idea/ .coverage coverage-*.xml -htmlcov/ \ No newline at end of file +htmlcov/ + +# tox / setuptools artefacts (built by `tomte tox` testenvs that +# run an editable install of the project) +.tox/ +*.egg-info/ +.tomte-* \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d290dac3..a35d508f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,63 +1,25 @@ # Contributing to OLAS AI Agents Quickstart -## How to Contribute - -### Before You Start - -1. **Open an Issue First**: Before starting work on a feature, bug fix, or significant change, please [open an issue](https://github.com/valory-xyz/quickstart/issues) to discuss your idea with the maintainers. This helps ensure your contribution aligns with the project's direction and avoids duplicate work. - -2. **Check Existing Issues**: Review open and closed issues to see if your concern has already been discussed. - -### Contribution Areas - -We welcome contributions in the following areas: - -- **`configs/`**: Agent configurations and setup files -- **`scripts/`**: Agent-specific scripts and utilities -- **`tests/`**: Test cases and testing improvements - -### Git Workflow - -We follow a standard git workflow: - -1. **Create a Branch**: Create a feature branch from `main` - ```bash - git checkout -b feature/your-feature-name - ``` - -2. **Make Your Changes**: Implement your changes following the guidelines below - -3. **Commit Your Work**: Make clear, descriptive commits - ```bash - git commit -m "Brief description of your changes" - ``` - -4. **Push and Create a PR**: Push your branch and create a pull request - ```bash - git push origin feature/your-feature-name - ``` - -5. **Code Review**: Address any feedback from reviewers - -6. **Merge**: Once approved, your PR will be merged into `main` - -### Code Quality - -- **CI Checks**: All code changes are validated through CI checks. Ensure your changes pass all automated checks (linting, testing, etc.) -- **Local Testing**: While not required, testing your changes locally is recommended -- **Documentation**: Include or update documentation as needed for new features or changes - -### Commit Guidelines - -- Use clear, descriptive commit messages -- Reference related issues when applicable: `Fixes #123` or `Related to #123` -- Keep commits focused on a single logical change - -## Questions? - -If you have questions or need clarification: -- Check existing [issues](https://github.com/valory-xyz/quickstart/issues) and [PRs](https://github.com/valory-xyz/quickstart/pulls) -- Open a new issue with your question +This repository follows the Valory Open-Autonomy contribution workflow. + +See the canonical guide for the PR checklist, pre-commit routine, +coding style, and linter / test commands: +**[open-autonomy/CONTRIBUTING.md](https://github.com/valory-xyz/open-autonomy/blob/main/CONTRIBUTING.md)** + +Repo-specific notes for this project: + +- **Scope**: contributions are welcome in `configs/` (agent configs), + `scripts/` (agent-specific scripts), and `tests/`. +- **Open an issue first** for any non-trivial change so it can be + discussed before you spend time on it. +- **Run lint locally** with `uv run tomte tox -p -e black-check -e + isort-check -e flake8 -e mypy -e pylint -e darglint -e safety -e + bandit -e liccheck`. CI runs the same set. +- **Run unit tests locally** with `PYTHONPATH=. uv run pytest tests + --ignore=tests/test_run_service.py + --ignore=tests/test_staking_service.py + --ignore=tests/test_migrate_to_pearl.py`. The ignored files are + e2e tests that need Docker and live RPC secrets. ## Guide for the AI agent `config.json` diff --git a/Makefile b/Makefile deleted file mode 100644 index 63a3d98a..00000000 --- a/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -.PHONY: install test-install test run_no_staking_tests - -# Install project dependencies for produ -install: - poetry install --only main - -# Install project dependencies for testing -test-install: - poetry install - -# Run tests without staking functionality -run_no_staking_tests: - poetry run pytest -v tests/test_run_service.py -s --log-cli-level=INFO - -# Run all commands in sequence -test: test-install run_no_staking_tests \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index ef809cdf..d3540fe6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,20 +27,51 @@ dependencies = [ [dependency-groups] dev = [ "pexpect==4.9.0", - "pytest==9.0.3", - "pytest-cov==7.1.0", + "tomte[cli, tests]==0.7.0", + "tox-uv==1.16.0", ] +# Note: pytest, pytest-cov and pytest-asyncio are brought in by +# tomte[tests] at pinned versions (pytest==8.4.2, pytest-cov==7.0.0, +# pytest-asyncio==1.3.0). Don't pin them separately or uv will +# refuse to resolve. [tool.uv] package = false default-groups = "all" +# Some tomte testenvs (pylint, py3.x) trigger an editable install via tox. +# Without a build-system block setuptools falls back to flat-layout +# auto-discovery and trips on `data/`, `images/`, `configs/` at repo root. +# Declare an empty package set so the build is a no-op. +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +py-modules = [] + +[tool.tomte] +tomte_dep_pin = "==0.7.0" +# Quickstart has no Valory `packages//...` tree (it consumes +# pre-published packages via `olas-operate-middleware`). Lint scope +# is `scripts/` only — `tests/` is excluded from lint following +# trader's pattern (production code lints, tests don't). +packages_paths = "." +service_specific_packages = ["scripts"] +pytest_targets = ["tests"] + [tool.pytest.ini_options] -# anchorpy ships a pytest plugin (via open-autonomy -> open-aea-ledger-solana -> -# anchorpy) that imports pytest_xprocess at startup, but xprocess is only in -# anchorpy's [pytest] extra and is never installed. Disable the plugin so -# pytest can boot. -addopts = "-p no:pytest_anchorpy" +# `-p no:pytest_anchorpy`: anchorpy ships a pytest plugin (via +# open-autonomy -> open-aea-ledger-solana -> anchorpy) that imports +# pytest_xprocess at startup, but xprocess is only in anchorpy's +# [pytest] extra and is never installed. Disabling the plugin lets +# pytest boot. +# `-p no:randomly`: tomte[tests] pulls in pytest-randomly which shuffles +# test order by default. tests/test_migrate_to_pearl.py uses numeric +# `test_NN_*` naming because each test depends on state set up by the +# previous one (e.g. test_03_*_mode_b asserts pearl_home from +# test_02_*_mode_a). Disable the plugin so the sequential order holds. +addopts = "-p no:pytest_anchorpy -p no:randomly" markers = [ "e2e: end-to-end tests that may use real network/external systems", "integration: integration tests that may use networked dependencies", diff --git a/scripts/optimus/README.md b/scripts/optimus/README.md index a327882a..0db1330b 100644 --- a/scripts/optimus/README.md +++ b/scripts/optimus/README.md @@ -15,8 +15,8 @@ If you were previously using [optimus-quickstart](https://github.com/valory-xyz/ 2. Run the migration script to create the new `.operate` folder compatible with unified quickstart: ```bash - poetry install - poetry run python -m scripts.optimus.migrate_legacy_optimus configs/config_optimus.json + uv sync + uv run python -m scripts.optimus.migrate_legacy_optimus configs/config_optimus.json ``` 3. Follow the prompts to complete the migration process. The script will: diff --git a/scripts/optimus/migrate_legacy_optimus.py b/scripts/optimus/migrate_legacy_optimus.py index 4470dc30..43293867 100644 --- a/scripts/optimus/migrate_legacy_optimus.py +++ b/scripts/optimus/migrate_legacy_optimus.py @@ -1,18 +1,24 @@ -from argparse import ArgumentParser -from pathlib import Path +"""Migrate a legacy `.optimus/` operator state directory into the modern `.operate/` layout.""" + import json import shutil +from argparse import ArgumentParser from dataclasses import dataclass -from operate.quickstart.run_service import QuickstartConfig +from pathlib import Path + from operate.operate_types import Chain +from operate.quickstart.run_service import QuickstartConfig from operate.quickstart.utils import print_section, print_title -from scripts.utils import validate_config_params, handle_missing_rpcs +from scripts.utils import handle_missing_rpcs, validate_config_params OPTIMUS_PATH = Path(__file__).parent.parent.parent / ".optimus" OPERATE_HOME = Path(__file__).parent.parent.parent / ".operate" + @dataclass class OptimusConfig: + """User-facing optimus quickstart configuration parsed from the legacy `.optimus/` JSON.""" + rpc: dict[str, str] tenderly_access_key: str tenderly_account_slug: str @@ -22,31 +28,36 @@ class OptimusConfig: staking_program_id: str principal_chain: str + def parse_optimus_config() -> OptimusConfig: """Parse essential config data from optimus.""" print_section("Parsing .optimus configuration...") - + config_file = OPTIMUS_PATH / "local_config.json" config = json.loads(config_file.read_text()) - + # Validate required parameters required_params = [ "tenderly_access_key", - "tenderly_account_slug", + "tenderly_account_slug", "tenderly_project_slug", - "coingecko_api_key" + "coingecko_api_key", ] validate_config_params(config, required_params) - + # Map RPC endpoints from source config to new format rpc_mapping = handle_missing_rpcs(config) - + use_staking = config.get("use_staking", False) staking_program_id = "optimus_alpha" if use_staking else "no_staking" - + # Default to OPTIMISM if available, otherwise first available chain - principal_chain = Chain.OPTIMISM.value if "optimistic" in rpc_mapping else next(iter(rpc_mapping), Chain.OPTIMISM.value) - + principal_chain = ( + Chain.OPTIMISM.value + if "optimistic" in rpc_mapping + else next(iter(rpc_mapping), Chain.OPTIMISM.value) + ) + return OptimusConfig( rpc=rpc_mapping, tenderly_access_key=config["tenderly_access_key"], @@ -55,16 +66,17 @@ def parse_optimus_config() -> OptimusConfig: coingecko_api_key=config["coingecko_api_key"], use_staking=use_staking, staking_program_id=staking_program_id, - principal_chain=principal_chain + principal_chain=principal_chain, ) + def copy_optimus_to_operate(): """Copy all files from .optimus to .operate.""" print_section("Copying files from .optimus to .operate...") - + # Create .operate directory if it doesn't exist OPERATE_HOME.mkdir(parents=True, exist_ok=True) - + # Copy all contents except local_config.json for item in OPTIMUS_PATH.iterdir(): if item.name != "local_config.json": @@ -74,10 +86,11 @@ def copy_optimus_to_operate(): else: shutil.copy2(item, target) + def create_operate_config(optimus_config: OptimusConfig, service_name: str): """Create new local_config.json for operate using QuickstartConfig.""" print_section("Creating new operate configuration...") - + for service_dir in (OPERATE_HOME / "services").iterdir(): config_path = service_dir / "config.json" if not config_path.exists(): @@ -85,7 +98,7 @@ def create_operate_config(optimus_config: OptimusConfig, service_name: str): with open(config_path, "r") as f: config = json.load(f) - + if config["name"] != "valory/optimus": continue @@ -101,44 +114,49 @@ def create_operate_config(optimus_config: OptimusConfig, service_name: str): "TENDERLY_ACCESS_KEY": optimus_config.tenderly_access_key, "TENDERLY_ACCOUNT_SLUG": optimus_config.tenderly_account_slug, "TENDERLY_PROJECT_SLUG": optimus_config.tenderly_project_slug, - "COINGECKO_API_KEY": optimus_config.coingecko_api_key + "COINGECKO_API_KEY": optimus_config.coingecko_api_key, }, staking_program_id=optimus_config.staking_program_id, ) qs_config.store() + def main(config_path: Path) -> None: + """Run the optimus-to-operate migration, copying state and rewriting config files.""" print_title("Optimus to Operate Migration") - + if not OPTIMUS_PATH.exists(): print("Error: No .optimus folder found!") return - + try: with open(config_path, "r") as f: config = json.load(f) # Parse optimus config first optimus_config = parse_optimus_config() - + # Copy all files copy_optimus_to_operate() - + # Create new config create_operate_config(optimus_config, config["name"]) - + print_section("Migration completed successfully!") - + except Exception as e: print(f"Error during migration: {e}") raise + if __name__ == "__main__": - parser = ArgumentParser(description="Migrate legacy quickstart to unified quickstart") + parser = ArgumentParser( + description="Migrate legacy quickstart to unified quickstart" + ) parser.add_argument( dest="config_path", type=Path, help="Quickstart config file path", ) args = parser.parse_args() - main(args.config_path) \ No newline at end of file + main(args.config_path) diff --git a/scripts/pearl_migration/detect.py b/scripts/pearl_migration/detect.py index 258ee014..ed3a8bc8 100644 --- a/scripts/pearl_migration/detect.py +++ b/scripts/pearl_migration/detect.py @@ -17,7 +17,7 @@ from dataclasses import dataclass, field from enum import Enum from pathlib import Path -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from typing import Dict, List, Optional, TYPE_CHECKING, Tuple from operate.constants import KEYS_DIR, SERVICES_DIR, WALLETS_DIR from operate.operate_types import LedgerType @@ -32,8 +32,8 @@ class Mode(Enum): """Migration mode.""" FRESH_COPY = "fresh_copy" # Pearl `.operate` does not exist (or has no wallet) - MERGE = "merge" # Pearl has its own master wallet; we must merge - NOOP = "noop" # Quickstart `.operate` already is `~/.operate` + MERGE = "merge" # Pearl has its own master wallet; we must merge + NOOP = "noop" # Quickstart `.operate` already is `~/.operate` @dataclass(frozen=True) @@ -50,12 +50,16 @@ class OperateStore: allowing memoization. """ - root: Path # absolute, resolved + root: Path # absolute, resolved _state: Dict[str, "OperateApp"] = field( - default_factory=dict, repr=False, compare=False, hash=False, + default_factory=dict, + repr=False, + compare=False, + hash=False, ) def __post_init__(self) -> None: + """Normalise `root` to an absolute, resolved path; reject unreadable paths.""" # Enforce the "absolute + resolved" invariant the docstring promises. # Always resolve so absolute paths with `..` segments / symlinks # also normalize. Use object.__setattr__ because `frozen=True` @@ -76,19 +80,23 @@ def __post_init__(self) -> None: # target). Lazy import: prompts.py depends on operate.* which # we want kept off the detect.py import path. from .prompts import info as _info + _info(f"Resolved store root: {self.root} -> {resolved}") object.__setattr__(self, "root", resolved) @property def wallets_dir(self) -> Path: + """Absolute path to the `wallets/` sub-directory inside this store root.""" return self.root / WALLETS_DIR @property def keys_dir(self) -> Path: + """Absolute path to the `keys/` sub-directory inside this store root.""" return self.root / KEYS_DIR @property def services_dir(self) -> Path: + """Absolute path to the `services/` sub-directory inside this store root.""" return self.root / SERVICES_DIR def has_master_wallet(self) -> bool: @@ -180,6 +188,7 @@ class Discovery: mode: Mode def __post_init__(self) -> None: + """Validate that `mode` is consistent with whether the two store roots match.""" same_root = self.quickstart.root == self.pearl.root if same_root and self.mode != Mode.NOOP: raise ValueError( @@ -194,6 +203,7 @@ def __post_init__(self) -> None: @property def is_noop(self) -> bool: + """True iff the discovery resolved to Mode.NOOP (no migration needed).""" return self.mode == Mode.NOOP diff --git a/scripts/pearl_migration/filesystem.py b/scripts/pearl_migration/filesystem.py index f022ce64..12803020 100644 --- a/scripts/pearl_migration/filesystem.py +++ b/scripts/pearl_migration/filesystem.py @@ -24,8 +24,7 @@ class MissingAgentKey(OSError): - """Raised by `merge_service` when an agent key referenced by the - service config is not present at the source `keys/` dir. + """Raised when `merge_service` can't find an agent key referenced by the service config. Subclasses `OSError` so the existing `(OSError, shutil.Error)` catch in `_run_mode_b` aggregates it into `MigrationOutcome.unmigratable` @@ -41,6 +40,8 @@ class MissingAgentKey(OSError): @dataclass class CopyOutcome: + """Per-service outcome of a quickstart-to-pearl filesystem merge.""" + service_id: str service_copied: bool service_skipped: bool @@ -100,7 +101,7 @@ def fix_root_ownership(store: OperateStore) -> None: f"'sudo chown -RP {uid}:{gid}'. You may be prompted for your sudo password." ) try: - subprocess.run( + subprocess.run( # nosec B607 # `sudo` looked up via PATH is the desired UX # Plain interactive `sudo` (no `-n`): legacy # `run_service.sh` does the same and prints the same # "Please enter sudo password" notice. Using `-n` here diff --git a/scripts/pearl_migration/migrate_to_pearl.py b/scripts/pearl_migration/migrate_to_pearl.py index 636ac2cf..48bf2e36 100644 --- a/scripts/pearl_migration/migrate_to_pearl.py +++ b/scripts/pearl_migration/migrate_to_pearl.py @@ -37,7 +37,6 @@ from dataclasses import dataclass from pathlib import Path from typing import ( - TYPE_CHECKING, Any, Callable, Dict, @@ -45,6 +44,7 @@ Literal, NoReturn, Optional, + TYPE_CHECKING, Tuple, Type, ) @@ -58,6 +58,46 @@ ) from operate.services.service import Service from operate.utils.gnosis import get_asset_balance +from scripts.pearl_migration.detect import ( + Discovery, + Mode, + OperateStore, + discover, +) +from scripts.pearl_migration.filesystem import ( + fix_root_ownership, + fresh_copy_store, + merge_service, + rename_source_for_rollback, + reset_services_staking_to_no_staking, +) +from scripts.pearl_migration.prompts import ( + ask_password_validating, + fatal, + info, + warn, + yes_no, +) +from scripts.pearl_migration.status import ( + docker_quickstart_containers, + pearl_daemon_running, + safe_owners, + safe_threshold, + service_nft_owner, +) +from scripts.pearl_migration.stop import ( + force_remove_known_containers, + stop_via_middleware, +) +from scripts.pearl_migration.wallet import ( + align_quickstart_password, + align_user_account_to_wallet, +) + +if TYPE_CHECKING: + from operate.cli import OperateApp + from operate.services.manage import ServiceManager + from operate.wallet.master import MasterWallet # Programming bugs we DO NOT want wrapped as `_Unmigratable("NFT transfer # failed: ...")`. Letting these propagate surfaces the real fault (with a @@ -91,7 +131,11 @@ def _reraise_if_programming_bug(exc: BaseException) -> None: def _wrap_step_failure( - *, sid: str, chain: str, prefix: str, exc: BaseException, + *, + sid: str, + chain: str, + prefix: str, + exc: BaseException, ) -> "_Unmigratable": """Build the `_Unmigratable` for a step's `except` block. @@ -113,7 +157,8 @@ def _wrap_step_failure( """ _reraise_if_programming_bug(exc) return _Unmigratable( - service_id=sid, chain=chain, + service_id=sid, + chain=chain, reason=f"{prefix}: {_format_exc_chain(exc)}.", ) @@ -148,8 +193,10 @@ def _format_exc_chain(exc: BaseException) -> str: def _is_insufficient_funds_error(exc: BaseException) -> bool: - """Detect the middleware's "no funds" failure so we can wait for the - user to top up instead of marking the service un-migratable. + """Detect the middleware's "no funds" failure so we can wait for a top-up. + + Returning True means the caller should pause and wait for the user + to fund the safe instead of marking the service un-migratable. olas-operate-middleware raises a plain `RuntimeError` (no dedicated exception class) when an EOA can't pay gas — message-matching is the @@ -159,10 +206,7 @@ def _is_insufficient_funds_error(exc: BaseException) -> bool: than looping forever on an unrelated error. """ msg = str(exc).lower() - return ( - "does not have any funds" in msg - or "insufficient funds" in msg - ) + return "does not have any funds" in msg or "insufficient funds" in msg def _wait_for_native_funds( @@ -228,8 +272,7 @@ def _retry_on_funds_shortage( recipient_name: str, log_indent: str = " ", ) -> Any: - """Run `fn`, blocking on insufficient-funds errors with a - wait-and-retry loop against `address`. + """Run `fn` and block on insufficient-funds errors with a retry loop against `address`. Used at the four signing sites where the middleware actually propagates funds errors: `_step_terminate`, `_step_transfer_nft`, @@ -270,8 +313,7 @@ def _create_pearl_safe_with_funds_wait( ledger_api: Any, log_indent: str = " ", ) -> None: - """Create the Pearl master Safe on `chain`, blocking on insufficient - funds with a wait-and-retry loop. + """Create the Pearl master Safe on `chain`, blocking on insufficient funds. On non-funds errors the original exception is re-raised so the caller can wrap it with the right granularity (`_Unmigratable` per-service @@ -289,50 +331,6 @@ def _create_pearl_safe_with_funds_wait( ) -from scripts.pearl_migration.detect import ( - Discovery, - Mode, - OperateStore, - discover, -) - -if TYPE_CHECKING: - from operate.cli import OperateApp - from operate.services.manage import ServiceManager - from operate.services.service import Service - from operate.operate_types import Chain - from operate.wallet.master import MasterWallet -from scripts.pearl_migration.filesystem import ( - fix_root_ownership, - fresh_copy_store, - merge_service, - rename_source_for_rollback, - reset_services_staking_to_no_staking, -) -from scripts.pearl_migration.prompts import ( - ask_password_validating, - fatal, - info, - warn, - yes_no, -) -from scripts.pearl_migration.status import ( - docker_quickstart_containers, - pearl_daemon_running, - safe_owners, - safe_threshold, - service_nft_owner, -) -from scripts.pearl_migration.stop import ( - force_remove_known_containers, - stop_via_middleware, -) -from scripts.pearl_migration.wallet import ( - align_quickstart_password, - align_user_account_to_wallet, -) - - # --------------------------------------------------------------------------- # args # --------------------------------------------------------------------------- @@ -377,6 +375,7 @@ def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: # Service selection # --------------------------------------------------------------------------- + def _select_services( qs: OperateStore, config_path: Optional[str], @@ -400,8 +399,10 @@ def _select_services( # Piped/closed stdin (CI, automation) — re-prompting forever # would hang. Surface as a clean fatal instead of a raw # `EOFError: EOF when reading a line` traceback. - fatal("No selection provided (stdin closed). Re-run with " - "`--quickstart-config ` or attach a tty.") + fatal( + "No selection provided (stdin closed). Re-run with " + "`--quickstart-config ` or attach a tty." + ) if raw.isdigit(): n = int(raw) if 1 <= n <= len(services): @@ -412,6 +413,7 @@ def _select_services( # config_path given: match by 'name' or 'hash' import json + try: cfg = json.loads(Path(config_path).read_text()) except (OSError, json.JSONDecodeError) as exc: @@ -430,6 +432,7 @@ def _select_services( # Pre-flight # --------------------------------------------------------------------------- + def _preflight(disc: Discovery) -> None: info(f"Quickstart .operate: {disc.quickstart.root}") info(f"Pearl .operate: {disc.pearl.root}") @@ -460,8 +463,10 @@ def _preflight(disc: Discovery) -> None: # Wallet loading # --------------------------------------------------------------------------- + def _load_wallet( - store: OperateStore, label: str, + store: OperateStore, + label: str, ) -> Tuple["OperateApp", "MasterWallet"]: """Prompt for the password for `store` and return (OperateApp, wallet). @@ -501,6 +506,7 @@ def _load_wallet( # Shared cleanup helper (used by both Mode A and Mode B) # --------------------------------------------------------------------------- + def _ensure_containers_stopped( on_failure: Callable[[str], NoReturn], ) -> List[str]: @@ -551,10 +557,12 @@ def _ensure_containers_stopped( # Mode A — fresh copy # --------------------------------------------------------------------------- + def _run_mode_a(disc: Discovery, dry_run: bool) -> MigrationOutcome: - """Run the fresh-copy mode and return an empty `MigrationOutcome` - (which is `is_complete=True` by construction) so the caller can - plumb a uniform `MigrationOutcome` through `_final_prompt`. + """Run the fresh-copy mode and return an empty `MigrationOutcome`. + + The returned `MigrationOutcome` is `is_complete=True` by construction + so the caller can plumb a uniform value through `_final_prompt`. Mode A has no per-service / per-chain partial-failure state: any `OSError` / `shutil.Error` from the copy or rename callees propagates @@ -567,6 +575,7 @@ def _run_mode_a(disc: Discovery, dry_run: bool) -> MigrationOutcome: # If Pearl dir exists but no wallet, back it up so the copy is clean. if disc.pearl.root.exists(): from scripts.pearl_migration.prompts import backup_suffix as _bs + bak = disc.pearl.root.with_name(f"{disc.pearl.root.name}.bak.{_bs()}") if dry_run: info(f"[dry-run] Would back up empty Pearl dir to {bak}.") @@ -614,6 +623,7 @@ def _run_mode_a(disc: Discovery, dry_run: bool) -> MigrationOutcome: # Mode B — merge # --------------------------------------------------------------------------- + @dataclass(frozen=True) class _Unmigratable(Exception): """Raised when a single service can't proceed but others might. @@ -698,6 +708,7 @@ class MigrationOutcome: @property def is_complete(self) -> bool: + """True iff every selected service migrated cleanly and all drains succeeded.""" return ( not self.unmigratable and not self.drain_failures @@ -706,6 +717,7 @@ def is_complete(self) -> bool: @property def migrated_count(self) -> int: + """Number of services successfully migrated in this run.""" return len(self.migrated) @@ -716,8 +728,10 @@ def _run_mode_b( dry_run: bool, subset_selected: bool = False, ) -> MigrationOutcome: - """Returns the structured `MigrationOutcome`. `outcome.is_complete` False - suppresses the source-`.operate/` rename so the user can resume. + """Return the structured `MigrationOutcome` for the merge-mode migration. + + `outcome.is_complete = False` suppresses the source-`.operate/` rename + so the user can resume. `subset_selected` is the caller's signal that `services` is only some of what the quickstart store currently exposes (the user picked via @@ -729,10 +743,15 @@ def _run_mode_b( existing single-service test fixtures rely on. """ print_section("MERGE") - info(f"Will migrate {len(services)} service(s): " + ", ".join(s.name for s in services)) + info( + f"Will migrate {len(services)} service(s): " + + ", ".join(s.name for s in services) + ) if dry_run: - info("[dry-run] Stopping at discovery; no on-chain or filesystem actions taken.") + info( + "[dry-run] Stopping at discovery; no on-chain or filesystem actions taken." + ) return MigrationOutcome(subset_selected=subset_selected) # If `OperateStore.services()` dropped any malformed configs we MUST NOT @@ -791,8 +810,6 @@ def _run_mode_b( # didn't have one yet (using the same RPC the migration was driven by). chain_rpcs: dict = {} - from operate.operate_types import Chain as _Chain - for svc in services: print_section(f"Migrating service: {svc.name} ({svc.service_config_id})") # Snapshot the service's chain RPCs up front — used by the drain @@ -801,7 +818,7 @@ def _run_mode_b( # wins (deterministic) but we warn so the user can investigate # rather than silently inheriting one over the other. for chain_str, chain_config in svc.chain_configs.items(): - chain_key = _Chain(chain_str) + chain_key = Chain(chain_str) new_rpc = chain_config.ledger_config.rpc existing_rpc = chain_rpcs.get(chain_key) if existing_rpc is None: @@ -838,16 +855,19 @@ def _run_mode_b( merge_service(service=svc, src=disc.quickstart, dest=disc.pearl) except (OSError, shutil.Error) as exc: warn(f"Skipping service {svc.name}: filesystem copy failed: {exc}") - unmigratable.append(_Unmigratable( - service_id=svc.service_config_id, chain=None, - reason=( - f"on-chain migration committed but filesystem copy " - f"failed: {exc}. Manually copy " - f"`services/{svc.service_config_id}/` and the agent " - f"keys (under `keys/`) from {disc.quickstart.root} to " - f"{disc.pearl.root} before launching Pearl." - ), - )) + unmigratable.append( + _Unmigratable( + service_id=svc.service_config_id, + chain=None, + reason=( + f"on-chain migration committed but filesystem copy " + f"failed: {exc}. Manually copy " + f"`services/{svc.service_config_id}/` and the agent " + f"keys (under `keys/`) from {disc.quickstart.root} to " + f"{disc.pearl.root} before launching Pearl." + ), + ) + ) continue migrated.append(svc) @@ -869,7 +889,8 @@ def _run_mode_b( ) else: drain_failures = _drain_master( - qs_wallet=qs_wallet, pearl_wallet=pearl_wallet, + qs_wallet=qs_wallet, + pearl_wallet=pearl_wallet, chain_rpcs=chain_rpcs, ) @@ -896,7 +917,11 @@ def _run_mode_b( ): rename_source_for_rollback(disc.quickstart) elif not outcome.is_complete: - if outcome.subset_selected and not outcome.unmigratable and not outcome.drain_failures: + if ( + outcome.subset_selected + and not outcome.unmigratable + and not outcome.drain_failures + ): reason = "remaining services still live in the source" else: reason = "migration incomplete" @@ -949,10 +974,15 @@ def _print_migration_summary(outcome: MigrationOutcome) -> None: def _step_terminate( - *, manager: "ServiceManager", service: "Service", sid: str, chain_str: str, - ensure_signable, # callable[[], None] + *, + manager: "ServiceManager", + service: "Service", + sid: str, + chain_str: str, + ensure_signable, # callable[[], None] on_chain_state_cls: OnChainState, - ledger_api: Any, qs_address: str, + ledger_api: Any, + qs_address: str, ) -> None: """Move on-chain to PRE_REGISTRATION (idempotent). @@ -967,16 +997,20 @@ def _step_terminate( try: _retry_on_funds_shortage( lambda: manager.terminate_service_on_chain_from_safe( - service_config_id=sid, chain=chain_str, + service_config_id=sid, + chain=chain_str, ), - ledger_api=ledger_api, address=qs_address, chain_str=chain_str, + ledger_api=ledger_api, + address=qs_address, + chain_str=chain_str, recipient_name="Quickstart master EOA", ) except Exception as exc: # pylint: disable=broad-except raise _wrap_step_failure( - sid=sid, chain=chain_str, + sid=sid, + chain=chain_str, prefix="could not unstake/terminate; funds left in master " - "Safe/EOA so you can retry", + "Safe/EOA so you can retry", exc=exc, ) # Verification read after terminate succeeded. Wrap so a transient @@ -987,24 +1021,32 @@ def _step_terminate( on_chain_state = manager._get_on_chain_state(service, chain_str) # noqa: SLF001 except Exception as exc: # pylint: disable=broad-except raise _wrap_step_failure( - sid=sid, chain=chain_str, + sid=sid, + chain=chain_str, prefix="terminate succeeded but post-state verification " - "failed (on-chain progressed; re-run will resume from " - "the next step)", + "failed (on-chain progressed; re-run will resume from " + "the next step)", exc=exc, ) if on_chain_state != on_chain_state_cls.PRE_REGISTRATION: raise _Unmigratable( - service_id=sid, chain=chain_str, + service_id=sid, + chain=chain_str, reason=f"after terminate, service is in state {on_chain_state.name}, " - "expected PRE_REGISTRATION.", + "expected PRE_REGISTRATION.", ) def _step_transfer_nft( - *, ledger_api: Any, qs_wallet: Any, registry_addr: str, - qs_master_safe: str, pearl_master_safe: str, - token_id: int, sid: str, chain_str: str, + *, + ledger_api: Any, + qs_wallet: Any, + registry_addr: str, + qs_master_safe: str, + pearl_master_safe: str, + token_id: int, + sid: str, + chain_str: str, ensure_signable, ) -> None: """Transfer the service NFT to Pearl's master Safe (idempotent).""" @@ -1020,32 +1062,39 @@ def _step_transfer_nft( ) if current_owner is None: raise _Unmigratable( - service_id=sid, chain=chain_str, + service_id=sid, + chain=chain_str, reason=f"could not read NFT owner for token {token_id}; " - "RPC error or contract revert.", + "RPC error or contract revert.", ) if current_owner.lower() == pearl_master_safe.lower(): info(" NFT already owned by Pearl master Safe; skipping transfer.") return if current_owner.lower() != qs_master_safe.lower(): raise _Unmigratable( - service_id=sid, chain=chain_str, + service_id=sid, + chain=chain_str, reason=f"NFT owner is {current_owner}, expected quickstart " - f"({qs_master_safe}) or Pearl ({pearl_master_safe}) Safe.", + f"({qs_master_safe}) or Pearl ({pearl_master_safe}) Safe.", ) ensure_signable() - info(f" transferring service NFT {token_id} {qs_master_safe} -> {pearl_master_safe}") + info( + f" transferring service NFT {token_id} {qs_master_safe} -> {pearl_master_safe}" + ) try: _retry_on_funds_shortage( lambda: transfer_service_nft( - ledger_api=ledger_api, crypto=qs_wallet.crypto, + ledger_api=ledger_api, + crypto=qs_wallet.crypto, service_registry_address=registry_addr, qs_master_safe=qs_master_safe, pearl_master_safe=pearl_master_safe, service_id=token_id, ), - ledger_api=ledger_api, address=qs_wallet.address, - chain_str=chain_str, recipient_name="Quickstart master EOA", + ledger_api=ledger_api, + address=qs_wallet.address, + chain_str=chain_str, + recipient_name="Quickstart master EOA", ) except PostConditionUnknown as exc: # Distinct from "inner reverted": tx submitted, but the post-tx @@ -1056,20 +1105,28 @@ def _step_transfer_nft( # intentional translation so the underlying RPC failure stays # visible in the traceback chain. raise _Unmigratable( - service_id=sid, chain=chain_str, + service_id=sid, + chain=chain_str, reason=str(exc), ) from exc except Exception as exc: # pylint: disable=broad-except raise _wrap_step_failure( - sid=sid, chain=chain_str, - prefix="NFT transfer failed", exc=exc, + sid=sid, + chain=chain_str, + prefix="NFT transfer failed", + exc=exc, ) def _step_swap_service_safe_owner( - *, ledger_api: Any, qs_wallet: Any, service_safe: str, - qs_master_safe: str, pearl_master_safe: str, - sid: str, chain_str: str, + *, + ledger_api: Any, + qs_wallet: Any, + service_safe: str, + qs_master_safe: str, + pearl_master_safe: str, + sid: str, + chain_str: str, ensure_signable, ) -> None: """Swap the service multisig's owner from qs Safe to Pearl Safe (idempotent).""" @@ -1082,7 +1139,8 @@ def _step_swap_service_safe_owner( owners = safe_owners(ledger_api=ledger_api, safe=service_safe) except Exception as exc: # pylint: disable=broad-except raise _wrap_step_failure( - sid=sid, chain=chain_str, + sid=sid, + chain=chain_str, prefix=f"could not read service Safe owners on {service_safe}", exc=exc, ) @@ -1094,21 +1152,26 @@ def _step_swap_service_safe_owner( return if not qs_present: raise _Unmigratable( - service_id=sid, chain=chain_str, + service_id=sid, + chain=chain_str, reason=f"service Safe {service_safe} owners are {owners}; " - f"quickstart Safe ({qs_master_safe}) not found, can't swap.", + f"quickstart Safe ({qs_master_safe}) not found, can't swap.", ) ensure_signable() info(f" swapping owner on service Safe {service_safe}") try: _retry_on_funds_shortage( lambda: swap_service_safe_owner( - ledger_api=ledger_api, crypto=qs_wallet.crypto, + ledger_api=ledger_api, + crypto=qs_wallet.crypto, service_safe=service_safe, - old_owner=qs_master_safe, new_owner=pearl_master_safe, + old_owner=qs_master_safe, + new_owner=pearl_master_safe, ), - ledger_api=ledger_api, address=qs_wallet.address, - chain_str=chain_str, recipient_name="Quickstart master EOA", + ledger_api=ledger_api, + address=qs_wallet.address, + chain_str=chain_str, + recipient_name="Quickstart master EOA", ) except PostConditionUnknown as exc: # State is INDETERMINATE — execTransaction submitted but the @@ -1120,12 +1183,14 @@ def _step_swap_service_safe_owner( # "verify on a block explorer first" guidance. `from exc` # preserves the underlying RPC failure in the traceback chain. raise _Unmigratable( - service_id=sid, chain=chain_str, + service_id=sid, + chain=chain_str, reason=str(exc), ) from exc except Exception as exc: # pylint: disable=broad-except raise _wrap_step_failure( - sid=sid, chain=chain_str, + sid=sid, + chain=chain_str, prefix=( f"service Safe owner swap failed; NFT is now owned by Pearl " f"Safe but service multisig {service_safe} still lists " @@ -1170,6 +1235,7 @@ def _migrate_one_service( # the on-chain branch. _reraise_if_programming_bug(exc) warn(f"stop_service via middleware failed: {exc}; trying force cleanup.") + # Per-service variant of the global stop-and-verify: convert any # cleanup failure into `_Unmigratable(service_id=...)` so the # orchestrator can keep migrating other services. Proceeding to NFT @@ -1184,7 +1250,9 @@ def _bail(reason: str) -> NoReturn: for chain_str, chain_config in service.chain_configs.items(): chain = Chain(chain_str) chain_data = chain_config.chain_data - info(f" chain {chain_str}: service NFT id = {chain_data.token}, multisig = {chain_data.multisig}") + info( + f" chain {chain_str}: service NFT id = {chain_data.token}, multisig = {chain_data.multisig}" + ) # Both `qs_wallet.safes[chain]` and `CONTRACTS[chain][...]` raise # KeyError on missing entries — legitimate per-service conditions @@ -1197,19 +1265,21 @@ def _bail(reason: str) -> NoReturn: qs_master_safe = qs_wallet.safes[chain] except KeyError: raise _Unmigratable( - service_id=sid, chain=chain_str, + service_id=sid, + chain=chain_str, reason=f"quickstart master wallet has no Safe on {chain_str}; " - "service references a chain the wallet was never " - "configured for.", + "service references a chain the wallet was never " + "configured for.", ) try: registry_addr = CONTRACTS[chain]["service_registry"] except KeyError: raise _Unmigratable( - service_id=sid, chain=chain_str, + service_id=sid, + chain=chain_str, reason=f"no ServiceRegistry contract registered for {chain_str} " - "in middleware's CONTRACTS table; cannot read NFT owner " - "or build the transfer.", + "in middleware's CONTRACTS table; cannot read NFT owner " + "or build the transfer.", ) # Ensure Pearl has a master Safe on this chain. `create_safe` is # the same factory that bootstraps Pearl on a new chain — required @@ -1224,13 +1294,15 @@ def _bail(reason: str) -> NoReturn: chain_str=chain_str, rpc=chain_config.ledger_config.rpc, ledger_api=pearl_wallet.ledger_api( - chain=chain, rpc=chain_config.ledger_config.rpc, + chain=chain, + rpc=chain_config.ledger_config.rpc, ), ) except Exception as exc: # pylint: disable=broad-except _reraise_if_programming_bug(exc) raise _Unmigratable( - service_id=sid, chain=chain_str, + service_id=sid, + chain=chain_str, reason=f"could not create Pearl master Safe on {chain_str}: {exc}.", ) pearl_master_safe = pearl_wallet.safes[chain] @@ -1255,54 +1327,79 @@ def _ledger_api(_cfg=chain_config) -> Any: ).ledger_api return ledger_api - def _ensure_signable() -> None: + def _ensure_signable( + _qs_safe: str = qs_master_safe, + _chain: str = chain_str, + ) -> None: """Raise `_Unmigratable` unless the qs master Safe is single-signer. Called lazily, only inside branches that actually sign. Resumed runs that find every step already done skip this entirely. + + `_qs_safe` and `_chain` use default-arg binding to capture the + loop-iteration's `qs_master_safe` / `chain_str` at definition + time. The closure is invoked synchronously within the same + iteration today, so the binding is defensive: it survives a + future refactor that defers the call to after the loop ends. + See B023 in the bugbear ruleset for the underlying pitfall. """ try: - t = safe_threshold(ledger_api=_ledger_api(), safe=qs_master_safe) + t = safe_threshold(ledger_api=_ledger_api(), safe=_qs_safe) except Exception as exc: # pylint: disable=broad-except _reraise_if_programming_bug(exc) raise _Unmigratable( - service_id=sid, chain=chain_str, + service_id=sid, + chain=_chain, reason=f"could not read quickstart master Safe threshold on " - f"{qs_master_safe}: {exc}.", + f"{_qs_safe}: {exc}.", ) if t == 1: return if t == 0: raise _Unmigratable( - service_id=sid, chain=chain_str, - reason=f"quickstart master Safe {qs_master_safe} reports " - f"threshold 0 — appears unconfigured. Inspect on-chain " - "state before retrying.", + service_id=sid, + chain=_chain, + reason=f"quickstart master Safe {_qs_safe} reports " + f"threshold 0 — appears unconfigured. Inspect on-chain " + "state before retrying.", ) raise _Unmigratable( - service_id=sid, chain=chain_str, - reason=f"quickstart master Safe {qs_master_safe} has threshold " - f"{t}; this script signs single-owner only. Lower the " - "threshold to 1 (temporarily) and re-run.", + service_id=sid, + chain=_chain, + reason=f"quickstart master Safe {_qs_safe} has threshold " + f"{t}; this script signs single-owner only. Lower the " + "threshold to 1 (temporarily) and re-run.", ) _step_terminate( - manager=manager, service=service, sid=sid, chain_str=chain_str, - ensure_signable=_ensure_signable, on_chain_state_cls=OnChainState, - ledger_api=_ledger_api(), qs_address=qs_wallet.address, + manager=manager, + service=service, + sid=sid, + chain_str=chain_str, + ensure_signable=_ensure_signable, + on_chain_state_cls=OnChainState, + ledger_api=_ledger_api(), + qs_address=qs_wallet.address, ) _step_transfer_nft( - ledger_api=_ledger_api(), qs_wallet=qs_wallet, + ledger_api=_ledger_api(), + qs_wallet=qs_wallet, registry_addr=registry_addr, - qs_master_safe=qs_master_safe, pearl_master_safe=pearl_master_safe, - token_id=chain_data.token, sid=sid, chain_str=chain_str, + qs_master_safe=qs_master_safe, + pearl_master_safe=pearl_master_safe, + token_id=chain_data.token, + sid=sid, + chain_str=chain_str, ensure_signable=_ensure_signable, ) _step_swap_service_safe_owner( - ledger_api=_ledger_api(), qs_wallet=qs_wallet, + ledger_api=_ledger_api(), + qs_wallet=qs_wallet, service_safe=chain_data.multisig, - qs_master_safe=qs_master_safe, pearl_master_safe=pearl_master_safe, - sid=sid, chain_str=chain_str, + qs_master_safe=qs_master_safe, + pearl_master_safe=pearl_master_safe, + sid=sid, + chain_str=chain_str, ensure_signable=_ensure_signable, ) @@ -1311,9 +1408,7 @@ def _ensure_signable() -> None: # NFT owned by Pearl Safe). It's required because Pearl's FE expects it # to continue the onbarding of this agent. for chain_config in service.chain_configs.values(): - chain_config.chain_data.user_params.staking_program_id = ( - NO_STAKING_PROGRAM_ID - ) + chain_config.chain_data.user_params.staking_program_id = NO_STAKING_PROGRAM_ID try: service.store() except OSError as exc: @@ -1321,7 +1416,8 @@ def _ensure_signable() -> None: # filesystem copy (`merge_service`) and surfaces in the summary # so the user can re-run after fixing the disk/permission issue. raise _Unmigratable( - service_id=sid, chain=None, + service_id=sid, + chain=None, reason=( "on-chain migration committed but local " f"`staking_program_id` write failed: {exc}. Re-run after " @@ -1332,8 +1428,10 @@ def _ensure_signable() -> None: def _format_amount(amount: int, chain: "Chain", asset: str) -> str: - """Best-effort token formatter; falls back to raw wei when the asset - isn't in the middleware's `CHAIN_TO_METADATA` table. + """Best-effort token formatter that falls back to raw wei for unknown assets. + + Falls back to raw wei when the asset isn't in the middleware's + `CHAIN_TO_METADATA` table. Only catches the lookup failures (`KeyError` for unknown chain/asset); everything else propagates so a bug in the formatter doesn't masquerade @@ -1346,7 +1444,8 @@ def _format_amount(amount: int, chain: "Chain", asset: str) -> str: def _drain_master( - qs_wallet: "MasterWallet", pearl_wallet: "MasterWallet", + qs_wallet: "MasterWallet", + pearl_wallet: "MasterWallet", chain_rpcs: Optional[Dict["Chain", str]] = None, ) -> List[_DrainFailure]: """Drain quickstart master Safe + EOA into Pearl per chain. @@ -1379,15 +1478,17 @@ def _drain_master( "(quickstart had a Safe here but no migrated service " "exposed an RPC for this chain)." ) - failures.append(_DrainFailure( - chain=chain, - source_kind="Safe+EOA", - source_address=qs_wallet.safes[chain], - reason="no RPC available — quickstart had a Safe on " - f"{chain.name} but no migrated service used " - "this chain; cannot create Pearl Safe to " - "drain into.", - )) + failures.append( + _DrainFailure( + chain=chain, + source_kind="Safe+EOA", + source_address=qs_wallet.safes[chain], + reason="no RPC available — quickstart had a Safe on " + f"{chain.name} but no migrated service used " + "this chain; cannot create Pearl Safe to " + "drain into.", + ) + ) continue try: _create_pearl_safe_with_funds_wait( @@ -1401,12 +1502,14 @@ def _drain_master( except Exception as exc: # pylint: disable=broad-except _reraise_if_programming_bug(exc) warn(f" could not create Pearl Safe on {chain.name}: {exc}") - failures.append(_DrainFailure( - chain=chain, - source_kind="Safe+EOA", - source_address=qs_wallet.safes[chain], - reason=f"Pearl Safe creation failed: {exc}.", - )) + failures.append( + _DrainFailure( + chain=chain, + source_kind="Safe+EOA", + source_address=qs_wallet.safes[chain], + reason=f"Pearl Safe creation failed: {exc}.", + ) + ) continue # NOTE: `qs_wallet.drain` is NOT wrapped in `_retry_on_funds_shortage`. @@ -1421,7 +1524,9 @@ def _drain_master( # after topping up; the drain is idempotent (per-asset balance # check before each transfer). # Master Safe -> Pearl master Safe (Pearl Safe receives ERC20s + native). - info(f" [{chain.name}] master Safe -> Pearl master Safe ({pearl_wallet.safes[chain]})") + info( + f" [{chain.name}] master Safe -> Pearl master Safe ({pearl_wallet.safes[chain]})" + ) try: moved = qs_wallet.drain( withdrawal_address=pearl_wallet.safes[chain], @@ -1439,13 +1544,19 @@ def _drain_master( except Exception as exc: # pylint: disable=broad-except _reraise_if_programming_bug(exc) warn(f" drain (Safe) on {chain.name} failed: {exc}") - failures.append(_DrainFailure( - chain=chain, source_kind="Safe", - source_address=qs_wallet.safes[chain], reason=str(exc), - )) + failures.append( + _DrainFailure( + chain=chain, + source_kind="Safe", + source_address=qs_wallet.safes[chain], + reason=str(exc), + ) + ) # Master EOA -> Pearl master EOA. - info(f" [{chain.name}] master EOA -> Pearl master EOA ({pearl_wallet.address})") + info( + f" [{chain.name}] master EOA -> Pearl master EOA ({pearl_wallet.address})" + ) try: moved = qs_wallet.drain( withdrawal_address=pearl_wallet.address, @@ -1460,10 +1571,14 @@ def _drain_master( except Exception as exc: # pylint: disable=broad-except _reraise_if_programming_bug(exc) warn(f" drain (EOA) on {chain.name} failed: {exc}") - failures.append(_DrainFailure( - chain=chain, source_kind="EOA", - source_address=qs_wallet.address, reason=str(exc), - )) + failures.append( + _DrainFailure( + chain=chain, + source_kind="EOA", + source_address=qs_wallet.address, + reason=str(exc), + ) + ) return failures @@ -1472,6 +1587,7 @@ def _drain_master( # Final user message # --------------------------------------------------------------------------- + def _final_prompt(outcome: MigrationOutcome) -> None: if not outcome.is_complete: # Don't ask the "different machine?" question on a partial migration @@ -1483,7 +1599,9 @@ def _final_prompt(outcome: MigrationOutcome) -> None: and not outcome.unmigratable and not outcome.drain_failures ): - print_section("Selected services migrated; remaining services left in source.") + print_section( + "Selected services migrated; remaining services left in source." + ) print() print(" The migrated services will appear in Pearl. The source") print(" quickstart `.operate/` is preserved with the unselected") @@ -1516,7 +1634,9 @@ def _final_prompt(outcome: MigrationOutcome) -> None: # Main # --------------------------------------------------------------------------- + def main(argv: Optional[List[str]] = None) -> int: + """Entry point for the quickstart-to-Pearl migration CLI; returns a process exit code.""" args = _parse_args(argv) print_title("Quickstart -> Pearl migration") diff --git a/scripts/pearl_migration/prompts.py b/scripts/pearl_migration/prompts.py index deb47346..481cbff5 100644 --- a/scripts/pearl_migration/prompts.py +++ b/scripts/pearl_migration/prompts.py @@ -20,6 +20,8 @@ class CollisionChoice(Enum): + """User's resolution for a filesystem collision on a per-service merge.""" + SKIP = "skip" OVERWRITE_WITH_BACKUP = "overwrite" @@ -62,11 +64,13 @@ def yes_no(question: str, default: Optional[bool] = None) -> bool: def _attended() -> bool: import os + return os.environ.get("ATTENDED", "true").lower() == "true" def password( - prompt: str = "Password: ", env_var_name: str = "OPERATE_PASSWORD", + prompt: str = "Password: ", + env_var_name: str = "OPERATE_PASSWORD", ) -> str: """Password prompt, defers to middleware's `ask_or_get_from_env`. @@ -99,8 +103,12 @@ def collision(target: Path, kind: str) -> CollisionChoice: " This usually means you're re-running a migration that previously " "got partway. Pick how to handle it:" ) - print(" 1) Skip — leave the destination untouched (safe if it's the migrated copy).") - print(" 2) Overwrite with backup — rename existing to .bak. then copy fresh.") + print( + " 1) Skip — leave the destination untouched (safe if it's the migrated copy)." + ) + print( + " 2) Overwrite with backup — rename existing to .bak. then copy fresh." + ) while True: try: raw = input(" Choice [1/2]: ").strip() @@ -118,19 +126,22 @@ def collision(target: Path, kind: str) -> CollisionChoice: def backup_suffix() -> str: - """Timestamp suffix used for `.bak.` paths.""" + """Return `bak.` so two concurrent backups can't collide.""" return f"bak.{int(time.time())}" def info(msg: str) -> None: + """Print `msg` as an info line to stdout.""" print(f" - {msg}") def warn(msg: str) -> None: + """Print `msg` as a warning line to stdout.""" print(f" ! {msg}") def fatal(msg: str, code: int = 1) -> None: + """Print `msg` to stderr and exit with `code` (default 1).""" print(f" X {msg}", file=sys.stderr) sys.exit(code) diff --git a/scripts/pearl_migration/status.py b/scripts/pearl_migration/status.py index e464353c..1f0435b6 100644 --- a/scripts/pearl_migration/status.py +++ b/scripts/pearl_migration/status.py @@ -10,7 +10,7 @@ import socket import subprocess from pathlib import Path -from typing import TYPE_CHECKING, List, Optional +from typing import List, Optional, TYPE_CHECKING if TYPE_CHECKING: from aea.crypto.base import LedgerApi @@ -27,7 +27,9 @@ QUICKSTART_CONTAINER_FRAGMENTS = ("abci0", "node0", "_abci_0", "_tm_0") -def pearl_daemon_running(host: str = "127.0.0.1", port: int = PEARL_DAEMON_PORT) -> bool: +def pearl_daemon_running( + host: str = "127.0.0.1", port: int = PEARL_DAEMON_PORT +) -> bool: """TCP probe: True if anything is listening on Pearl's daemon port. Only treats "connection refused" / "timeout" as "not running". Any @@ -52,7 +54,7 @@ def docker_quickstart_containers() -> List[str]: proceed believing there are no containers when there actually are. """ try: - result = subprocess.run( + result = subprocess.run( # nosec B607 # `docker` looked up via PATH is the desired UX ["docker", "ps", "-a", "--format", "{{.Names}}"], check=False, capture_output=True, @@ -60,7 +62,7 @@ def docker_quickstart_containers() -> List[str]: timeout=10, ) except FileNotFoundError: - return [] # docker not installed at all — caller can proceed. + return [] # docker not installed at all — caller can proceed. # subprocess.TimeoutExpired and any other error propagates. if result.returncode != 0: # Non-zero with docker installed = daemon refusing to talk @@ -79,7 +81,8 @@ def docker_quickstart_containers() -> List[str]: # let the migration race a still-running deployment. names = result.stdout.split() return sorted( - name for name in names + name + for name in names if any(frag in name for frag in QUICKSTART_CONTAINER_FRAGMENTS) ) @@ -128,9 +131,10 @@ def service_nft_owner( other unexpected exception propagate so the caller can distinguish "token doesn't exist" from "we can't tell right now". """ - from autonomy.chain.base import registry_contracts from web3.exceptions import ContractLogicError + from autonomy.chain.base import registry_contracts + instance = registry_contracts.service_registry.get_instance( ledger_api=ledger_api, contract_address=service_registry_address, diff --git a/scripts/pearl_migration/stop.py b/scripts/pearl_migration/stop.py index 41ce1550..6c38149f 100644 --- a/scripts/pearl_migration/stop.py +++ b/scripts/pearl_migration/stop.py @@ -3,7 +3,7 @@ from __future__ import annotations import subprocess -from typing import TYPE_CHECKING, List +from typing import List, TYPE_CHECKING from .status import QUICKSTART_CONTAINER_FRAGMENTS, docker_quickstart_containers @@ -70,7 +70,7 @@ def force_remove_known_containers() -> List[str]: if not leftovers: return [] try: - subprocess.run( + subprocess.run( # nosec B607 # `docker` looked up via PATH is the desired UX ["docker", "rm", "-f", *leftovers], check=True, capture_output=True, diff --git a/scripts/pearl_migration/transfer.py b/scripts/pearl_migration/transfer.py index 0e45f8ed..9d396e27 100644 --- a/scripts/pearl_migration/transfer.py +++ b/scripts/pearl_migration/transfer.py @@ -54,7 +54,7 @@ import asyncio import time -from typing import TYPE_CHECKING, Callable, TypeVar +from typing import Callable, TYPE_CHECKING, TypeVar import requests.exceptions import web3.exceptions @@ -112,6 +112,7 @@ class PostConditionUnknown(RuntimeError): """ def __init__(self, tx_hash: str, last_exc: BaseException) -> None: + """Build the indeterminate-state error for `tx_hash`, carrying `last_exc` as cause.""" super().__init__( f"On-chain state INDETERMINATE after Safe tx {tx_hash}: the " f"outer tx mined, but the post-tx verification read failed " @@ -160,7 +161,7 @@ def _read_with_retry( except _RPC_EXCEPTION_TYPES as exc: last_exc = exc if n + 1 < attempts: - time.sleep(backoff * (2 ** n)) + time.sleep(backoff * (2**n)) raise PostConditionUnknown(tx_hash=tx_hash, last_exc=last_exc) @@ -180,9 +181,10 @@ def transfer_service_nft( Returns the transaction hash from `send_safe_txs`. Raises on chain errors. """ - from autonomy.chain.base import registry_contracts from operate.utils.gnosis import send_safe_txs + from autonomy.chain.base import registry_contracts + instance = registry_contracts.service_registry.get_instance( ledger_api=ledger_api, contract_address=service_registry_address, @@ -254,7 +256,6 @@ def swap_service_safe_owner( and proceeds. The inner call is a self-call into swapOwner, which then sees msg.sender == address(this).) """ - from autonomy.chain.base import registry_contracts from operate.services.protocol import ( get_packed_signature_for_approved_hash, ) @@ -265,8 +266,12 @@ def swap_service_safe_owner( send_safe_txs, ) + from autonomy.chain.base import registry_contracts + prev_owner = get_prev_owner( - ledger_api=ledger_api, safe=service_safe, owner=old_owner, + ledger_api=ledger_api, + safe=service_safe, + owner=old_owner, ) service_safe_instance = registry_contracts.gnosis_safe.get_instance( ledger_api=ledger_api, @@ -308,15 +313,15 @@ def swap_service_safe_owner( exec_data_hex = service_safe_instance.encode_abi( abi_element_identifier="execTransaction", args=[ - service_safe, # to (self-call) - 0, # value - swap_owner_data, # data - SafeOperation.CALL.value, # operation - 0, # safeTxGas - 0, # baseGas - 0, # gasPrice - "0x" + "00" * 20, # gasToken - "0x" + "00" * 20, # refundReceiver + service_safe, # to (self-call) + 0, # value + swap_owner_data, # data + SafeOperation.CALL.value, # operation + 0, # safeTxGas + 0, # baseGas + 0, # gasPrice + "0x" + "00" * 20, # gasToken + "0x" + "00" * 20, # refundReceiver get_packed_signature_for_approved_hash( owners=(old_owner,), ), @@ -337,7 +342,8 @@ def swap_service_safe_owner( # read so a transient RPC hiccup post-mining doesn't surface as # "swap failed" when the on-chain side actually completed. owners_after = { - o.lower() for o in _read_with_retry( + o.lower() + for o in _read_with_retry( lambda: get_owners(ledger_api=ledger_api, safe=service_safe), tx_hash=exec_tx_hash, ) diff --git a/scripts/pearl_migration/wallet.py b/scripts/pearl_migration/wallet.py index 38daa7e3..7221a8b2 100644 --- a/scripts/pearl_migration/wallet.py +++ b/scripts/pearl_migration/wallet.py @@ -24,6 +24,7 @@ recovery artifact, so removing it before the issue is resolved would be unrecoverable. """ + from __future__ import annotations import json diff --git a/scripts/predict_trader/README.md b/scripts/predict_trader/README.md index b5da1d7f..d222f241 100644 --- a/scripts/predict_trader/README.md +++ b/scripts/predict_trader/README.md @@ -13,19 +13,19 @@ A quickstart for the Omenstrat AI agent for AI prediction markets on Gnosis at h 1. Use the `trades` command to display information about placed trades by a given address: ```bash - poetry run python -m scripts.predict_trader.trades --creator YOUR_SAFE_ADDRESS + uv run python -m scripts.predict_trader.trades --creator YOUR_SAFE_ADDRESS ``` Or restrict the search to specific dates by defining the "from" and "to" dates: ```bash - poetry run python -m scripts.predict_trader.trades --creator YOUR_SAFE_ADDRESS --from-date 2023-08-15:03:50:00 --to-date 2023-08-20:13:45:00 + uv run python -m scripts.predict_trader.trades --creator YOUR_SAFE_ADDRESS --from-date 2023-08-15:03:50:00 --to-date 2023-08-20:13:45:00 ``` 2. Use the `report` command to display a summary of the AI agent status: ```bash - poetry run python -m scripts.predict_trader.report + uv run python -m scripts.predict_trader.report ``` 3. Use this command to investigate your agent instance's logs: @@ -194,8 +194,8 @@ If you were previously using [trader-quickstart](https://github.com/valory-xyz/t 2. Run the migration script to create the new `.operate` folder compatible with unified quickstart: ```bash - poetry install - poetry run python -m scripts.predict_trader.migrate_legacy_quickstart configs/config_predict_trader.json + uv sync + uv run python -m scripts.predict_trader.migrate_legacy_quickstart configs/config_predict_trader.json ``` 3. Follow the prompts to complete the migration process. The script will: diff --git a/scripts/predict_trader/mech_events.py b/scripts/predict_trader/mech_events.py index b4428052..52c92bd3 100644 --- a/scripts/predict_trader/mech_events.py +++ b/scripts/predict_trader/mech_events.py @@ -22,21 +22,20 @@ import json import os -import traceback -import requests import sys import time +import traceback from dataclasses import dataclass from pathlib import Path from string import Template -from tqdm import tqdm from typing import Any, ClassVar, Dict, Optional +import requests from gql import Client, gql from gql.transport.requests import RequestsHTTPTransport +from tqdm import tqdm from web3.datastructures import AttributeDict - SCRIPT_PATH = Path(__file__).resolve().parent MECH_EVENTS_JSON_PATH = Path(SCRIPT_PATH.parents[1], "data", "mech_events.json") HTTP = "http://" @@ -47,14 +46,15 @@ DEFAULT_MECH_FEE = 10000000000000000 DEFAULT_FROM_TIMESTAMP = 0 DEFAULT_TO_TIMESTAMP = 2147483647 -MECH_SUBGRAPH_URL_TEMPLATE = "https://api.subgraph.autonolas.tech/api/proxy/marketplace-gnosis" +MECH_SUBGRAPH_URL_TEMPLATE = ( + "https://api.subgraph.autonolas.tech/api/proxy/marketplace-gnosis" +) SUBGRAPH_HEADERS = { "Accept": "application/json, multipart/mixed", "Content-Type": "application/json", } QUERY_BATCH_SIZE = 1000 -MECH_EVENTS_SUBGRAPH_QUERY_TEMPLATE = Template( - """ +MECH_EVENTS_SUBGRAPH_QUERY_TEMPLATE = Template(""" query mech_events_subgraph_query($sender: String, $id_gt: ID, $first: Int) { ${subgraph_event_set_name}( where: {sender: $sender, id_gt: $id_gt} @@ -69,20 +69,20 @@ transactionHash blockNumber blockTimestamp - + # Fetch from legacy mech request mechRequest { ipfsHash } - + # Fetch from marketplace request marketplaceRequest { ipfsHashBytes } } } - """ -) + """) + @dataclass class MechBaseEvent: # pylint: disable=too-many-instance-attributes @@ -131,12 +131,14 @@ def _populate_ipfs_contents(self) -> None: elif self.ipfs_hash_bytes: url = f"{IPFS_ADDRESS}{CID_PREFIX}{self.ipfs_hash_bytes}" else: - print(f"WARNING: No IPFS hash found for Mech event {self.event_name} with ID {self.event_id}.") + print( + f"WARNING: No IPFS hash found for Mech event {self.event_name} with ID {self.event_id}." + ) return for _url in [f"{url}/metadata.json", url]: try: - response = requests.get(_url) + response = requests.get(_url, timeout=30) response.raise_for_status() self.ipfs_contents = response.json() self.ipfs_link = _url @@ -145,7 +147,7 @@ def _populate_ipfs_contents(self) -> None: except Exception: # pylint: disable=broad-except print(traceback.format_exc()) - input(f"Press Enter to continue...") + input("Press Enter to continue...") @dataclass @@ -164,8 +166,14 @@ def __init__(self, event: AttributeDict): super().__init__( event_id=event["id"], sender=event["sender"]["id"], - ipfs_hash=event["mechRequest"]["ipfsHash"] if event["mechRequest"] else None, - ipfs_hash_bytes=event["marketplaceRequest"]["ipfsHashBytes"] if event["marketplaceRequest"] else None, + ipfs_hash=( + event["mechRequest"]["ipfsHash"] if event["mechRequest"] else None + ), + ipfs_hash_bytes=( + event["marketplaceRequest"]["ipfsHashBytes"] + if event["marketplaceRequest"] + else None + ), transaction_hash=event["transactionHash"], block_number=int(event["blockNumber"]), block_timestamp=int(event["blockTimestamp"]), @@ -186,7 +194,9 @@ def _read_mech_events_data_from_file() -> Dict[str, Any]: if mech_events_data.get("db_version", 0) < MECH_EVENTS_DB_VERSION: current_time = time.strftime("%Y-%m-%d_%H-%M-%S") old_db_filename = f"mech_events.{current_time}.old.json" - os.rename(MECH_EVENTS_JSON_PATH, MECH_EVENTS_JSON_PATH.parent / old_db_filename) + os.rename( + MECH_EVENTS_JSON_PATH, MECH_EVENTS_JSON_PATH.parent / old_db_filename + ) mech_events_data = {} mech_events_data["db_version"] = MECH_EVENTS_DB_VERSION except FileNotFoundError: @@ -227,7 +237,9 @@ def _query_mech_events_subgraph( subgraph_event_set_name = f"{event_cls.subgraph_event_name}s" all_results: dict[str, Any] = {"data": {subgraph_event_set_name: []}} - query = MECH_EVENTS_SUBGRAPH_QUERY_TEMPLATE.safe_substitute(subgraph_event_set_name=subgraph_event_set_name) + query = MECH_EVENTS_SUBGRAPH_QUERY_TEMPLATE.safe_substitute( + subgraph_event_set_name=subgraph_event_set_name + ) id_gt = "" while True: variables = { @@ -277,13 +289,9 @@ def _update_mech_events_db( miniters=1, desc=" Processing", ): - if subgraph_event[ - "id" - ] not in stored_events or not stored_events.get( + if subgraph_event["id"] not in stored_events or not stored_events.get( subgraph_event["id"], {} - ).get( - "ipfs_contents" - ): + ).get("ipfs_contents"): mech_event = event_cls(subgraph_event) # type: ignore stored_events[mech_event.event_id] = mech_event.__dict__ diff --git a/scripts/predict_trader/migrate_legacy_quickstart.py b/scripts/predict_trader/migrate_legacy_quickstart.py index 1ed1c069..f727b759 100644 --- a/scripts/predict_trader/migrate_legacy_quickstart.py +++ b/scripts/predict_trader/migrate_legacy_quickstart.py @@ -1,18 +1,17 @@ +"""Migrate a legacy `.trader_runner/` directory into the modern operate-managed layout.""" + +import json +import os +import sys from argparse import ArgumentParser from dataclasses import dataclass -import os -from typing import TypedDict -from dotenv import load_dotenv from getpass import getpass -from halo import Halo -import json from pathlib import Path -import sys +from typing import TypedDict from aea_ledger_ethereum import Account, EthereumCrypto, LocalAccount -from autonomy.chain.config import ChainType -from autonomy.chain.base import registry_contracts -from autonomy.constants import DEFAULT_KEYS_FILE +from dotenv import load_dotenv +from halo import Halo from operate.cli import OperateApp from operate.constants import ( OPERATE, @@ -21,21 +20,24 @@ from operate.ledger.profiles import ERC20_TOKENS from operate.operate_types import Chain, LedgerType, OnChainState, ServiceTemplate from operate.quickstart.run_service import ( + NO_STAKING_PROGRAM_ID, + QuickstartConfig, ask_password_if_needed, get_service, - QuickstartConfig, ) -from operate.services.protocol import StakingManager, StakingState -from operate.services.service import Service from operate.quickstart.utils import ( - ask_yes_or_no, CHAIN_TO_METADATA, + ask_yes_or_no, print_section, print_title, ) -from operate.quickstart.run_service import NO_STAKING_PROGRAM_ID +from operate.services.protocol import StakingManager, StakingState +from operate.services.service import Service from operate.utils.gnosis import get_asset_balance, get_assets_balances +from autonomy.chain.base import registry_contracts +from autonomy.constants import DEFAULT_KEYS_FILE + ROOT_PATH = Path(__file__).parent.parent.parent TRADER_RUNNER_PATH = ROOT_PATH / ".trader_runner" OPERATE_HOME = ROOT_PATH / OPERATE @@ -50,6 +52,8 @@ class StakingVariables(TypedDict): + """Shape of the staking-related env vars carried over from `.trader_runner/.env`.""" + USE_STAKING: bool STAKING_PROGRAM: str AGENT_ID: int @@ -64,6 +68,8 @@ class StakingVariables(TypedDict): @dataclass class TraderData: + """Bundle of state read off a legacy `.trader_runner/` directory.""" + password: str agent_eoa: Path master_eoa: Path @@ -75,9 +81,17 @@ class TraderData: def decrypt_private_keys(eoa: Path, password: str) -> dict[str, str]: + """Decrypt the EOA file (with or without password) and return its key payload.""" if not password: private_key = "0x" + eoa.read_text() + # `Account.from_key` is bound via eth-account's `@combomethod` descriptor; + # pylint sees the bound `self` slot and thinks `private_key` is missing. + # Block-style disable + enable scopes the suppression to the one call + # (a trailing same-line suppression lands on the wrapped closing paren + # after black reformats and stops applying to the call itself). + # pylint: disable=no-value-for-parameter account: LocalAccount = Account.from_key(private_key) + # pylint: enable=no-value-for-parameter address = account.address else: crypto = EthereumCrypto(private_key_path=eoa, password=password) @@ -92,6 +106,7 @@ def decrypt_private_keys(eoa: Path, password: str) -> dict[str, str]: def parse_trader_runner() -> TraderData: + """Read every persistent value out of `.trader_runner/` and return it as a TraderData.""" load_dotenv(TRADER_RUNNER_PATH / ".env") subgraph_api_key = os.getenv("SUBGRAPH_API_KEY") @@ -149,6 +164,7 @@ def parse_trader_runner() -> TraderData: def populate_operate( operate: OperateApp, trader_data: TraderData, service_template: ServiceTemplate ) -> Service: + """Set up OperateApp using `trader_data` and return the resulting Service object.""" print_section("Setting up Operate") operate.setup() if operate.user_account is None: @@ -248,9 +264,9 @@ def populate_operate( service.agent_addresses = [agent_eoa["address"]] service.chain_configs[Chain.GNOSIS.value].chain_data.token = trader_data.service_id - service.chain_configs[ - Chain.GNOSIS.value - ].chain_data.multisig = trader_data.service_safe + service.chain_configs[Chain.GNOSIS.value].chain_data.multisig = ( + trader_data.service_safe + ) service.store() spinner.succeed("Service created") @@ -280,6 +296,7 @@ def populate_operate( def migrate_to_master_safe( operate: OperateApp, trader_data: TraderData, service: Service ) -> None: + """Move funds and ownership from the legacy trader-runner setup to the master safe.""" print_section("Migrating service to .operate") chain_config = service.chain_configs[service.home_chain] ledger_config = chain_config.ledger_config @@ -561,6 +578,7 @@ def migrate_to_master_safe( def main(config_path: Path) -> None: + """Run the predict-trader quickstart migration end-to-end against the given config.""" print_title("Predict Trader Quickstart Migration") if not TRADER_RUNNER_PATH.exists(): print("No .trader_runner file found!") diff --git a/scripts/predict_trader/rank_traders.py b/scripts/predict_trader/rank_traders.py index bd063e40..3cf257c1 100644 --- a/scripts/predict_trader/rank_traders.py +++ b/scripts/predict_trader/rank_traders.py @@ -19,20 +19,24 @@ # ------------------------------------------------------------------------------ """This script queries the OMEN subgraph to obtain the trades of a given address.""" - import datetime -import requests import sys from argparse import ArgumentParser from collections import defaultdict from string import Template -from typing import Any +from typing import Any, Final +from operate.cli import OperateApp from operate.operate_types import Chain from operate.quickstart.run_service import load_local_config +from scripts.predict_trader.trades import ( + MarketAttribute, + MarketState, + _post_subgraph_query, + parse_user, + wei_to_xdai, +) from scripts.utils import get_subgraph_api_key -from scripts.predict_trader.trades import MarketAttribute, MarketState, parse_user, wei_to_xdai - QUERY_BATCH_SIZE = 1000 DUST_THRESHOLD = 10000000000000 @@ -41,15 +45,16 @@ DEFAULT_FROM_DATE = "2024-12-01T00:00:00" DEFAULT_TO_DATE = "2038-01-19T03:14:07" - -headers = { - "Accept": "application/json, multipart/mixed", - "Content-Type": "application/json", -} +# Must match the `name` field in `configs/config_predict_trader.json`. +# `load_local_config` looks the service up by exact name — if either +# side (this constant or the JSON value) changes, update the other. +# The test_rank_traders.py mock asserts the script's call site passes +# this constant; a drift between this value and the JSON's `name` is +# NOT covered by CI and will only surface at production runtime. +PREDICT_TRADER_SERVICE_NAME: Final[str] = "Trader Agent" -omen_xdai_trades_query = Template( - """ +omen_xdai_trades_query = Template(""" { fpmmTrades( where: { @@ -98,8 +103,7 @@ } } } - """ -) + """) ATTRIBUTE_CHOICES = {i.name: i for i in MarketAttribute} @@ -187,8 +191,7 @@ def _query_omen_xdai_subgraph( id_gt=id_gt, ) content_json = _to_content(query) - res = requests.post(url, headers=headers, json=content_json) - result_json = res.json() + result_json = _post_subgraph_query(url, content_json, label="omen") user_trades = result_json.get("data", {}).get("fpmmTrades", []) if not user_trades: @@ -276,7 +279,9 @@ def _print_user_summary( wei_to_xdai(statistics_table[MarketAttribute.INVESTMENT][state]).rjust(13), wei_to_xdai(statistics_table[MarketAttribute.FEES][state]).rjust(13), wei_to_xdai(statistics_table[MarketAttribute.EARNINGS][state]).rjust(13), - wei_to_xdai(statistics_table[MarketAttribute.NET_EARNINGS][state]).rjust(13), + wei_to_xdai(statistics_table[MarketAttribute.NET_EARNINGS][state]).rjust( + 13 + ), wei_to_xdai(statistics_table[MarketAttribute.REDEMPTIONS][state]).rjust(13), f"{statistics_table[MarketAttribute.ROI][state] * 100.0:7.2f}%".rjust(9), "\n", @@ -300,9 +305,11 @@ def _print_progress_bar( # pylint: disable=too-many-arguments percent = ("{0:.1f}").format(100 * (iteration / float(total))) filled_length = int(length * iteration // total) - bar = fill * filled_length + "-" * (length - filled_length) + progress_bar = fill * filled_length + "-" * (length - filled_length) progress_string = f"({iteration} of {total}) - {percent}%" - sys.stdout.write("\r%s |%s| %s %s" % (prefix, bar, progress_string, suffix)) + sys.stdout.write( + "\r%s |%s| %s %s" % (prefix, progress_bar, progress_string, suffix) + ) sys.stdout.flush() @@ -310,7 +317,11 @@ def _print_progress_bar( # pylint: disable=too-many-arguments print("Starting script") user_args = _parse_args() - config = load_local_config() + # `load_local_config` requires an OperateApp + service name since + # olas-operate-middleware 0.15.x. + config = load_local_config( + operate=OperateApp(), service_name=PREDICT_TRADER_SERVICE_NAME + ) rpc = config.rpc[Chain.GNOSIS.value] print("Querying Thegraph...") diff --git a/scripts/predict_trader/report.py b/scripts/predict_trader/report.py index c9b72a35..35dd0607 100644 --- a/scripts/predict_trader/report.py +++ b/scripts/predict_trader/report.py @@ -27,15 +27,28 @@ import traceback from argparse import ArgumentParser from collections import Counter - from enum import Enum from pathlib import Path from typing import Any import docker -from psutil import pid_exists import requests import scripts.predict_trader.trades as trades +from operate.cli import OperateApp +from operate.constants import ( + DEPLOYMENT_DIR, + MECH_ACTIVITY_CHECKER_JSON_URL, + MECH_CONTRACT_JSON_URL, + OPERATE_HOME, + SERVICE_REGISTRY_TOKEN_UTILITY_JSON_URL, + STAKING_TOKEN_INSTANCE_ABI_PATH, +) +from operate.ledger.profiles import get_staking_contract +from operate.operate_types import Chain +from operate.quickstart.run_service import ask_password_if_needed, load_local_config +from operate.quickstart.utils import print_title +from operate.services.service import Service +from psutil import pid_exists from scripts.predict_trader.trades import ( MarketAttribute, MarketState, @@ -46,25 +59,10 @@ wei_to_wxdai, wei_to_xdai, ) +from scripts.utils import get_service_from_config from web3 import HTTPProvider, Web3 from web3.exceptions import ABIFunctionNotFound, ContractLogicError -from operate.constants import ( - DEPLOYMENT_DIR, - OPERATE_HOME, - STAKING_TOKEN_INSTANCE_ABI_PATH, - SERVICE_REGISTRY_TOKEN_UTILITY_JSON_URL, - MECH_ACTIVITY_CHECKER_JSON_URL, - MECH_CONTRACT_JSON_URL, -) -from operate.cli import OperateApp -from operate.ledger.profiles import get_staking_contract -from operate.operate_types import Chain -from operate.quickstart.run_service import ask_password_if_needed, load_local_config -from operate.quickstart.utils import print_title -from operate.services.service import Service -from scripts.utils import get_service_from_config - SCRIPT_PATH = Path(__file__).resolve().parent SAFE_BALANCE_THRESHOLD = 500000000000000000 AGENT_XDAI_BALANCE_THRESHOLD = 50000000000000000 @@ -111,8 +109,8 @@ def _color_bool( def _color_percent(p: float, multiplier: float = 100, symbol: str = "%") -> str: if p >= 0: - return f"{p*multiplier:.2f} {symbol}" - return _color_string(f"{p*multiplier:.2f} {symbol}", ColorCode.RED) + return f"{p * multiplier:.2f} {symbol}" + return _color_string(f"{p * multiplier:.2f} {symbol}", ColorCode.RED) def _trades_since_message(trades_json: dict[str, Any], utc_ts: float = 0) -> str: @@ -127,16 +125,21 @@ def _trades_since_message(trades_json: dict[str, Any], utc_ts: float = 0) -> str return f"{trades_count} trades on {markets_count} markets" -def _calculate_retrades_since(trades_json: dict[str, Any], utc_ts: float = 0) -> tuple[Counter[Any], int, int, int]: - filtered_trades = Counter(( - trade.get("fpmm", {}).get("id", None) - for trade in trades_json.get("data", {}).get("fpmmTrades", []) - if float(trade.get("creationTimestamp", 0)) >= utc_ts - )) +def _calculate_retrades_since( + trades_json: dict[str, Any], utc_ts: float = 0 +) -> tuple[Counter[Any], int, int, int]: + filtered_trades = Counter( + ( + trade.get("fpmm", {}).get("id", None) + for trade in trades_json.get("data", {}).get("fpmmTrades", []) + if float(trade.get("creationTimestamp", 0)) >= utc_ts + ) + ) if None in filtered_trades: raise ValueError( - f"Unexpected format in trades_json: {filtered_trades[None]} trades have no associated market ID.") + f"Unexpected format in trades_json: {filtered_trades[None]} trades have no associated market ID." + ) unique_markets = set(filtered_trades) n_unique_markets = len(unique_markets) @@ -145,9 +148,13 @@ def _calculate_retrades_since(trades_json: dict[str, Any], utc_ts: float = 0) -> return filtered_trades, n_unique_markets, n_trades, n_retrades -def _retrades_since_message(n_unique_markets: int, n_trades: int, n_retrades: int) -> str: + +def _retrades_since_message( + n_unique_markets: int, n_trades: int, n_retrades: int +) -> str: return f"{n_retrades} re-trades on total {n_trades} trades in {n_unique_markets} markets" + def _average_trades_since_message(n_trades: int, n_markets: int) -> str: if not n_markets: average_trades = 0 @@ -156,6 +163,7 @@ def _average_trades_since_message(n_trades: int, n_markets: int) -> str: return f"{average_trades} trades per market" + def _max_trades_per_market_since_message(filtered_trades: Counter[Any]) -> str: if not filtered_trades: max_trades = 0 @@ -208,9 +216,13 @@ def _get_agent_status(service: Service) -> str: trader_abci_container = None trader_tm_container = None for container in client.containers.list(): - if container.name.startswith("traderpearl") and container.name.endswith("abci_0"): + if container.name.startswith("traderpearl") and container.name.endswith( + "abci_0" + ): trader_abci_container = container - elif container.name.startswith("traderpearl") and container.name.endswith("tm_0"): + elif container.name.startswith("traderpearl") and container.name.endswith( + "tm_0" + ): trader_tm_container = container if trader_abci_container and trader_tm_container: break @@ -243,7 +255,9 @@ def _parse_args() -> Any: with open(operate_wallet_path) as file: operator_wallet_data = json.load(file) - template_path = Path(SCRIPT_PATH.parents[1], "configs", "config_predict_trader.json") + template_path = Path( + SCRIPT_PATH.parents[1], "configs", "config_predict_trader.json" + ) service = get_service_from_config(template_path, operate) config = load_local_config(operate=operate, service_name=service.name) chain_config = service.chain_configs["gnosis"] @@ -275,14 +289,16 @@ def _parse_args() -> Any: try: get_balance(safe_address, rpc, block_identifier=current_block_number) except TypeError: - print("RPC does not support block number queries, falling back to latest block.") + print( + "RPC does not support block number queries, falling back to latest block." + ) current_block_number = "latest" print("") print_title(f"\nService report on block number {current_block_number}\n") # Performance - _print_section_header(f"Performance") + _print_section_header("Performance") _print_subsection_header("Staking") staking_token_address = get_staking_contract( @@ -293,7 +309,9 @@ def _parse_args() -> Any: is_staked = False staking_state = StakingState.UNSTAKED else: - staking_token_data = requests.get(STAKING_TOKEN_INSTANCE_ABI_PATH).json() + staking_token_data = requests.get( + STAKING_TOKEN_INSTANCE_ABI_PATH, timeout=30 + ).json() staking_token_abi = staking_token_data.get("abi", []) staking_token_contract = w3.eth.contract( @@ -301,9 +319,9 @@ def _parse_args() -> Any: ) staking_state = StakingState( - staking_token_contract.functions.getStakingState( - service_id - ).call(block_identifier=current_block_number) + staking_token_contract.functions.getStakingState(service_id).call( + block_identifier=current_block_number + ) ) is_staked = ( @@ -317,21 +335,33 @@ def _parse_args() -> Any: if staking_state == StakingState.STAKED: _print_status("Staking state", staking_state.name) elif staking_state == StakingState.EVICTED: - _print_status("Staking state", _color_string(staking_state.name, ColorCode.RED)) + _print_status( + "Staking state", _color_string(staking_state.name, ColorCode.RED) + ) if is_staked: - activity_checker_address = staking_token_contract.functions.activityChecker().call(block_identifier=current_block_number) - activity_checker_data = requests.get(MECH_ACTIVITY_CHECKER_JSON_URL).json() + activity_checker_address = ( + staking_token_contract.functions.activityChecker().call( + block_identifier=current_block_number + ) + ) + activity_checker_data = requests.get( + MECH_ACTIVITY_CHECKER_JSON_URL, timeout=30 + ).json() activity_checker_abi = activity_checker_data.get("abi", []) activity_checker_contract = w3.eth.contract( address=activity_checker_address, abi=activity_checker_abi # type: ignore ) - service_registry_token_utility_data = requests.get(SERVICE_REGISTRY_TOKEN_UTILITY_JSON_URL).json() + service_registry_token_utility_data = requests.get( + SERVICE_REGISTRY_TOKEN_UTILITY_JSON_URL, timeout=30 + ).json() service_registry_token_utility_contract_address = ( - staking_token_contract.functions.serviceRegistryTokenUtility().call(block_identifier=current_block_number) + staking_token_contract.functions.serviceRegistryTokenUtility().call( + block_identifier=current_block_number + ) ) service_registry_token_utility_abi = ( service_registry_token_utility_data.get("abi", []) @@ -342,57 +372,73 @@ def _parse_args() -> Any: ) try: - activity_checker_data = requests.get(MECH_CONTRACT_JSON_URL).json() + activity_checker_data = requests.get( + MECH_CONTRACT_JSON_URL, timeout=30 + ).json() activity_checker_abi = activity_checker_data.get("abi", []) mm_activity_checker_contract = w3.eth.contract( address=activity_checker_address, abi=activity_checker_abi # type: ignore ) - mech_contract_address = mm_activity_checker_contract.functions.mechMarketplace().call(block_identifier=current_block_number) + mech_contract_address = ( + mm_activity_checker_contract.functions.mechMarketplace().call( + block_identifier=current_block_number + ) + ) except (ContractLogicError, ValueError): - mech_contract_address = activity_checker_contract.functions.agentMech().call(block_identifier=current_block_number) + mech_contract_address = ( + activity_checker_contract.functions.agentMech().call( + block_identifier=current_block_number + ) + ) mech_contract_abi = [ { "inputs": [ { - "internalType": "address", - "name": "account", - "type": "address" + "internalType": "address", + "name": "account", + "type": "address", } ], "name": function_name, "outputs": [ - { - "internalType": "uint256", - "name": "", - "type": "uint256" - } + {"internalType": "uint256", "name": "", "type": "uint256"} ], "stateMutability": "view", - "type": "function" + "type": "function", } for function_name in ("mapRequestsCounts", "mapRequestCounts") ] mech_contract = w3.eth.contract( - address=mech_contract_address, abi=mech_contract_abi # type: ignore + address=mech_contract_address, abi=mech_contract_abi # type: ignore ) try: - mech_request_count = mech_contract.functions.mapRequestsCounts(safe_address).call(block_identifier=current_block_number) + mech_request_count = mech_contract.functions.mapRequestsCounts( + safe_address + ).call(block_identifier=current_block_number) except (ContractLogicError, ABIFunctionNotFound, ValueError): # Use mapRequestCounts for newer mechs - mech_request_count = mech_contract.functions.mapRequestCounts(safe_address).call(block_identifier=current_block_number) + mech_request_count = mech_contract.functions.mapRequestCounts( + safe_address + ).call(block_identifier=current_block_number) security_deposit = ( service_registry_token_utility_contract.functions.getOperatorBalance( operator_address, service_id ).call(block_identifier=current_block_number) ) - agent_id = int(staking_token_contract.functions.getAgentIds().call(block_identifier=current_block_number)[0]) + agent_id = int( + staking_token_contract.functions.getAgentIds().call( + block_identifier=current_block_number + )[0] + ) agent_bond = service_registry_token_utility_contract.functions.getAgentBond( service_id, agent_id ).call(block_identifier=current_block_number) min_staking_deposit = ( - staking_token_contract.functions.minStakingDeposit().call(block_identifier=current_block_number) + staking_token_contract.functions.minStakingDeposit().call( + block_identifier=current_block_number + ) ) # In the setting 1 agent instance as of now: minOwnerBond = minStakingDeposit @@ -412,32 +458,35 @@ def _parse_args() -> Any: rewards = service_info[3] _print_status("Accrued rewards", f"{wei_to_olas(rewards)}") - liveness_ratio = ( - activity_checker_contract.functions.livenessRatio().call(block_identifier=current_block_number) + liveness_ratio = activity_checker_contract.functions.livenessRatio().call( + block_identifier=current_block_number ) current_timestamp = w3.eth.get_block(current_block_number).timestamp - last_ts_checkpoint = staking_token_contract.functions.tsCheckpoint().call(block_identifier=current_block_number) - liveness_period = ( - staking_token_contract.functions.livenessPeriod().call(block_identifier=current_block_number) + last_ts_checkpoint = staking_token_contract.functions.tsCheckpoint().call( + block_identifier=current_block_number + ) + liveness_period = staking_token_contract.functions.livenessPeriod().call( + block_identifier=current_block_number ) mech_requests_24h_threshold = math.ceil( max(liveness_period, (current_timestamp - last_ts_checkpoint)) * liveness_ratio - / 10 ** 18 + / 10**18 ) - next_checkpoint_ts = ( - staking_token_contract.functions.getNextRewardCheckpointTimestamp().call(block_identifier=current_block_number) + next_checkpoint_ts = staking_token_contract.functions.getNextRewardCheckpointTimestamp().call( + block_identifier=current_block_number ) last_checkpoint_ts = next_checkpoint_ts - liveness_period mech_request_count_on_last_checkpoint = ( - staking_token_contract.functions.getServiceInfo(service_id).call(block_identifier=current_block_number) + staking_token_contract.functions.getServiceInfo(service_id).call( + block_identifier=current_block_number + ) )[2][1] - mech_requests_since_last_cp = mech_request_count - mech_request_count_on_last_checkpoint - # mech_requests_current_epoch = _get_mech_requests_count( - # mech_requests, last_checkpoint_ts - # ) + mech_requests_since_last_cp = ( + mech_request_count - mech_request_count_on_last_checkpoint + ) mech_requests_current_epoch = mech_requests_since_last_cp _print_status( "Num. Mech txs current epoch", @@ -460,13 +509,25 @@ def _parse_args() -> Any: _trades_since_message(trades_json, since_ts), ) - #Multi trade strategy + # Multi trade strategy retrades_since_ts = time.time() - SECONDS_PER_DAY * MULTI_TRADE_LOOKBACK_DAYS - filtered_trades, n_unique_markets, n_trades, n_retrades = _calculate_retrades_since(trades_json, retrades_since_ts) - _print_subsection_header(f"Multi-trade markets in previous {MULTI_TRADE_LOOKBACK_DAYS} days") - _print_status(f"Multi-trade markets", _retrades_since_message(n_unique_markets, n_trades, n_retrades)) - _print_status(f"Average trades per market", _average_trades_since_message(n_trades, n_unique_markets)) - _print_status(f"Max trades per market", _max_trades_per_market_since_message(filtered_trades)) + filtered_trades, n_unique_markets, n_trades, n_retrades = _calculate_retrades_since( + trades_json, retrades_since_ts + ) + _print_subsection_header( + f"Multi-trade markets in previous {MULTI_TRADE_LOOKBACK_DAYS} days" + ) + _print_status( + "Multi-trade markets", + _retrades_since_message(n_unique_markets, n_trades, n_retrades), + ) + _print_status( + "Average trades per market", + _average_trades_since_message(n_trades, n_unique_markets), + ) + _print_status( + "Max trades per market", _max_trades_per_market_since_message(filtered_trades) + ) # Service _print_section_header("Service") @@ -485,7 +546,12 @@ def _parse_args() -> Any: # Safe safe_xdai = get_balance(safe_address, rpc, block_identifier=current_block_number) - safe_wxdai = get_token_balance(safe_address, trades.WXDAI_CONTRACT_ADDRESS, rpc, block_identifier=current_block_number) + safe_wxdai = get_token_balance( + safe_address, + trades.WXDAI_CONTRACT_ADDRESS, + rpc, + block_identifier=current_block_number, + ) _print_subsection_header( f"Safe {_warning_message(safe_xdai + safe_wxdai, SAFE_BALANCE_THRESHOLD)}" ) @@ -494,7 +560,9 @@ def _parse_args() -> Any: _print_status("WxDAI Balance", wei_to_wxdai(safe_wxdai)) # Master Safe - Agent Owner/Operator - operator_xdai = get_balance(operator_address, rpc, block_identifier=current_block_number) + operator_xdai = get_balance( + operator_address, rpc, block_identifier=current_block_number + ) _print_subsection_header("Master Safe - Agent Owner/Operator") _print_status("Address", operator_address) _print_status( @@ -503,7 +571,9 @@ def _parse_args() -> Any: ) # Master EOA - Master Safe Owner - master_eoa_xdai = get_balance(master_eoa, rpc, block_identifier=current_block_number) + master_eoa_xdai = get_balance( + master_eoa, rpc, block_identifier=current_block_number + ) _print_subsection_header("Master EOA - Master Safe Owner") _print_status("Address", master_eoa) _print_status( diff --git a/scripts/predict_trader/trades.py b/scripts/predict_trader/trades.py index 6cb27106..daa5f396 100644 --- a/scripts/predict_trader/trades.py +++ b/scripts/predict_trader/trades.py @@ -22,7 +22,6 @@ import datetime import re -import requests import sys from argparse import Action, ArgumentError, ArgumentParser, Namespace from collections import defaultdict @@ -31,13 +30,13 @@ from string import Template from typing import Any, Dict, Optional +import requests from operate.cli import OperateApp from operate.operate_types import Chain from operate.quickstart.run_service import ask_password_if_needed, load_local_config from scripts.predict_trader.mech_events import get_mech_requests from scripts.utils import get_service_from_config, get_subgraph_api_key - IRRELEVANT_TOOLS = [ "openai-text-davinci-002", "openai-text-davinci-003", @@ -55,7 +54,7 @@ INVALID_ANSWER = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF FPMM_CREATORS = ( "0x89c5cc945dd550BcFfb72Fe42BfF002429F46Fec", - "0xFfc8029154ECD55ABED15BD428bA596E7D23f557" + "0xFfc8029154ECD55ABED15BD428bA596E7D23f557", ) DEFAULT_FROM_DATE = "1970-01-01T00:00:00" DEFAULT_TO_DATE = "2038-01-19T03:14:07" @@ -71,8 +70,7 @@ } -omen_xdai_trades_query = Template( - """ +omen_xdai_trades_query = Template(""" { fpmmTrades( where: { @@ -122,12 +120,10 @@ } } } - """ -) + """) -conditional_tokens_gc_user_query = Template( - """ +conditional_tokens_gc_user_query = Template(""" { user(id: "${id}") { userPositions( @@ -148,8 +144,7 @@ } } } - """ -) + """) class MarketState(Enum): @@ -214,12 +209,15 @@ def get_balance(address: str, rpc_url: str, block_identifier: str = "latest") -> "params": [address, block_identifier], "id": 1, } - response = requests.post(rpc_url, headers=headers, json=data) + response = requests.post(rpc_url, headers=headers, json=data, timeout=30) return int(response.json().get("result"), 16) def get_token_balance( - gnosis_address: str, token_contract_address: str, rpc_url: str, block_identifier: str = "latest" + gnosis_address: str, + token_contract_address: str, + rpc_url: str, + block_identifier: str = "latest", ) -> int: """Get the token balance of an address in wei.""" function_selector = "70a08231" # function selector for balanceOf(address) @@ -234,7 +232,7 @@ def get_token_balance( "params": [{"to": token_contract_address, "data": data}, block_identifier], "id": 1, } - response = requests.post(rpc_url, json=payload) + response = requests.post(rpc_url, json=payload, timeout=30) result = response.json().get("result", "0x0") balance_wei = int(result, 16) # convert hex to int return balance_wei @@ -293,12 +291,18 @@ def _parse_args() -> Any: args = parser.parse_args() if args.creator is None: - template_path = Path(SCRIPT_PATH.parents[1], "configs", "config_predict_trader.json") + template_path = Path( + SCRIPT_PATH.parents[1], "configs", "config_predict_trader.json" + ) if not template_path.exists(): print("No Safe address found!") sys.exit(1) - args.creator = get_service_from_config(template_path).chain_configs["gnosis"].chain_data.multisig + args.creator = ( + get_service_from_config(template_path) + .chain_configs["gnosis"] + .chain_data.multisig + ) args.from_date = args.from_date.replace(tzinfo=datetime.timezone.utc) args.to_date = args.to_date.replace(tzinfo=datetime.timezone.utc) @@ -322,6 +326,71 @@ def _to_content(q: str) -> Dict[str, Any]: return finalized_query +# Matches the billable key segment after `/api/` in The Graph gateway +# URLs, regardless of whether the surrounding string is the full URL +# (`https:///api//subgraphs/...`, as in `HTTPError.__str__`) +# or only the path (`/api//subgraphs/...`, as embedded by +# `urllib3.MaxRetryError` inside `ConnectionError`/`Timeout`/`SSLError`). +# The key segment is everything up to the next `/` or whitespace. +_SUBGRAPH_KEY_RE = re.compile(r"(/api/)[^/\s]+") + + +def _redact_subgraph_key(text: str) -> str: + """Replace any `/api/` segment in `text` with `/api/`. + + Handles both forms in which the key surfaces: + - Full URL in `requests.HTTPError.__str__`: + `... for url: https:///api//subgraphs/...` + - Path-only in `urllib3.MaxRetryError` (the cause inside + `requests.ConnectionError` / `Timeout` / `SSLError`): + `HTTPSConnectionPool(host=..., port=...): ... with url: + /api//subgraphs/... (Caused by ...)` + + A single regex covers both, so the key cannot reach an error + message via either branch. + """ + return _SUBGRAPH_KEY_RE.sub(r"\1", text) + + +def _post_subgraph_query( + url: str, payload: Dict[str, Any], *, label: str +) -> Dict[str, Any]: + """POST a subgraph query and return its parsed JSON body. + + Wraps `requests.post`, `raise_for_status`, and `.json()` in one + try-block so a network failure, a 4xx/5xx response (e.g. the gateway + returning an HTML error page), or a malformed body all surface as a + single `RuntimeError("