diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml new file mode 100644 index 00000000..d4f427a7 --- /dev/null +++ b/.github/workflows/openapi.yml @@ -0,0 +1,28 @@ +name: OpenAPI Sync Check + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + check-openapi: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pyyaml pydantic numpy + + - name: Verify OpenAPI sync + run: | + python scripts/generate_openapi.py --check diff --git a/docs/openapi.yaml b/docs/openapi.yaml new file mode 100644 index 00000000..90d70039 --- /dev/null +++ b/docs/openapi.yaml @@ -0,0 +1,544 @@ +openapi: 3.0.3 +info: + title: Engram Miner API + version: 1.0.0 + description: Auto-generated OpenAPI spec for Engram Miner. +servers: +- url: http://localhost:8091 + description: Local Miner Node +paths: + /IngestSynapse: + post: + summary: POST /IngestSynapse + responses: + '200': + description: Success + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/IngestSynapse' + /QuerySynapse: + post: + summary: POST /QuerySynapse + responses: + '200': + description: Success + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/QuerySynapse' + /ChallengeSynapse: + post: + summary: POST /ChallengeSynapse + responses: + '200': + description: Success + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ChallengeSynapse' + /namespace: + post: + summary: POST /namespace + responses: + '200': + description: Success + /AttestNamespace: + post: + summary: POST /AttestNamespace + responses: + '200': + description: Success + /attestation/{namespace}: + get: + summary: GET /attestation/{namespace} + responses: + '200': + description: Success + /chat-history/{user_id}: + get: + summary: GET /chat-history/{user_id} + responses: + '200': + description: Success + /chat-history: + post: + summary: POST /chat-history + responses: + '200': + description: Success + /conversations/{user_id}: + get: + summary: GET /conversations/{user_id} + responses: + '200': + description: Success + /conversations: + post: + summary: POST /conversations + responses: + '200': + description: Success + /conversations/{conv_id}: + patch: + summary: PATCH /conversations/{conv_id} + responses: + '200': + description: Success + delete: + summary: DELETE /conversations/{conv_id} + responses: + '200': + description: Success + /retrieve/{cid}: + get: + summary: GET /retrieve/{cid} + responses: + '200': + description: Success + delete: + summary: DELETE /retrieve/{cid} + responses: + '200': + description: Success + /RepairSynapse: + post: + summary: POST /RepairSynapse + responses: + '200': + description: Success + /KeyShareSynapse: + post: + summary: POST /KeyShareSynapse + responses: + '200': + description: Success + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/KeyShareSynapse' + /KeyShareRetrieve: + post: + summary: POST /KeyShareRetrieve + responses: + '200': + description: Success + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/KeyShareRetrieve' + /list: + post: + summary: POST /list + responses: + '200': + description: Success + /health: + get: + summary: GET /health + responses: + '200': + description: Success + /stats: + get: + summary: GET /stats + responses: + '200': + description: Success + /metagraph: + get: + summary: GET /metagraph + responses: + '200': + description: Success + /metrics: + get: + summary: GET /metrics + responses: + '200': + description: Success + /wallet-stats: + get: + summary: GET /wallet-stats + responses: + '200': + description: Success + /wallet-stats/{hotkey}: + get: + summary: GET /wallet-stats/{hotkey} + responses: + '200': + description: Success + /commitment: + get: + summary: GET /commitment + responses: + '200': + description: Success + /prove-memory: + post: + summary: POST /prove-memory + responses: + '200': + description: Success +components: + schemas: + IngestSynapse: + description: "Sent by client/validator to a miner to store an embedding.\n\n\ + Request: text OR raw_embedding (one must be provided)\nResponse: cid (set\ + \ by miner on success)\n\nPrivate collections \u2014 two auth modes (prefer\ + \ sig-based):\n Sig-based (secure): namespace + namespace_hotkey + namespace_sig\ + \ + namespace_timestamp_ms\n Key-based (legacy): namespace + namespace_key" + properties: + text: + anyOf: + - type: string + - type: 'null' + default: null + description: Raw text to embed and store. Mutually exclusive with raw_embedding. + title: Text + raw_embedding: + anyOf: + - items: + type: number + type: array + - type: 'null' + default: null + description: Pre-computed embedding vector. Skips the embedding step on + the miner. + title: Raw Embedding + metadata: + additionalProperties: true + description: Arbitrary key-value metadata stored alongside the vector. + title: Metadata + type: object + model_version: + default: v1 + description: Subnet model epoch version for CID generation. + title: Model Version + type: string + namespace: + anyOf: + - type: string + - type: 'null' + default: null + description: Private collection name. + title: Namespace + namespace_hotkey: + anyOf: + - type: string + - type: 'null' + default: null + description: Bittensor SS58 hotkey that owns this namespace. + title: Namespace Hotkey + namespace_sig: + anyOf: + - type: string + - type: 'null' + default: null + description: "sr25519 hex signature over 'engram-ns:{namespace}:{namespace_timestamp_ms}'.\ + \ Replaces namespace_key \u2014 key never travels over the wire." + title: Namespace Sig + namespace_timestamp_ms: + anyOf: + - type: integer + - type: 'null' + default: null + description: "Unix ms timestamp for namespace_sig replay prevention (\xB1\ + 60s window)." + title: Namespace Timestamp Ms + namespace_key: + anyOf: + - type: string + - type: 'null' + default: null + description: '[Deprecated] Secret key for the namespace. Use namespace_sig + instead.' + title: Namespace Key + cid: + anyOf: + - type: string + - type: 'null' + default: null + description: Content identifier returned by the miner. + title: Cid + error: + anyOf: + - type: string + - type: 'null' + default: null + description: Error message if ingest failed. + title: Error + title: IngestSynapse + type: object + QuerySynapse: + description: 'Sent by validator/client to miners for approximate nearest-neighbor + search. + + + Request: query_text OR query_vector, top_k + + Response: results (list of CID + score + metadata)' + properties: + query_text: + anyOf: + - type: string + - type: 'null' + default: null + title: Query Text + query_vector: + anyOf: + - items: + type: number + type: array + - type: 'null' + default: null + title: Query Vector + top_k: + default: 10 + maximum: 100 + minimum: 1 + title: Top K + type: integer + namespace: + anyOf: + - type: string + - type: 'null' + default: null + title: Namespace + namespace_hotkey: + anyOf: + - type: string + - type: 'null' + default: null + title: Namespace Hotkey + namespace_sig: + anyOf: + - type: string + - type: 'null' + default: null + title: Namespace Sig + namespace_timestamp_ms: + anyOf: + - type: integer + - type: 'null' + default: null + title: Namespace Timestamp Ms + namespace_key: + anyOf: + - type: string + - type: 'null' + default: null + title: Namespace Key + results: + description: List of {cid, score, metadata} dicts ordered by descending + similarity. + items: + additionalProperties: true + type: object + title: Results + type: array + latency_ms: + anyOf: + - type: number + - type: 'null' + default: null + description: Miner-reported query latency in milliseconds. + title: Latency Ms + error: + anyOf: + - type: string + - type: 'null' + default: null + title: Error + title: QuerySynapse + type: object + ChallengeSynapse: + description: 'Validator issues a storage proof challenge to a miner. + + + Request: cid + nonce_hex + expires_at + + Response: embedding_hash + proof (HMAC) + + + The Rust engram_core module handles challenge generation and verification.' + properties: + cid: + description: CID the miner is being challenged to prove storage of. + title: Cid + type: string + nonce_hex: + description: 32-byte random nonce as hex string. + title: Nonce Hex + type: string + expires_at: + description: Unix timestamp after which the proof is invalid. + title: Expires At + type: integer + embedding_hash: + anyOf: + - type: string + - type: 'null' + default: null + description: SHA-256 of the stored embedding bytes (hex). + title: Embedding Hash + proof: + anyOf: + - type: string + - type: 'null' + default: null + description: HMAC-SHA256(nonce || embedding_hash) proving possession. + title: Proof + error: + anyOf: + - type: string + - type: 'null' + default: null + title: Error + required: + - cid + - nonce_hex + - expires_at + title: ChallengeSynapse + type: object + KeyShareSynapse: + description: "Client deposits one Shamir key share with a miner for threshold\ + \ decryption.\n\nThe miner stores the share associated with the namespace.\ + \ It cannot reconstruct\nthe full key alone \u2014 that requires K miners\ + \ to cooperate at retrieval time.\n\nAuth: namespace_hotkey + namespace_sig\ + \ + namespace_timestamp_ms (sig-based only)." + properties: + namespace: + description: Namespace this share belongs to. + title: Namespace + type: string + share_index: + description: 1-based share index (1..total). + title: Share Index + type: integer + share_hex: + description: Hex-encoded share bytes. + title: Share Hex + type: string + threshold: + description: Minimum shares needed to reconstruct (k). + title: Threshold + type: integer + total: + description: Total shares created (n). + title: Total + type: integer + namespace_hotkey: + anyOf: + - type: string + - type: 'null' + default: null + title: Namespace Hotkey + namespace_sig: + anyOf: + - type: string + - type: 'null' + default: null + title: Namespace Sig + namespace_timestamp_ms: + anyOf: + - type: integer + - type: 'null' + default: null + title: Namespace Timestamp Ms + stored: + default: false + title: Stored + type: boolean + error: + anyOf: + - type: string + - type: 'null' + default: null + title: Error + required: + - namespace + - share_index + - share_hex + - threshold + - total + title: KeyShareSynapse + type: object + KeyShareRetrieve: + description: 'Client retrieves its key share from a miner. + + + The miner returns the share only after verifying namespace ownership. + + The client collects K shares from K different miners and reconstructs locally. + + + Auth: namespace_hotkey + namespace_sig + namespace_timestamp_ms.' + properties: + namespace: + description: Namespace to retrieve the share for. + title: Namespace + type: string + namespace_hotkey: + anyOf: + - type: string + - type: 'null' + default: null + title: Namespace Hotkey + namespace_sig: + anyOf: + - type: string + - type: 'null' + default: null + title: Namespace Sig + namespace_timestamp_ms: + anyOf: + - type: integer + - type: 'null' + default: null + title: Namespace Timestamp Ms + share_index: + anyOf: + - type: integer + - type: 'null' + default: null + title: Share Index + share_hex: + anyOf: + - type: string + - type: 'null' + default: null + title: Share Hex + threshold: + anyOf: + - type: integer + - type: 'null' + default: null + title: Threshold + total: + anyOf: + - type: integer + - type: 'null' + default: null + title: Total + error: + anyOf: + - type: string + - type: 'null' + default: null + title: Error + required: + - namespace + title: KeyShareRetrieve + type: object diff --git a/scripts/generate_openapi.py b/scripts/generate_openapi.py new file mode 100644 index 00000000..d85d84d3 --- /dev/null +++ b/scripts/generate_openapi.py @@ -0,0 +1,106 @@ +import ast +import json +import yaml +import sys +from pathlib import Path +from pydantic import BaseModel + +# Add parent dir to path so we can import engram +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from engram.protocol import ( + IngestSynapse, QuerySynapse, ChallengeSynapse, KeyShareSynapse, KeyShareRetrieve +) + +def generate(): + miner_path = Path("neurons/miner.py") + with open(miner_path, "r", encoding="utf-8") as f: + miner_src = f.read() + + tree = ast.parse(miner_src) + routes = [] + + # Simple AST extraction of app.router.add_* + for node in ast.walk(tree): + if isinstance(node, ast.Call): + if isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Attribute): + if getattr(node.func.value.value, "id", "") == "app" and node.func.value.attr == "router": + method = node.func.attr.replace("add_", "").lower() + if method in ("get", "post", "put", "delete", "patch"): + path = node.args[0].value + routes.append((method, path)) + + openapi = { + "openapi": "3.0.3", + "info": { + "title": "Engram Miner API", + "version": "1.0.0", + "description": "Auto-generated OpenAPI spec for Engram Miner." + }, + "servers": [{"url": "http://localhost:8091", "description": "Local Miner Node"}], + "paths": {}, + "components": { + "schemas": { + "IngestSynapse": IngestSynapse.model_json_schema(), + "QuerySynapse": QuerySynapse.model_json_schema(), + "ChallengeSynapse": ChallengeSynapse.model_json_schema(), + "KeyShareSynapse": KeyShareSynapse.model_json_schema(), + "KeyShareRetrieve": KeyShareRetrieve.model_json_schema(), + } + } + } + + synapse_map = { + "/IngestSynapse": "IngestSynapse", + "/QuerySynapse": "QuerySynapse", + "/ChallengeSynapse": "ChallengeSynapse", + "/KeyShareSynapse": "KeyShareSynapse", + "/KeyShareRetrieve": "KeyShareRetrieve", + } + + for method, path in routes: + if path not in openapi["paths"]: + openapi["paths"][path] = {} + + op = { + "summary": f"{method.upper()} {path}", + "responses": { + "200": {"description": "Success"} + } + } + + if path in synapse_map and method == "post": + schema_name = synapse_map[path] + op["requestBody"] = { + "required": True, + "content": { + "application/json": { + "schema": {"$ref": f"#/components/schemas/{schema_name}"} + } + } + } + + openapi["paths"][path][method] = op + + docs_path = Path("docs/openapi.yaml") + docs_path.parent.mkdir(exist_ok=True) + with open(docs_path, "w", encoding="utf-8") as f: + yaml.dump(openapi, f, sort_keys=False) + +if __name__ == "__main__": + import sys + if "--check" in sys.argv: + with open("docs/openapi.yaml", "r") as f: + old = f.read() + generate() + with open("docs/openapi.yaml", "r") as f: + new = f.read() + if old != new: + print("ERROR: docs/openapi.yaml is out of sync. Please run python scripts/generate_openapi.py") + sys.exit(1) + else: + print("docs/openapi.yaml is up to date.") + sys.exit(0) + else: + generate() + print("Generated docs/openapi.yaml successfully.")