{safe_name}
-ID: {safe_id}
+{safe_name}
+ID: {safe_id}
diff --git a/.env.example b/.env.example index 828a6bf..0ab073a 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,18 @@ # Copy to .env and adjust values # ── Neo4j ── -NEO4J_PASSWORD=beyond-ai-2026 +NEO4J_PASSWORD= + +# ── yente / Elasticsearch ── +# Use a strong URL-safe password because it is embedded into YENTE_INDEX_URL. +YENTE_ES_PASSWORD= + +# ── Beyond AI access protection ── +BEYOND_AI_BASIC_AUTH_USER= +BEYOND_AI_BASIC_AUTH_PASSWORD= +BEYOND_AI_AUTH_REQUIRED=true +BEYOND_AI_RATE_LIMIT_REQUESTS=30 +BEYOND_AI_RATE_LIMIT_WINDOW_SECONDS=60 # ── Ollama (Primary LLM) ── OLLAMA_URL=http://localhost:11434 @@ -10,6 +21,7 @@ OLLAMA_MODEL=kimi-k2.5 # ── Claude API (Fallback LLM) ── ANTHROPIC_API_KEY= +BEYOND_AI_ALLOW_EXTERNAL_LLM_FALLBACK=false # ── OpenSanctions ── # Nur für kommerzielle Nutzung: diff --git a/.env.prod.example b/.env.prod.example index 5050fe4..9adbc00 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -6,9 +6,22 @@ NEO4J_CONTAINER_NAME=neo4j NEO4J_PASSWORD=DEIN_NEO4J_PASSWORT_HIER +# ─── yente / Elasticsearch ───────────────────────────────────── +# URL-safe Passwort verwenden, da es in YENTE_INDEX_URL eingebettet wird. +YENTE_ES_PASSWORD=SETZE_EIN_LANGES_URL_SICHERES_PASSWORT + +# ─── Beyond AI Zugriffsschutz ────────────────────────────────── +BEYOND_AI_BASIC_AUTH_USER=screening +BEYOND_AI_BASIC_AUTH_PASSWORD=SETZE_EIN_LANGES_PASSWORT +BEYOND_AI_AUTH_REQUIRED=true +BEYOND_AI_RATE_LIMIT_REQUESTS=30 +BEYOND_AI_RATE_LIMIT_WINDOW_SECONDS=60 +BEYOND_AI_TRUST_PROXY_HEADERS=true + # ─── Ollama (bestehende ollama-eksw-Instanz) ────────────────── # Container-Namen prüfen: docker ps --format '{{.Names}}' | grep ollama OLLAMA_CONTAINER_NAME=ollama-eksw-ollama-1 # ─── Optional: Claude API als LLM-Fallback ──────────────────── ANTHROPIC_API_KEY= +BEYOND_AI_ALLOW_EXTERNAL_LLM_FALLBACK=false diff --git a/Dockerfile b/Dockerfile index acad3ac..ad03ff6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,11 @@ RUN pip install --no-cache-dir -r requirements.txt COPY shared/ ./shared/ COPY sanctions/ ./sanctions/ +RUN addgroup --system beyondai && adduser --system --ingroup beyondai beyondai \ + && chown -R beyondai:beyondai /app + +USER beyondai + EXPOSE 8000 CMD ["uvicorn", "sanctions.src.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index 07e727b..e514f43 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,11 @@ git clone https://github.com/endvater/beyond-ai.git cd beyond-ai cp .env.example .env +# Pflichtfelder in .env setzen: +# - NEO4J_PASSWORD +# - YENTE_ES_PASSWORD +# - BEYOND_AI_BASIC_AUTH_USER +# - BEYOND_AI_BASIC_AUTH_PASSWORD docker compose up -d ``` @@ -140,6 +145,7 @@ Danach verfügbar: ```bash curl -X POST http://localhost:8000/api/screen \ + -u "$BEYOND_AI_BASIC_AUTH_USER:$BEYOND_AI_BASIC_AUTH_PASSWORD" \ -H "Content-Type: application/json" \ -d '{"name": "Wladimir Putin"}' ``` diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 6f92e9b..eff760e 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -23,12 +23,15 @@ services: - "9200" environment: discovery.type: single-node - xpack.security.enabled: "false" + xpack.security.enabled: "true" + xpack.security.autoconfiguration.enabled: "false" + xpack.security.http.ssl.enabled: "false" + ELASTIC_PASSWORD: ${YENTE_ES_PASSWORD:?Set YENTE_ES_PASSWORD in .env} ES_JAVA_OPTS: "-Xms1g -Xmx1g" # KVM 2 (bei Upgrade auf KVM 4 → -Xms2g -Xmx2g) volumes: - yente_es_data:/usr/share/elasticsearch/data healthcheck: - test: ["CMD", "curl", "-sf", "http://localhost:9200/_cluster/health"] + test: ["CMD-SHELL", "curl -sf -u elastic:$$ELASTIC_PASSWORD http://localhost:9200/_cluster/health >/dev/null"] interval: 10s timeout: 5s retries: 10 @@ -48,7 +51,7 @@ services: yente-es: condition: service_healthy environment: - YENTE_INDEX_URL: http://yente-es:9200 + YENTE_INDEX_URL: http://elastic:${YENTE_ES_PASSWORD:?Set YENTE_ES_PASSWORD in .env}@yente-es:9200 YENTE_MANIFEST: /data/manifest.yml volumes: - ./data/yente:/data:ro @@ -80,6 +83,13 @@ services: OLLAMA_MODEL: kimi-k2.5:cloud # Optional: Claude API als Fallback ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + BEYOND_AI_AUTH_REQUIRED: ${BEYOND_AI_AUTH_REQUIRED:-true} + BEYOND_AI_BASIC_AUTH_USER: ${BEYOND_AI_BASIC_AUTH_USER:?Set BEYOND_AI_BASIC_AUTH_USER in .env} + BEYOND_AI_BASIC_AUTH_PASSWORD: ${BEYOND_AI_BASIC_AUTH_PASSWORD:?Set BEYOND_AI_BASIC_AUTH_PASSWORD in .env} + BEYOND_AI_RATE_LIMIT_REQUESTS: ${BEYOND_AI_RATE_LIMIT_REQUESTS:-30} + BEYOND_AI_RATE_LIMIT_WINDOW_SECONDS: ${BEYOND_AI_RATE_LIMIT_WINDOW_SECONDS:-60} + BEYOND_AI_TRUST_PROXY_HEADERS: ${BEYOND_AI_TRUST_PROXY_HEADERS:-true} + BEYOND_AI_ALLOW_EXTERNAL_LLM_FALLBACK: ${BEYOND_AI_ALLOW_EXTERNAL_LLM_FALLBACK:-false} volumes: - ./shared:/app/shared - ./sanctions:/app/sanctions diff --git a/docker-compose.yml b/docker-compose.yml index acf9d1f..af0e94e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,14 +11,14 @@ services: image: neo4j:5-community container_name: beyond-ai-neo4j ports: - - "7474:7474" # Browser - - "7687:7687" # Bolt + - "127.0.0.1:7474:7474" # Browser + - "127.0.0.1:7687:7687" # Bolt volumes: - neo4j_data:/data - neo4j_logs:/logs - ./data/neo4j-import:/var/lib/neo4j/import environment: - NEO4J_AUTH: neo4j/${NEO4J_PASSWORD:-beyond-ai-2026} + NEO4J_AUTH: neo4j/${NEO4J_PASSWORD:?Set NEO4J_PASSWORD in .env} NEO4J_PLUGINS: '["apoc"]' NEO4J_server_memory_heap_initial__size: 1G NEO4J_server_memory_heap_max__size: 2G @@ -31,12 +31,12 @@ services: image: ghcr.io/opensanctions/yente:latest container_name: beyond-ai-yente ports: - - "8100:8000" + - "127.0.0.1:8100:8000" depends_on: yente-es: condition: service_healthy environment: - YENTE_INDEX_URL: http://yente-es:9200 + YENTE_INDEX_URL: http://elastic:${YENTE_ES_PASSWORD:?Set YENTE_ES_PASSWORD in .env}@yente-es:9200 YENTE_MANIFEST: /data/manifest.yml # Für kommerzielle Nutzung: YENTE_DATA_TOKEN setzen # YENTE_DATA_TOKEN: ${OPENSANCTIONS_TOKEN:-} @@ -49,12 +49,15 @@ services: container_name: beyond-ai-yente-es environment: discovery.type: single-node - xpack.security.enabled: "false" + xpack.security.enabled: "true" + xpack.security.autoconfiguration.enabled: "false" + xpack.security.http.ssl.enabled: "false" + ELASTIC_PASSWORD: ${YENTE_ES_PASSWORD:?Set YENTE_ES_PASSWORD in .env} ES_JAVA_OPTS: "-Xms1g -Xmx1g" volumes: - yente_es_data:/usr/share/elasticsearch/data healthcheck: - test: ["CMD", "curl", "-sf", "http://localhost:9200/_cluster/health"] + test: ["CMD-SHELL", "curl -sf -u elastic:$$ELASTIC_PASSWORD http://localhost:9200/_cluster/health >/dev/null"] interval: 10s timeout: 5s retries: 10 @@ -69,18 +72,25 @@ services: dockerfile: Dockerfile container_name: beyond-ai-api ports: - - "8000:8000" + - "127.0.0.1:8000:8000" depends_on: - neo4j - yente environment: NEO4J_URI: bolt://neo4j:7687 NEO4J_USER: neo4j - NEO4J_PASSWORD: ${NEO4J_PASSWORD:-beyond-ai-2026} + NEO4J_PASSWORD: ${NEO4J_PASSWORD:?Set NEO4J_PASSWORD in .env} YENTE_URL: http://yente:8000 OLLAMA_URL: ${OLLAMA_URL:-http://host.docker.internal:11434} OLLAMA_MODEL: ${OLLAMA_MODEL:-kimi-k2.5} ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + BEYOND_AI_AUTH_REQUIRED: ${BEYOND_AI_AUTH_REQUIRED:-true} + BEYOND_AI_BASIC_AUTH_USER: ${BEYOND_AI_BASIC_AUTH_USER:?Set BEYOND_AI_BASIC_AUTH_USER in .env} + BEYOND_AI_BASIC_AUTH_PASSWORD: ${BEYOND_AI_BASIC_AUTH_PASSWORD:?Set BEYOND_AI_BASIC_AUTH_PASSWORD in .env} + BEYOND_AI_RATE_LIMIT_REQUESTS: ${BEYOND_AI_RATE_LIMIT_REQUESTS:-30} + BEYOND_AI_RATE_LIMIT_WINDOW_SECONDS: ${BEYOND_AI_RATE_LIMIT_WINDOW_SECONDS:-60} + BEYOND_AI_TRUST_PROXY_HEADERS: "false" + BEYOND_AI_ALLOW_EXTERNAL_LLM_FALLBACK: ${BEYOND_AI_ALLOW_EXTERNAL_LLM_FALLBACK:-false} volumes: - ./shared:/app/shared - ./sanctions:/app/sanctions diff --git a/requirements.txt b/requirements.txt index 69954f2..1bd4248 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ httpx>=0.27.0 neo4j>=5.0.0 pydantic>=2.0.0 python-dotenv>=1.0.0 +python-multipart>=0.0.20 diff --git a/sanctions/README.md b/sanctions/README.md index dfc3013..e65efc3 100644 --- a/sanctions/README.md +++ b/sanctions/README.md @@ -71,6 +71,7 @@ python src/api.py # Einzelnen Namen screenen curl -X POST http://localhost:8000/api/screen \ + -u "$BEYOND_AI_BASIC_AUTH_USER:$BEYOND_AI_BASIC_AUTH_PASSWORD" \ -H "Content-Type: application/json" \ -d '{"name": "Wladimir Wladimirowitsch Putin", "threshold": 0.7}' diff --git a/sanctions/src/main.py b/sanctions/src/main.py index 8a61fec..cb44bd9 100644 --- a/sanctions/src/main.py +++ b/sanctions/src/main.py @@ -4,17 +4,25 @@ Sprint 1: Name screening against OpenSanctions (yente) with LLM enhancement. """ +import base64 +import binascii import html as html_lib import json +import logging import os +import secrets +import time +from collections import defaultdict, deque +from threading import Lock from typing import Annotated import httpx -from fastapi import FastAPI, HTTPException, Request -from fastapi.responses import HTMLResponse -from pydantic import BaseModel, StringConstraints +from fastapi import FastAPI, Form, HTTPException, Request +from fastapi.responses import HTMLResponse, JSONResponse +from pydantic import BaseModel, Field, StringConstraints APP_VERSION = os.getenv("BEYOND_AI_VERSION", "0.1.0-alpha.1") +logger = logging.getLogger(__name__) app = FastAPI( title="Beyond AI — Sanctions Screener", @@ -28,6 +36,15 @@ "person": "Person", "organization": "Organization", } +CACHE_CONTROL_NO_STORE = "no-store, max-age=0, private" +PROTECTED_DOC_PATHS = frozenset({ + "/docs", + "/docs/oauth2-redirect", + "/openapi.json", + "/redoc", +}) +_RATE_LIMIT_BUCKETS: dict[str, deque[float]] = defaultdict(deque) +_RATE_LIMIT_LOCK = Lock() ScreenName = Annotated[ str, @@ -37,7 +54,7 @@ class ScreenRequest(BaseModel): name: ScreenName - threshold: float = 0.7 + threshold: float = Field(default=0.7, ge=0.0, le=1.0) class ScreenMatch(BaseModel): @@ -54,6 +71,171 @@ class ScreenResponse(BaseModel): total: int +def _env_flag(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() not in {"0", "false", "no", "off"} + + +def _is_protected_path(path: str) -> bool: + return path.startswith("/search") or path in { + "/api/screen", + "/api/status", + *PROTECTED_DOC_PATHS, + } + + +def _auth_required() -> bool: + return _env_flag("BEYOND_AI_AUTH_REQUIRED", True) + + +def _trusted_proxy_headers() -> bool: + return _env_flag("BEYOND_AI_TRUST_PROXY_HEADERS", False) + + +def _basic_auth_config() -> tuple[str, str] | None: + username = os.getenv("BEYOND_AI_BASIC_AUTH_USER", "").strip() + password = os.getenv("BEYOND_AI_BASIC_AUTH_PASSWORD", "") + if not username or not password: + return None + return username, password + + +def _parse_basic_auth(header_value: str | None) -> tuple[str, str] | None: + if not header_value: + return None + + scheme, _, token = header_value.partition(" ") + if scheme.lower() != "basic" or not token: + return None + + try: + decoded = base64.b64decode(token, validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + return None + + username, separator, password = decoded.partition(":") + if not separator: + return None + return username, password + + +def _authenticated(request: Request) -> bool: + expected = _basic_auth_config() + if expected is None: + return False + + provided = _parse_basic_auth(request.headers.get("Authorization")) + if provided is None: + return False + + expected_user, expected_password = expected + provided_user, provided_password = provided + return secrets.compare_digest(provided_user, expected_user) and secrets.compare_digest( + provided_password, expected_password + ) + + +def _rate_limit_window_seconds() -> int: + return max(1, int(os.getenv("BEYOND_AI_RATE_LIMIT_WINDOW_SECONDS", "60"))) + + +def _rate_limit_requests() -> int: + return max(1, int(os.getenv("BEYOND_AI_RATE_LIMIT_REQUESTS", "30"))) + + +def _client_identifier(request: Request) -> str: + forwarded_for = request.headers.get("X-Forwarded-For") + if _trusted_proxy_headers() and forwarded_for: + return forwarded_for.split(",")[0].strip() + if request.client is not None: + return request.client.host + return "unknown" + + +async def _rate_limit_exceeded(request: Request) -> tuple[bool, int]: + window_seconds = _rate_limit_window_seconds() + max_requests = _rate_limit_requests() + now = time.monotonic() + bucket_key = f"{request.url.path}:{_client_identifier(request)}" + window_start = now - window_seconds + + with _RATE_LIMIT_LOCK: + bucket = _RATE_LIMIT_BUCKETS[bucket_key] + while bucket and bucket[0] <= window_start: + bucket.popleft() + + if len(bucket) >= max_requests: + retry_after = max(1, int(bucket[0] + window_seconds - now)) + return True, retry_after + + bucket.append(now) + + return False, 0 + + +def _apply_security_headers(response, path: str): + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["Referrer-Policy"] = "no-referrer" + response.headers["X-Frame-Options"] = "DENY" + + if _is_protected_path(path): + response.headers["Cache-Control"] = CACHE_CONTROL_NO_STORE + response.headers["Pragma"] = "no-cache" + response.headers["Vary"] = "Authorization" + + if path.startswith("/search"): + response.headers["Content-Security-Policy"] = ( + "default-src 'self'; " + "style-src 'self' 'unsafe-inline'; " + "img-src 'self' data:; " + "base-uri 'none'; " + "form-action 'self'; " + "frame-ancestors 'none'" + ) + + return response + + +@app.middleware("http") +async def protect_sensitive_endpoints(request: Request, call_next): + path = request.url.path + + if _is_protected_path(path): + if _auth_required(): + auth_config = _basic_auth_config() + if auth_config is None: + logger.error( + "Protected endpoint requested without configured auth credentials." + ) + response = JSONResponse( + status_code=503, + content={"detail": "service is not securely configured"}, + ) + return _apply_security_headers(response, path) + + if not _authenticated(request): + response = JSONResponse( + status_code=401, + content={"detail": "authentication required"}, + headers={"WWW-Authenticate": 'Basic realm="Beyond AI"'}, + ) + return _apply_security_headers(response, path) + + limited, retry_after = await _rate_limit_exceeded(request) + if limited: + response = JSONResponse( + status_code=429, + content={"detail": "rate limit exceeded"}, + headers={"Retry-After": str(retry_after)}, + ) + return _apply_security_headers(response, path) + + response = await call_next(request) + return _apply_security_headers(response, path) + + async def _query_yente(name: str, threshold: float = 0.7) -> list[dict]: query_name = name.strip() queries = { @@ -97,199 +279,456 @@ async def health(): return {"status": "ok", "service": "beyond-ai-api", "version": APP_VERSION} -@app.get("/search", response_class=HTMLResponse) -async def search_ui(request: Request, q: str = "", threshold: float = 0.7): - """Browser-Suche: HTML + JSON side-by-side.""" - matches = [] - error = None - raw_json = "" - normalized_query = q.strip() - +def _build_search_page( + q: str = "", + threshold: float = 0.7, + matches: list[dict] | None = None, + error: str | None = None, + raw_json: str = "", +) -> str: def escape_text(value: object) -> str: return html_lib.escape(str(value), quote=True) - if q and not normalized_query: - error = "Bitte einen Namen eingeben." - elif normalized_query: - try: - results = await _query_yente(normalized_query, threshold) - matches = [ - { - "id": r.get("id", ""), - "name": r.get("caption", r.get("name", "")), - "score": r.get("score", 0.0), - "datasets": r.get("datasets", []), - "properties": r.get("properties", {}), - } - for r in results - ] - raw_json = json.dumps( - { - "query": normalized_query, - "total": len(matches), - "matches": matches, - }, - indent=2, - ensure_ascii=False, - ) - except httpx.ConnectError: - error = "yente nicht erreichbar — läuft der Container?" - except Exception as e: - error = str(e) + normalized_query = q.strip() + matches = matches or [] def score_color(score: float) -> str: if score >= 0.9: - return "#dc2626" # rot — hohe Übereinstimmung + return "#dc2626" if score >= 0.8: - return "#d97706" # orange - return "#ca8a04" # gelb + return "#d97706" + return "#ca8a04" def dataset_badge(ds: str) -> str: colors = { - "sanctions": ("bg-red-100 text-red-800", "Sanctions"), - "peps": ("bg-orange-100 text-orange-800", "PEP"), + "sanctions": ("badge badge-danger", "Sanctions"), + "peps": ("badge badge-warning", "PEP"), } - cls, label = colors.get(ds, ("bg-gray-100 text-gray-700", ds.upper())) - return ( - f'' - f"{escape_text(label)}" - ) + cls, label = colors.get(ds, ("badge badge-neutral", ds.upper())) + return f'{escape_text(label)}' def prop_row(key: str, vals: list) -> str: if not vals: return "" joined = " · ".join(escape_text(v) for v in vals[:5]) if len(vals) > 5: - joined += f" +{len(vals)-5} weitere" + joined += f" +{len(vals)-5} weitere" return f""" -
ID: {safe_id}
+ID: {safe_id}
{safe_raw_json}
+ {safe_raw_json}