diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..6cd05ed --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +addopts = --import-mode=importlib +testpaths = + services/gateway-api/tests + services/evaluator/tests diff --git a/scripts/test-pytest.sh b/scripts/test-pytest.sh new file mode 100755 index 0000000..2234ff2 --- /dev/null +++ b/scripts/test-pytest.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +cd "$ROOT_DIR" + +uv run --project services/gateway-api --group dev pytest services/gateway-api/tests "$@" +uv run --project services/evaluator --group dev pytest services/evaluator/tests "$@" diff --git a/services/evaluator/app/config.py b/services/evaluator/app/config.py index 4352e1a..361d610 100644 --- a/services/evaluator/app/config.py +++ b/services/evaluator/app/config.py @@ -1,4 +1,6 @@ -from pydantic_settings import BaseSettings +from functools import lru_cache + +from pydantic_settings import BaseSettings, SettingsConfigDict class Settings(BaseSettings): @@ -7,11 +9,11 @@ class Settings(BaseSettings): log_level: str = "INFO" # DB - database_url: str + database_url: str | None = None # LLM (Judge 용) llm_api_base_url: str | None = None - llm_api_key: str + llm_api_key: str | None = None openai_model_judge: str = "gpt-5-mini" # Batch Evaluation Scheduler @@ -33,9 +35,12 @@ class Settings(BaseSettings): smtp_from_email: str | None = None # 발신자 이메일 smtp_to_emails: str | None = None # 수신자 이메일들 (쉼표로 구분) - class Config: - env_file = ".env" - env_file_encoding = "utf-8" + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + ) -settings = Settings() +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/services/evaluator/app/db.py b/services/evaluator/app/db.py index 27657bc..2cf20d3 100644 --- a/services/evaluator/app/db.py +++ b/services/evaluator/app/db.py @@ -1,16 +1,29 @@ +from functools import lru_cache + from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, declarative_base -from .config import settings - -engine = create_engine(settings.database_url) -SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +from .config import get_settings Base = declarative_base() +@lru_cache +def get_engine(): + settings = get_settings() + if not settings.database_url: + raise RuntimeError("DATABASE_URL is required to initialize the database engine.") + + return create_engine(settings.database_url) + + +@lru_cache +def get_session_factory(): + return sessionmaker(autocommit=False, autoflush=False, bind=get_engine()) + + def get_db(): - db = SessionLocal() + db = get_session_factory()() try: yield db finally: diff --git a/services/evaluator/app/llm_judge.py b/services/evaluator/app/llm_judge.py index 09eaab0..6564cfb 100644 --- a/services/evaluator/app/llm_judge.py +++ b/services/evaluator/app/llm_judge.py @@ -1,6 +1,7 @@ import json import textwrap import time +from functools import lru_cache from typing import TypedDict from fastapi import HTTPException @@ -12,7 +13,7 @@ AuthenticationError, ) -from .config import settings +from .config import get_settings from .models import LLMLog @@ -24,10 +25,19 @@ class EvaluationResult(TypedDict): raw_judge_response: str -client = OpenAI( - api_key=settings.llm_api_key, - base_url=settings.llm_api_base_url or None, -) +@lru_cache +def get_client() -> OpenAI: + settings = get_settings() + if not settings.llm_api_key: + raise HTTPException( + status_code=500, + detail="LLM_API_KEY is required for LLM judge evaluation.", + ) + + return OpenAI( + api_key=settings.llm_api_key, + base_url=settings.llm_api_base_url or None, + ) def build_evaluation_prompt(log: LLMLog) -> str: @@ -110,7 +120,9 @@ def run_judge(log: LLMLog) -> EvaluationResult: """ 하나의 LLMLog에 대해 Judge LLM을 호출하고 EvaluationResult 반환. """ + settings = get_settings() prompt = build_evaluation_prompt(log) + client = get_client() try: start = time.perf_counter() diff --git a/services/evaluator/app/main.py b/services/evaluator/app/main.py index ccb48cc..47c923e 100644 --- a/services/evaluator/app/main.py +++ b/services/evaluator/app/main.py @@ -6,16 +6,18 @@ import time from prometheus_client import generate_latest, CONTENT_TYPE_LATEST -from .db import Base, engine, get_db +from .db import Base, get_db, get_engine from .models import LLMLog, LLMEvaluation from .rules import basic_rule_evaluate from .llm_judge import run_judge -from .config import settings +from .config import get_settings from .scheduler import start_scheduler, stop_scheduler from .utils import get_pending_logs from .metrics import record_evaluation, update_pending_logs_count from .notifier import send_low_quality_alert +settings = get_settings() + # 로깅 설정 logging.basicConfig( level=getattr(logging, settings.log_level.upper()), @@ -24,29 +26,33 @@ logger = logging.getLogger(__name__) -@asynccontextmanager -async def lifespan(app: FastAPI): - """ - FastAPI 앱의 수명 주기 관리. - 시작 시 테이블 생성 및 스케줄러 시작, 종료 시 스케줄러 중지. - """ - # Startup - logger.info("Starting Evaluator Service...") - Base.metadata.create_all(bind=engine) - start_scheduler() - yield - # Shutdown - logger.info("Stopping Evaluator Service...") - stop_scheduler() - - -# FastAPI 앱 생성 -app = FastAPI( - title="LLM Quality Observer - Evaluator Service", - description="룰 기반 및 LLM-as-a-judge 방식으로 LLM 응답 품질을 평가하는 서비스", - version="1.0.0", - lifespan=lifespan, -) +def create_app(*, testing: bool = False) -> FastAPI: + @asynccontextmanager + async def lifespan(app: FastAPI): + """ + FastAPI 앱의 수명 주기 관리. + 시작 시 테이블 생성 및 스케줄러 시작, 종료 시 스케줄러 중지. + """ + if not getattr(app.state, "testing", False): + logger.info("Starting Evaluator Service...") + Base.metadata.create_all(bind=get_engine()) + start_scheduler() + yield + if not getattr(app.state, "testing", False): + logger.info("Stopping Evaluator Service...") + stop_scheduler() + + app = FastAPI( + title="LLM Quality Observer - Evaluator Service", + description="룰 기반 및 LLM-as-a-judge 방식으로 LLM 응답 품질을 평가하는 서비스", + version="1.0.0", + lifespan=lifespan, + ) + app.state.testing = testing + return app + + +app = create_app() @app.get("/health") @@ -57,7 +63,7 @@ def health_check(): """ return { "status": "ok", - "env": settings.app_env, + "env": get_settings().app_env, } @@ -93,6 +99,7 @@ def evaluate_once( pending_logs = get_pending_logs(db, limit=limit) if not pending_logs: + settings = get_settings() return { "evaluated": 0, "judge_type": judge_type, @@ -102,6 +109,7 @@ def evaluate_once( # 2. 각 로그에 대해 평가 수행 evaluated_count = 0 judge_model_name = "" + settings = get_settings() for log in pending_logs: try: diff --git a/services/evaluator/app/notifier.py b/services/evaluator/app/notifier.py index 8134d44..5e1ac91 100644 --- a/services/evaluator/app/notifier.py +++ b/services/evaluator/app/notifier.py @@ -11,7 +11,7 @@ from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart -from .config import settings +from .config import get_settings from .models import LLMLog, LLMEvaluation from .metrics import record_notification, record_low_quality_alert @@ -29,6 +29,8 @@ def send_slack_notification(message: str, notification_type: str = "alert") -> b Returns: bool: 전송 성공 여부 """ + settings = get_settings() + if not settings.slack_webhook_url: logger.debug("Slack webhook URL이 설정되지 않았습니다.") return False @@ -61,6 +63,8 @@ def send_discord_notification(message: str, notification_type: str = "alert") -> Returns: bool: 전송 성공 여부 """ + settings = get_settings() + if not settings.discord_webhook_url: logger.debug("Discord webhook URL이 설정되지 않았습니다.") return False @@ -95,6 +99,8 @@ async def send_email_notification(subject: str, message: str, notification_type: Returns: bool: 전송 성공 여부 """ + settings = get_settings() + if not all([ settings.smtp_host, settings.smtp_username, @@ -154,6 +160,8 @@ def send_low_quality_alert(log: LLMLog, evaluation: LLMEvaluation): log: LLM 로그 evaluation: 평가 결과 """ + settings = get_settings() + if evaluation.overall_score >= settings.notification_score_threshold: # 임계값 이상이면 알림 안 보냄 return diff --git a/services/evaluator/app/scheduler.py b/services/evaluator/app/scheduler.py index b447acc..6e789d5 100644 --- a/services/evaluator/app/scheduler.py +++ b/services/evaluator/app/scheduler.py @@ -9,8 +9,8 @@ from apscheduler.triggers.interval import IntervalTrigger from sqlalchemy.orm import Session -from .config import settings -from .db import SessionLocal +from .config import get_settings +from .db import get_session_factory from .utils import get_pending_logs from .models import LLMLog, LLMEvaluation from .rules import basic_rule_evaluate @@ -36,7 +36,8 @@ def run_batch_evaluation(): """ logger.info("Starting batch evaluation...") - db: Session = SessionLocal() + settings = get_settings() + db: Session = get_session_factory()() try: # 1. 평가 대기 중인 로그 가져오기 pending_logs = get_pending_logs( @@ -148,6 +149,7 @@ def start_scheduler(): 스케줄러를 시작합니다. """ global scheduler + settings = get_settings() if not settings.enable_auto_evaluation: logger.info("Auto evaluation is disabled") diff --git a/services/evaluator/app/schemas.py b/services/evaluator/app/schemas.py index 1100250..291b600 100644 --- a/services/evaluator/app/schemas.py +++ b/services/evaluator/app/schemas.py @@ -1,5 +1,5 @@ from typing import Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class EvaluationResult(BaseModel): @@ -14,5 +14,4 @@ class EvaluationResult(BaseModel): judge_model: str = Field(default="rule-basic-v1", description="평가에 사용된 모델/룰 버전") comment: Optional[str] = Field(default=None, description="평가 근거 또는 코멘트") - class Config: - from_attributes = True # Pydantic v2에서 ORM 모드 활성화 + model_config = ConfigDict(from_attributes=True) diff --git a/services/evaluator/pyproject.toml b/services/evaluator/pyproject.toml index b10681f..c8c4322 100644 --- a/services/evaluator/pyproject.toml +++ b/services/evaluator/pyproject.toml @@ -23,5 +23,10 @@ dependencies = [ requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" +[dependency-groups] +dev = [ + "pytest>=8.0", +] + [tool.setuptools] packages = ["app"] diff --git a/services/evaluator/tests/test_health.py b/services/evaluator/tests/test_health.py index e8d2cf3..f949c0a 100644 --- a/services/evaluator/tests/test_health.py +++ b/services/evaluator/tests/test_health.py @@ -2,9 +2,20 @@ Health check endpoint tests """ +import sys +from pathlib import Path + from fastapi.testclient import TestClient + +SERVICE_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SERVICE_ROOT)) + +for module_name in [name for name in list(sys.modules) if name == "app" or name.startswith("app.")]: + sys.modules.pop(module_name) + from app.main import app +app.state.testing = True client = TestClient(app) diff --git a/services/gateway-api/app/config.py b/services/gateway-api/app/config.py index 77797d9..0e1537c 100644 --- a/services/gateway-api/app/config.py +++ b/services/gateway-api/app/config.py @@ -1,10 +1,12 @@ -from pydantic_settings import BaseSettings +from functools import lru_cache + +from pydantic_settings import BaseSettings, SettingsConfigDict class Settings(BaseSettings): app_env: str = "local" - database_url: str + database_url: str | None = None openai_model_main: str = "gpt-5-mini" llm_api_base_url: str | None = None @@ -16,9 +18,12 @@ class Settings(BaseSettings): log_level: str = "INFO" - class Config: - env_file = ".env" - env_file_encoding = "utf-8" + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + ) -settings = Settings() +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/services/gateway-api/app/db.py b/services/gateway-api/app/db.py index 12aa1a1..ab8b1cd 100644 --- a/services/gateway-api/app/db.py +++ b/services/gateway-api/app/db.py @@ -1,21 +1,34 @@ +from functools import lru_cache + from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, declarative_base -from .config import settings - -engine = create_engine(settings.database_url, future=True) -SessionLocal = sessionmaker( - autocommit=False, - autoflush=False, - bind=engine, - future=True, -) +from .config import get_settings Base = declarative_base() +@lru_cache +def get_engine(): + settings = get_settings() + if not settings.database_url: + raise RuntimeError("DATABASE_URL is required to initialize the database engine.") + + return create_engine(settings.database_url, future=True) + + +@lru_cache +def get_session_factory(): + return sessionmaker( + autocommit=False, + autoflush=False, + bind=get_engine(), + future=True, + ) + + def get_db(): - db = SessionLocal() + db = get_session_factory()() try: yield db finally: diff --git a/services/gateway-api/app/llm_client.py b/services/gateway-api/app/llm_client.py index 5707337..0a8b31c 100644 --- a/services/gateway-api/app/llm_client.py +++ b/services/gateway-api/app/llm_client.py @@ -1,23 +1,30 @@ -import time -import os -from litellm import completion - -from .config import settings - -# LiteLLM 설정 -os.environ["OPENAI_API_KEY"] = settings.llm_api_key or "" -if settings.llm_api_base_url: - os.environ["OPENAI_API_BASE"] = settings.llm_api_base_url - - -def _resolve_model(model_version: str | None) -> str: - """ - 요청에서 온 model_version이 이상하면 무시하고 - 기본 모델(openai_model_main)을 쓰도록 정리. +import time +import os +from litellm import completion + +from .config import get_settings + + +def _configure_provider_env() -> None: + settings = get_settings() + + if settings.llm_api_key: + os.environ["OPENAI_API_KEY"] = settings.llm_api_key + + if settings.llm_api_base_url: + os.environ["OPENAI_API_BASE"] = settings.llm_api_base_url + + +def _resolve_model(model_version: str | None) -> str: """ - if not model_version: - return settings.openai_model_main - + 요청에서 온 model_version이 이상하면 무시하고 + 기본 모델(openai_model_main)을 쓰도록 정리. + """ + settings = get_settings() + + if not model_version: + return settings.openai_model_main + # Swagger 기본 예제가 "string"이라서, 그 값이 오면 무시 if model_version == "string": return settings.openai_model_main @@ -34,19 +41,22 @@ def call_llm(prompt: str, model_version: str | None = None) -> dict: "response": str, "model_version": str, "latency_ms": float, - "usage": { - "input_tokens": int, - "output_tokens": int, + "usage": { + "input_tokens": int, + "output_tokens": int, "total_tokens": int, "cached_tokens": int, "reasoning_tokens": int } - } - """ - model = _resolve_model(model_version) - models_to_try = [model] + settings.fallback_models - - last_error = None + } + """ + settings = get_settings() + _configure_provider_env() + + model = _resolve_model(model_version) + models_to_try = list(dict.fromkeys([model, *settings.fallback_models])) + + last_error = None for attempt_model in models_to_try: try: diff --git a/services/gateway-api/app/main.py b/services/gateway-api/app/main.py index d46a2d4..03c4049 100644 --- a/services/gateway-api/app/main.py +++ b/services/gateway-api/app/main.py @@ -1,13 +1,15 @@ -from fastapi import FastAPI, Depends, Query, Response -from fastapi.middleware.cors import CORSMiddleware -from sqlalchemy.orm import Session -from sqlalchemy import func, select, distinct +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Depends, Query, Response +from fastapi.middleware.cors import CORSMiddleware +from sqlalchemy.orm import Session +from sqlalchemy import func, select, distinct import math import time from prometheus_client import generate_latest, CONTENT_TYPE_LATEST -from .db import Base, engine, get_db -from .models import LLMLog, LLMEvaluation, LLMModelPricing +from .db import Base, get_db, get_engine +from .models import LLMLog, LLMEvaluation, LLMModelPricing from .schemas import ( ChatRequest, ChatResponse, @@ -38,32 +40,43 @@ ModelPricingResponse, ModelPricingInfo, ) -from .llm_client import call_llm -from .config import settings +from .llm_client import call_llm +from .config import get_settings from .metrics import ( MetricsMiddleware, record_llm_request, record_db_query, record_log_saved, ) -from .cost_utils import get_model_pricing, calculate_cost - -# 최초 실행 시 테이블 생성 (간단 버전) -Base.metadata.create_all(bind=engine) - -app = FastAPI(title="LLM Quality Observer - Gateway API") - -# Prometheus 메트릭 미들웨어 추가 -app.add_middleware(MetricsMiddleware) - -# CORS 설정 추가 (웹 대시보드에서 API 호출을 위해 필요) -app.add_middleware( - CORSMiddleware, - allow_origins=["http://localhost:3000"], # Next.js dev server - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) +from .cost_utils import get_model_pricing, calculate_cost + + +def create_app(*, testing: bool = False) -> FastAPI: + @asynccontextmanager + async def lifespan(app: FastAPI): + if not getattr(app.state, "testing", False): + Base.metadata.create_all(bind=get_engine()) + yield + + app = FastAPI(title="LLM Quality Observer - Gateway API", lifespan=lifespan) + app.state.testing = testing + + # Prometheus 메트릭 미들웨어 추가 + app.add_middleware(MetricsMiddleware) + + # CORS 설정 추가 (웹 대시보드에서 API 호출을 위해 필요) + app.add_middleware( + CORSMiddleware, + allow_origins=["http://localhost:3000"], # Next.js dev server + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + return app + + +app = create_app() @app.get("/health") @@ -77,14 +90,15 @@ def metrics(): return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST) -def resolve_model_version(request_model: str | None) -> str: - """ - 요청에서 들어온 model_version이 없거나 Swagger 기본값("string")이면 - 환경변수로 설정한 기본 모델(openai_model_main)을 사용한다. +def resolve_model_version(request_model: str | None) -> str: """ - if not request_model or request_model == "string": - return settings.openai_model_main - return request_model + 요청에서 들어온 model_version이 없거나 Swagger 기본값("string")이면 + 환경변수로 설정한 기본 모델(openai_model_main)을 사용한다. + """ + settings = get_settings() + if not request_model or request_model == "string": + return settings.openai_model_main + return request_model @app.post("/chat", response_model=ChatResponse) @@ -808,8 +822,8 @@ def get_cost_summary( @app.get("/cost/trends", response_model=CostTrendResponse) -def get_cost_trends( - granularity: str = Query("hour", regex="^(hour|day|week|month)$", description="시간 단위"), +def get_cost_trends( + granularity: str = Query("hour", pattern="^(hour|day|week|month)$", description="시간 단위"), hours: int | None = Query(None, ge=1, le=720, description="조회할 시간 (최대 30일)"), days: int | None = Query(None, ge=1, le=90, description="조회할 일수 (최대 90일)"), db: Session = Depends(get_db), diff --git a/services/gateway-api/app/schemas.py b/services/gateway-api/app/schemas.py index 445de44..b3b0160 100644 --- a/services/gateway-api/app/schemas.py +++ b/services/gateway-api/app/schemas.py @@ -1,7 +1,7 @@ -from datetime import datetime -from decimal import Decimal - -from pydantic import BaseModel +from datetime import datetime +from decimal import Decimal + +from pydantic import BaseModel, ConfigDict class ChatRequest(BaseModel): @@ -34,7 +34,7 @@ class ChatResponse(BaseModel): cost: CostInfo | None = None -class LLMLogRead(BaseModel): +class LLMLogRead(BaseModel): id: int created_at: datetime user_id: str | None @@ -44,8 +44,7 @@ class LLMLogRead(BaseModel): latency_ms: float | None status: str - class Config: - from_attributes = True + model_config = ConfigDict(from_attributes=True) # Dashboard API Schemas @@ -58,7 +57,7 @@ class DashboardSummary(BaseModel): avg_score: float | None -class LogListItem(BaseModel): +class LogListItem(BaseModel): """로그 목록용 간략한 로그 정보""" id: int created_at: datetime @@ -69,8 +68,7 @@ class LogListItem(BaseModel): latency_ms: float | None status: str - class Config: - from_attributes = True + model_config = ConfigDict(from_attributes=True) class LogListResponse(BaseModel): @@ -82,7 +80,7 @@ class LogListResponse(BaseModel): total_pages: int -class EvaluationRead(BaseModel): +class EvaluationRead(BaseModel): """평가 결과 읽기용 스키마""" id: int created_at: datetime @@ -100,8 +98,7 @@ class EvaluationRead(BaseModel): log_response: str | None = None log_model_version: str | None = None - class Config: - from_attributes = True + model_config = ConfigDict(from_attributes=True) class EvaluationListResponse(BaseModel): diff --git a/services/gateway-api/pyproject.toml b/services/gateway-api/pyproject.toml index 0ffaff0..c598815 100644 --- a/services/gateway-api/pyproject.toml +++ b/services/gateway-api/pyproject.toml @@ -21,5 +21,10 @@ dependencies = [ requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" +[dependency-groups] +dev = [ + "pytest>=8.0", +] + [tool.uv] # uv 관련 설정 필요하면 나중에 추가 diff --git a/services/gateway-api/tests/test_health.py b/services/gateway-api/tests/test_health.py index e8d2cf3..f949c0a 100644 --- a/services/gateway-api/tests/test_health.py +++ b/services/gateway-api/tests/test_health.py @@ -2,9 +2,20 @@ Health check endpoint tests """ +import sys +from pathlib import Path + from fastapi.testclient import TestClient + +SERVICE_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SERVICE_ROOT)) + +for module_name in [name for name in list(sys.modules) if name == "app" or name.startswith("app.")]: + sys.modules.pop(module_name) + from app.main import app +app.state.testing = True client = TestClient(app)