diff --git a/README.md b/README.md index d24cc19..46192b0 100644 --- a/README.md +++ b/README.md @@ -138,9 +138,11 @@ docker compose up -d - `openremit-mysql` (3306, binlog ROW + GTID 활성) - `openremit-redis` (6379) - `openremit-kafka` (9092 host / 29092 internal, KRaft single-node) -- `openremit-mock-fx` (9999), `openremit-mock-payout` (9998) — WireMock +- `openremit-mock-fx` (9999), `openremit-mock-payout` (9998), `openremit-mock-webhook` (9997) — WireMock - `openremit-debezium` (8083) — Connect standalone - `openremit-debezium-register` — outbox 커넥터 자동 등록 (1회성, retry 6회) +- `openremit-prometheus` (9090) — `host.docker.internal`로 호스트 JVM의 4개 모듈 scrape +- `openremit-grafana` (3000) — anonymous Viewer 허용, datasource(`prometheus`) + 대시보드(`openremit`) provisioning 검증: ```bash @@ -158,3 +160,37 @@ docker exec -it openremit-kafka /opt/kafka/bin/kafka-console-consumer.sh \ ``` 볼륨 초기화가 필요하면 `docker compose down -v` 후 재기동 (mysql-init 스크립트가 다시 실행되어 Debezium 사용자가 생성됨). + +## 관측성 — 메트릭 / 로그 (Day 12) + +### 메트릭 + +각 Spring Boot 모듈이 `/actuator/prometheus`를 노출하고, `docker-compose`의 Prometheus가 `host.docker.internal:808x`로 scrape 합니다. + +```bash +curl -s http://localhost:8080/actuator/prometheus | head # remittance-api +curl -s http://localhost:9090/api/v1/targets | jq '.data.activeTargets[].health' +``` + +| 모듈 | actuator 포트 | prometheus 라벨 | +|---|---|---| +| `remittance-api` | 8080 | `service="remittance-api"` | +| `payout-worker` | 8081 | `service="payout-worker"` | +| `webhook-dispatcher` | 8082 | `service="webhook-dispatcher"` | +| `reconciler` | 8084 | `service="reconciler"` | + +Grafana 접속: `http://localhost:3000` (anonymous Viewer). 좌측 메뉴 → Dashboards → **OpenRemit / OpenRemit — Service Overview** 자동 로드. 대시보드 UID는 `openremit`이며 [`docker/grafana/dashboards/openremit.json`](docker/grafana/dashboards/openremit.json)에서 관리합니다. + +### 로깅 + +`logback-spring.xml`은 `common` 모듈에 단일 정의되며 profile에 따라 분기합니다. + +- **`local`** (개발): 사람이 읽기 좋은 콘솔 텍스트 +- **그 외 default/docker/prod**: 한 줄 JSON (`logstash-logback-encoder` 8.0) + +```bash +SPRING_PROFILES_ACTIVE=local ./gradlew :remittance-api:bootRun # 텍스트 +./gradlew :remittance-api:bootRun # JSON (default) +``` + +JSON 한 줄에는 `@timestamp`, `level`, `logger_name`, `thread_name`, `message`, `appName`(=spring.application.name), `app`(customField, 동일 값) 필드가 부착되어 Loki/ELK/Datadog 같은 수집기에서 즉시 라벨링 가능합니다. diff --git a/common/src/main/resources/logback-spring.xml b/common/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..1643fb7 --- /dev/null +++ b/common/src/main/resources/logback-spring.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + %d{HH:mm:ss.SSS} %-5level [%thread] %logger{36} - %msg%n + + + + + + + + + traceId + spanId + {"app":"${appName}"} + + + + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index 4e77a67..977202e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -169,6 +169,47 @@ services: exit 1 restart: "no" + prometheus: + image: prom/prometheus:v2.55.0 + container_name: openremit-prometheus + ports: + - "9090:9090" + volumes: + - ./docker/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + extra_hosts: + - "host.docker.internal:host-gateway" + healthcheck: + test: ["CMD-SHELL", "wget -q -O- http://localhost:9090/-/ready >/dev/null || exit 1"] + interval: 10s + timeout: 3s + retries: 10 + + grafana: + image: grafana/grafana:11.3.0 + container_name: openremit-grafana + depends_on: + prometheus: + condition: service_healthy + ports: + - "3000:3000" + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + volumes: + - ./docker/grafana/provisioning:/etc/grafana/provisioning:ro + - ./docker/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + healthcheck: + test: ["CMD-SHELL", "wget -q -O- http://localhost:3000/api/health >/dev/null || exit 1"] + interval: 10s + timeout: 3s + retries: 10 + volumes: mysql-data: redis-data: + prometheus-data: + grafana-data: diff --git a/docker/grafana/dashboards/openremit.json b/docker/grafana/dashboards/openremit.json new file mode 100644 index 0000000..bd69c47 --- /dev/null +++ b/docker/grafana/dashboards/openremit.json @@ -0,0 +1,112 @@ +{ + "uid": "openremit", + "title": "OpenRemit — Service Overview", + "tags": ["openremit"], + "schemaVersion": 39, + "timezone": "browser", + "refresh": "10s", + "time": { "from": "now-30m", "to": "now" }, + "templating": { + "list": [ + { + "name": "service", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(http_server_requests_seconds_count, service)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "HTTP p95 latency by URI", + "gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, uri) (rate(http_server_requests_seconds_bucket{service=~\"$service\"}[1m])))", + "legendFormat": "{{uri}}" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Remittance throughput (POST /api/v1/remittances req/s)", + "gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 }, + "fieldConfig": { "defaults": { "unit": "reqps" } }, + "targets": [ + { + "expr": "sum by (status) (rate(http_server_requests_seconds_count{service=\"remittance-api\", uri=\"/api/v1/remittances\", method=\"POST\"}[1m]))", + "legendFormat": "status={{status}}" + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Circuit Breaker state (fx-rate)", + "gridPos": { "x": 0, "y": 8, "w": 8, "h": 6 }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "type": "value", "options": { "0": { "text": "INACTIVE", "color": "text" } } }, + { "type": "value", "options": { "1": { "text": "ACTIVE", "color": "green" } } } + ], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }, + "targets": [ + { + "expr": "resilience4j_circuitbreaker_state{name=\"fx-rate\"}", + "legendFormat": "{{state}}" + } + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "CB calls by kind (fx-rate)", + "gridPos": { "x": 8, "y": 8, "w": 16, "h": 6 }, + "fieldConfig": { "defaults": { "unit": "ops" } }, + "targets": [ + { + "expr": "sum by (kind) (rate(resilience4j_circuitbreaker_calls_seconds_count{name=\"fx-rate\"}[1m]))", + "legendFormat": "{{kind}}" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "JVM heap used", + "gridPos": { "x": 0, "y": 14, "w": 12, "h": 7 }, + "fieldConfig": { "defaults": { "unit": "bytes" } }, + "targets": [ + { + "expr": "sum by (service) (jvm_memory_used_bytes{area=\"heap\", service=~\"$service\"})", + "legendFormat": "{{service}}" + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "HTTP request rate by service", + "gridPos": { "x": 12, "y": 14, "w": 12, "h": 7 }, + "fieldConfig": { "defaults": { "unit": "reqps" } }, + "targets": [ + { + "expr": "sum by (service) (rate(http_server_requests_seconds_count{service=~\"$service\"}[1m]))", + "legendFormat": "{{service}}" + } + ] + } + ] +} diff --git a/docker/grafana/provisioning/dashboards/provider.yml b/docker/grafana/provisioning/dashboards/provider.yml new file mode 100644 index 0000000..ee81986 --- /dev/null +++ b/docker/grafana/provisioning/dashboards/provider.yml @@ -0,0 +1,12 @@ +apiVersion: 1 +providers: + - name: openremit + orgId: 1 + folder: OpenRemit + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/docker/grafana/provisioning/datasources/prometheus.yml b/docker/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..433ef5c --- /dev/null +++ b/docker/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/docker/prometheus.yml b/docker/prometheus.yml new file mode 100644 index 0000000..4219b3d --- /dev/null +++ b/docker/prometheus.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: openremit + metrics_path: /actuator/prometheus + static_configs: + - targets: ["host.docker.internal:8080"] + labels: { service: remittance-api } + - targets: ["host.docker.internal:8081"] + labels: { service: payout-worker } + - targets: ["host.docker.internal:8082"] + labels: { service: webhook-dispatcher } + - targets: ["host.docker.internal:8084"] + labels: { service: reconciler } diff --git a/payout-worker/build.gradle.kts b/payout-worker/build.gradle.kts index cdd5e67..ddbf190 100644 --- a/payout-worker/build.gradle.kts +++ b/payout-worker/build.gradle.kts @@ -13,6 +13,8 @@ dependencies { implementation(project(":common")) implementation("org.springframework.boot:spring-boot-starter-actuator") + implementation("io.micrometer:micrometer-registry-prometheus") + implementation("net.logstash.logback:logstash-logback-encoder:8.0") implementation("org.springframework.boot:spring-boot-starter-data-jpa") implementation("org.springframework.boot:spring-boot-starter-flyway") implementation("org.springframework.boot:spring-boot-starter-webmvc") diff --git a/payout-worker/src/main/resources/application.yaml b/payout-worker/src/main/resources/application.yaml index a41421b..614fa4f 100644 --- a/payout-worker/src/main/resources/application.yaml +++ b/payout-worker/src/main/resources/application.yaml @@ -47,7 +47,14 @@ management: endpoints: web: exposure: - include: health,info,metrics + include: health,info,metrics,prometheus endpoint: health: show-details: when-authorized + metrics: + tags: + application: ${spring.application.name} + prometheus: + metrics: + export: + enabled: true diff --git a/reconciler/build.gradle.kts b/reconciler/build.gradle.kts index 5e0911b..ed5c010 100644 --- a/reconciler/build.gradle.kts +++ b/reconciler/build.gradle.kts @@ -13,6 +13,10 @@ dependencies { implementation(project(":common")) implementation("org.springframework.boot:spring-boot-starter-actuator") + implementation("io.micrometer:micrometer-registry-prometheus") + implementation("net.logstash.logback:logstash-logback-encoder:8.0") + // batch 모듈이지만 actuator/prometheus HTTP scrape를 위해 webmvc 임베디드 (Day 12) + implementation("org.springframework.boot:spring-boot-starter-webmvc") implementation("org.springframework.boot:spring-boot-starter-data-jpa") implementation("org.springframework.boot:spring-boot-starter-flyway") implementation("org.springframework.boot:spring-boot-starter-batch") diff --git a/reconciler/src/main/resources/application.yaml b/reconciler/src/main/resources/application.yaml index 32595eb..a29a4a2 100644 --- a/reconciler/src/main/resources/application.yaml +++ b/reconciler/src/main/resources/application.yaml @@ -41,10 +41,17 @@ management: endpoints: web: exposure: - include: health,info,metrics + include: health,info,metrics,prometheus endpoint: health: show-details: when-authorized + metrics: + tags: + application: ${spring.application.name} + prometheus: + metrics: + export: + enabled: true openremit: reconcile: diff --git a/remittance-api/build.gradle.kts b/remittance-api/build.gradle.kts index c62eeb1..184c4c1 100644 --- a/remittance-api/build.gradle.kts +++ b/remittance-api/build.gradle.kts @@ -11,6 +11,8 @@ allOpen { dependencies { implementation("org.springframework.boot:spring-boot-starter-actuator") + implementation("io.micrometer:micrometer-registry-prometheus") + implementation("net.logstash.logback:logstash-logback-encoder:8.0") implementation("org.springframework.boot:spring-boot-starter-data-jpa") implementation("org.springframework.boot:spring-boot-starter-flyway") implementation("org.springframework.boot:spring-boot-starter-webmvc") diff --git a/remittance-api/src/main/resources/application.yaml b/remittance-api/src/main/resources/application.yaml index 20045f7..13de563 100644 --- a/remittance-api/src/main/resources/application.yaml +++ b/remittance-api/src/main/resources/application.yaml @@ -78,7 +78,14 @@ management: endpoints: web: exposure: - include: health,info,metrics + include: health,info,metrics,prometheus endpoint: health: show-details: when-authorized + metrics: + tags: + application: ${spring.application.name} + prometheus: + metrics: + export: + enabled: true diff --git a/webhook-dispatcher/build.gradle.kts b/webhook-dispatcher/build.gradle.kts index cdd5e67..ddbf190 100644 --- a/webhook-dispatcher/build.gradle.kts +++ b/webhook-dispatcher/build.gradle.kts @@ -13,6 +13,8 @@ dependencies { implementation(project(":common")) implementation("org.springframework.boot:spring-boot-starter-actuator") + implementation("io.micrometer:micrometer-registry-prometheus") + implementation("net.logstash.logback:logstash-logback-encoder:8.0") implementation("org.springframework.boot:spring-boot-starter-data-jpa") implementation("org.springframework.boot:spring-boot-starter-flyway") implementation("org.springframework.boot:spring-boot-starter-webmvc") diff --git a/webhook-dispatcher/src/main/resources/application.yaml b/webhook-dispatcher/src/main/resources/application.yaml index 194d746..3fb54a4 100644 --- a/webhook-dispatcher/src/main/resources/application.yaml +++ b/webhook-dispatcher/src/main/resources/application.yaml @@ -52,7 +52,14 @@ management: endpoints: web: exposure: - include: health,info,metrics + include: health,info,metrics,prometheus endpoint: health: show-details: when-authorized + metrics: + tags: + application: ${spring.application.name} + prometheus: + metrics: + export: + enabled: true