Skip to content

Commit 9b32349

Browse files
rajashish147rajashish147
committed
infra: add remaining observability config files
- infra/nginx/fieldtrack.conf: updated reverse proxy config - infra/prometheus/alerts.yml: Prometheus alerting rules - infra/promtail/promtail.yml: Promtail log scrape config for Loki - infra/tempo/tempo.yml: Grafana Tempo config (OTLP HTTP+gRPC receivers)
1 parent 9e25a76 commit 9b32349

4 files changed

Lines changed: 176 additions & 76 deletions

File tree

‎infra/nginx/fieldtrack.conf‎

Lines changed: 94 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,81 +1,90 @@
1-
# FieldTrack 2.0 — Nginx Reverse Proxy Configuration
2-
#
1+
# ============================================================================
2+
# FieldTrack 2.0 — Nginx Reverse Proxy
33
# Domain: fieldtrack.meowsician.tech
4-
#
5-
# SETUP:
6-
# 1. Symlink to /etc/nginx/sites-enabled/fieldtrack
7-
# 2. Run: sudo certbot --nginx -d fieldtrack.meowsician.tech
8-
# 3. Test: sudo nginx -t && sudo systemctl reload nginx
9-
#
10-
# DEPLOYMENT:
11-
# The deploy-bluegreen.sh script swaps the upstream server definition
12-
# between 127.0.0.1:3001 and 127.0.0.1:3002 using sed.
13-
# Only ONE line needs to change — the "server" directive in the upstream block.
14-
#
15-
# PUBLIC ROUTES:
16-
# / → root status JSON
17-
# /health → backend health check
18-
# /api/* → backend API (prefix stripped)
19-
# /monitor → Grafana dashboard
20-
#
21-
# BLOCKED (403):
22-
# /metrics, /internal/, /prometheus, /node-exporter
23-
24-
# ── Backend upstream ──────────────────────────────────────────────────────────
25-
# Single source of truth for the active backend container.
26-
# deploy-bluegreen.sh changes ONLY this line during deployment.
4+
# ============================================================================
5+
6+
7+
# ─────────────────────────────────────────────────────────────────────────────
8+
# Backend upstream (blue/green deployment target)
9+
# deploy-bluegreen.sh switches this port between 3001 and 3002
10+
# ─────────────────────────────────────────────────────────────────────────────
2711
upstream fieldtrack_backend {
2812
server 127.0.0.1:3001;
13+
keepalive 32;
2914
}
3015

31-
# ── Rate limiting zones ────────────────────────────────────────────────────────
16+
17+
# ─────────────────────────────────────────────────────────────────────────────
18+
# Rate limiting
19+
# ─────────────────────────────────────────────────────────────────────────────
3220
limit_req_zone $binary_remote_addr zone=fieldtrack_api:10m rate=30r/s;
3321
limit_req_zone $binary_remote_addr zone=fieldtrack_health:10m rate=5r/s;
3422

35-
# ── HTTP → HTTPS redirect ─────────────────────────────────────────────────────
23+
24+
# ─────────────────────────────────────────────────────────────────────────────
25+
# HTTP → HTTPS redirect
26+
# ─────────────────────────────────────────────────────────────────────────────
3627
server {
28+
3729
listen 80;
3830
listen [::]:80;
31+
3932
server_name fieldtrack.meowsician.tech;
4033

41-
# Allow ACME challenge for Certbot renewals
4234
location /.well-known/acme-challenge/ {
4335
root /var/www/certbot;
44-
allow all;
4536
}
4637

4738
location / {
4839
return 301 https://$host$request_uri;
4940
}
5041
}
5142

52-
# ── HTTPS server ───────────────────────────────────────────────────────────────
43+
44+
# ─────────────────────────────────────────────────────────────────────────────
45+
# HTTPS server
46+
# ─────────────────────────────────────────────────────────────────────────────
5347
server {
48+
5449
listen 443 ssl http2;
5550
listen [::]:443 ssl http2;
51+
5652
server_name fieldtrack.meowsician.tech;
5753

58-
# ── SSL (Certbot will manage these after first run) ────────────────────────
59-
# ssl_certificate /etc/letsencrypt/live/fieldtrack.meowsician.tech/fullchain.pem;
60-
# ssl_certificate_key /etc/letsencrypt/live/fieldtrack.meowsician.tech/privkey.pem;
61-
# include /etc/letsencrypt/options-ssl-nginx.conf;
62-
# ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
63-
64-
# ── Security headers ───────────────────────────────────────────────────────
65-
add_header X-Frame-Options "SAMEORIGIN" always;
66-
add_header X-Content-Type-Options "nosniff" always;
67-
add_header X-XSS-Protection "1; mode=block" always;
68-
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
54+
55+
# ── SSL (managed by Certbot) ─────────────────────────────────────────────
56+
ssl_certificate /etc/letsencrypt/live/fieldtrack.meowsician.tech/fullchain.pem;
57+
ssl_certificate_key /etc/letsencrypt/live/fieldtrack.meowsician.tech/privkey.pem;
58+
include /etc/letsencrypt/options-ssl-nginx.conf;
59+
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
60+
61+
62+
# ── Real client IP ───────────────────────────────────────────────────────
63+
real_ip_header X-Forwarded-For;
64+
set_real_ip_from 127.0.0.1;
65+
real_ip_recursive on;
66+
67+
68+
# ── Security headers ─────────────────────────────────────────────────────
69+
add_header X-Frame-Options "SAMEORIGIN" always;
70+
add_header X-Content-Type-Options "nosniff" always;
71+
add_header X-XSS-Protection "1; mode=block" always;
72+
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
6973
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
7074

71-
# ── Logging ────────────────────────────────────────────────────────────────
75+
76+
# ── Logging ──────────────────────────────────────────────────────────────
7277
access_log /var/log/nginx/fieldtrack_access.log;
7378
error_log /var/log/nginx/fieldtrack_error.log;
7479

75-
# ── Client limits ─────────────────────────────────────────────────────────
80+
81+
# ── Limits ───────────────────────────────────────────────────────────────
7682
client_max_body_size 10M;
7783

78-
# ── BLOCKED: Internal/monitoring endpoints (return 403) ─────────────────────
84+
85+
# ─────────────────────────────────────────────────────────────────────────
86+
# BLOCKED endpoints
87+
# ─────────────────────────────────────────────────────────────────────────
7988
location /metrics {
8089
return 403;
8190
}
@@ -92,81 +101,90 @@ server {
92101
return 403;
93102
}
94103

95-
# ── Root — bare domain status ──────────────────────────────────────────────
104+
105+
# ─────────────────────────────────────────────────────────────────────────
106+
# Root status
107+
# ─────────────────────────────────────────────────────────────────────────
96108
location = / {
97109
default_type application/json;
98110
return 200 '{"service":"FieldTrack 2.0","status":"online"}';
99111
}
100112

101-
# ── Health check (deployment verification + external monitors) ─────────────
113+
114+
# ─────────────────────────────────────────────────────────────────────────
115+
# Health endpoint
116+
# ─────────────────────────────────────────────────────────────────────────
102117
location = /health {
118+
103119
limit_req zone=fieldtrack_health burst=10 nodelay;
104120

105121
proxy_pass http://fieldtrack_backend;
106-
proxy_set_header Host $host;
107-
proxy_set_header X-Real-IP $remote_addr;
108-
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
122+
123+
proxy_set_header Host $host;
124+
proxy_set_header X-Real-IP $remote_addr;
125+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
109126
proxy_set_header X-Forwarded-Proto $scheme;
110127
}
111128

112-
# ── API route (production backend — strips /api prefix) ────────────────────
129+
130+
# ─────────────────────────────────────────────────────────────────────────
131+
# API
132+
# strips /api prefix
133+
# ─────────────────────────────────────────────────────────────────────────
113134
location /api/ {
135+
114136
limit_req zone=fieldtrack_api burst=50 nodelay;
115137

116-
# Strip /api prefix: /api/users → /users
117138
rewrite ^/api/(.*) /$1 break;
118139

119140
proxy_pass http://fieldtrack_backend;
120141

121-
# Standard proxy headers
122-
proxy_set_header Host $host;
123-
proxy_set_header X-Real-IP $remote_addr;
124-
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
142+
proxy_set_header Host $host;
143+
proxy_set_header X-Real-IP $remote_addr;
144+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
125145
proxy_set_header X-Forwarded-Proto $scheme;
126-
proxy_set_header X-Request-ID $request_id;
146+
proxy_set_header X-Request-ID $request_id;
127147

128-
# WebSocket support
129148
proxy_http_version 1.1;
130-
proxy_set_header Upgrade $http_upgrade;
149+
150+
proxy_set_header Upgrade $http_upgrade;
131151
proxy_set_header Connection "upgrade";
132152

133-
# Timeouts
134153
proxy_connect_timeout 10s;
135-
proxy_send_timeout 30s;
136-
proxy_read_timeout 30s;
154+
proxy_send_timeout 30s;
155+
proxy_read_timeout 30s;
137156

138-
# Buffering
139157
proxy_buffering on;
140158
proxy_buffer_size 4k;
141159
proxy_buffers 8 16k;
142160
}
143161

144-
# ── Grafana dashboard (/monitor) ───────────────────────────────────────────
145-
# Grafana runs with GF_SERVER_SERVE_FROM_SUB_PATH=true and
146-
# GF_SERVER_ROOT_URL=https://fieldtrack.meowsician.tech/monitor
147-
# so it handles the /monitor subpath internally. We proxy everything
148-
# under /monitor/ to Grafana's root — Grafana does the rest.
162+
163+
# ─────────────────────────────────────────────────────────────────────────
164+
# Grafana dashboard
165+
# ─────────────────────────────────────────────────────────────────────────
149166
location /monitor/ {
167+
150168
proxy_pass http://127.0.0.1:3333/monitor/;
151169

152-
proxy_set_header Host $host;
153-
proxy_set_header X-Real-IP $remote_addr;
154-
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
170+
proxy_set_header Host $host;
171+
proxy_set_header X-Real-IP $remote_addr;
172+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
155173
proxy_set_header X-Forwarded-Proto $scheme;
156174

157-
# WebSocket support (Grafana live features)
158175
proxy_http_version 1.1;
159-
proxy_set_header Upgrade $http_upgrade;
176+
177+
proxy_set_header Upgrade $http_upgrade;
160178
proxy_set_header Connection "upgrade";
161179

162-
# Longer timeouts for dashboard queries
163180
proxy_connect_timeout 10s;
164-
proxy_send_timeout 60s;
165-
proxy_read_timeout 60s;
181+
proxy_send_timeout 60s;
182+
proxy_read_timeout 60s;
166183
}
167184

168-
# Redirect /monitor (no trailing slash) → /monitor/
185+
169186
location = /monitor {
170187
return 301 $scheme://$host/monitor/;
171188
}
189+
172190
}

‎infra/prometheus/alerts.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
groups:
2+
- name: fieldtrack-backend-alerts
3+
rules:
4+
5+
- alert: BackendDown
6+
expr: sum(up{job=~"fieldtrack-backend.*"}) == 0
7+
for: 1m
8+
labels:
9+
severity: critical
10+
annotations:
11+
summary: "Backend unavailable"
12+
description: "All FieldTrack backend instances are down."
13+
14+
- alert: HighAPIErrorRate
15+
expr: rate(http_requests_total{status_code=~"5.."}[5m]) > 0.05
16+
for: 2m
17+
labels:
18+
severity: warning
19+
annotations:
20+
summary: "High API error rate"
21+
description: "More than 5% of requests are failing."
22+
23+
- alert: HighAPILatency
24+
expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 1
25+
for: 2m
26+
labels:
27+
severity: warning
28+
annotations:
29+
summary: "High API latency"
30+
description: "p95 latency above 1 second."
31+
32+
- alert: HighMemoryUsage
33+
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1
34+
for: 2m
35+
labels:
36+
severity: warning
37+
annotations:
38+
summary: "High memory usage"
39+
description: "System memory usage above 90%."
40+
41+
- alert: DiskAlmostFull
42+
expr: (node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) / node_filesystem_size_bytes{mountpoint="/"} > 0.9
43+
for: 5m
44+
labels:
45+
severity: warning
46+
annotations:
47+
summary: "Disk almost full"
48+
description: "Root filesystem usage above 90%."

‎infra/promtail/promtail.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
server:
2+
http_listen_port: 9080
3+
4+
positions:
5+
filename: /tmp/positions.yaml
6+
7+
clients:
8+
- url: http://loki:3100/loki/api/v1/push
9+
10+
scrape_configs:
11+
12+
- job_name: docker-containers
13+
14+
static_configs:
15+
- targets:
16+
- localhost
17+
labels:
18+
job: docker
19+
__path__: /var/lib/docker/containers/*/*.log

‎infra/tempo/tempo.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
server:
2+
http_listen_port: 3200
3+
4+
distributor:
5+
receivers:
6+
otlp:
7+
protocols:
8+
http:
9+
grpc:
10+
11+
storage:
12+
trace:
13+
backend: local
14+
local:
15+
path: /tmp/tempo/traces

0 commit comments

Comments
 (0)