From 59fa03efd34bb8e20ec0b85f5f6c91e5153dca78 Mon Sep 17 00:00:00 2001 From: or4dx <19908496+or4dx@users.noreply.github.com> Date: Sat, 19 Jul 2025 12:48:09 +0400 Subject: [PATCH 1/3] Added observability files for cost monitoring --- .github/workflows/dashboards.yml | 2 +- .../cloud-cost/cloud-cost-billing.json | 118 ++++++++++++++++ .../k8s-kube-cost/kube-cost.json | 133 ++++++++++++++++++ .../logging/grafana-stack/app.log | 64 +++++++++ .../clv_overview_bitblock_metrics.prom | 70 +++++++++ .../conversion_rate_metrics.prom | 27 ++++ .../cost_billing_metrics.prom | 74 ++++++++++ .../daily_active_users_metrics.prom | 19 +++ .../edu_content_bitblock_metrics.prom | 43 ++++++ .../market_perf_local_metrics.prom | 0 .../revenue_per_user_metrics.prom | 35 +++++ .../seven_day_retention_rate_metrics.prom | 36 +++++ .../user_journey_metrics.prom | 53 +++++++ .../metrics/grafana-stack/prometheus.yml | 92 ++++++++++++ .../alert_summary_mttr_metrics.prom | 41 ++++++ .../app_observability_metrics.prom | 46 ++++++ .../cluster_overview_metrics.prom | 58 ++++++++ .../container_insights_metrics.prom | 47 +++++++ .../control_plane_health_metrics.prom | 25 ++++ .../external_api_metrics.prom | 36 +++++ .../incident_response_timeline_metrics.prom | 60 ++++++++ .../infra_capacity_planning_metrics.prom | 77 ++++++++++ .../k8s_scheduler_health_metrics.prom | 33 +++++ .../logging_overview_metrics.prom | 0 .../sec_compliance_prometheus_metrics.prom | 54 +++++++ .../technical_metrics/sli_slo_metrics.prom | 20 +++ .../tracing_summary_metrics.prom | 48 +++++++ 27 files changed, 1310 insertions(+), 1 deletion(-) create mode 100644 bit-blocks-starter/observability/cost-monitoring/cloud-cost/cloud-cost-billing.json create mode 100644 bit-blocks-starter/observability/cost-monitoring/k8s-kube-cost/kube-cost.json create mode 100644 bit-blocks-starter/observability/logging/grafana-stack/app.log create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/clv_overview_bitblock_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/conversion_rate_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/cost_billing_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/daily_active_users_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/edu_content_bitblock_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/market_perf_local_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/revenue_per_user_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/seven_day_retention_rate_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/user_journey_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/prometheus.yml create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/alert_summary_mttr_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/app_observability_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/cluster_overview_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/container_insights_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/control_plane_health_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/external_api_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/incident_response_timeline_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/infra_capacity_planning_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/k8s_scheduler_health_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/logging_overview_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/sec_compliance_prometheus_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/sli_slo_metrics.prom create mode 100644 bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/tracing_summary_metrics.prom diff --git a/.github/workflows/dashboards.yml b/.github/workflows/dashboards.yml index 0ac2d2c..1ad253e 100644 --- a/.github/workflows/dashboards.yml +++ b/.github/workflows/dashboards.yml @@ -1,4 +1,4 @@ -name: CI – Dashboards QA +name: CI Dashboards QA on: push: diff --git a/bit-blocks-starter/observability/cost-monitoring/cloud-cost/cloud-cost-billing.json b/bit-blocks-starter/observability/cost-monitoring/cloud-cost/cloud-cost-billing.json new file mode 100644 index 0000000..373f2d1 --- /dev/null +++ b/bit-blocks-starter/observability/cost-monitoring/cloud-cost/cloud-cost-billing.json @@ -0,0 +1,118 @@ +{ + "id": null, + "uid": "cloud-cost-billing", + "title": "Cloud Cost & Billing Analytics", + "tags": ["cost", "cloud", "billing", "analytics"], + "timezone": "browser", + "schemaVersion": 36, + "version": 1, + "refresh": "1h", + "style": "dark", + "editable": true, + "panels": [ + { + "type": "stat", + "title": "Total Cloud Spend (This Month)", + "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 }, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "sum(cloud_cost_total{period='month'})", + "legendFormat": "Total" + } + ], + "fieldConfig": { "defaults": { "unit": "currency", "decimals": 2 } } + }, + { + "type": "stat", + "title": "Total Cloud Spend (This Week)", + "gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 }, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "sum(cloud_cost_total{period='week'})", + "legendFormat": "Total" + } + ], + "fieldConfig": { "defaults": { "unit": "currency", "decimals": 2 } } + }, + { + "type": "bar", + "title": "Cloud Spend by Provider", + "gridPos": { "x": 0, "y": 4, "w": 12, "h": 6 }, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "sum by(provider) (cloud_cost_total{period='month'})", + "legendFormat": "{{provider}}" + } + ], + "fieldConfig": { "defaults": { "unit": "currency", "decimals": 2 } } + }, + { + "type": "table", + "title": "Top 5 Services by Cost", + "gridPos": { "x": 0, "y": 10, "w": 12, "h": 7 }, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "topk(5, sum by(service) (cloud_cost_service_total{period='month'}))", + "legendFormat": "{{service}}" + } + ], + "fieldConfig": { "defaults": { "unit": "currency", "decimals": 2 } } + }, + { + "type": "timeseries", + "title": "Cost Trend (Last 30 Days)", + "gridPos": { "x": 0, "y": 17, "w": 12, "h": 6 }, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "sum by(date) (cloud_cost_daily{provider=~'.*'})", + "legendFormat": "Daily Spend" + } + ], + "fieldConfig": { "defaults": { "unit": "currency", "decimals": 2 } } + }, + { + "type": "stat", + "title": "Cost Forecast (Next 30 Days)", + "gridPos": { "x": 0, "y": 23, "w": 6, "h": 4 }, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "predict_linear(cloud_cost_daily[30d], 30*24*60*60)", + "legendFormat": "Forecast" + } + ], + "fieldConfig": { "defaults": { "unit": "currency", "decimals": 2 } } + }, + { + "type": "stat", + "title": "Budget/Threshold Alert", + "gridPos": { "x": 6, "y": 23, "w": 6, "h": 4 }, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "max_over_time(cloud_cost_total{period='month'}[30d]) > on() budget_limit", + "legendFormat": "Over Budget" + } + ], + "fieldConfig": { "defaults": { "unit": "none" } } + }, + { + "type": "stat", + "title": "Anomaly Detector", + "gridPos": { "x": 0, "y": 27, "w": 12, "h": 4 }, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "sum_over_time(cloud_cost_anomaly[1d])", + "legendFormat": "Detected Anomalies" + } + ], + "fieldConfig": { "defaults": { "unit": "none" } } + } + ] +} diff --git a/bit-blocks-starter/observability/cost-monitoring/k8s-kube-cost/kube-cost.json b/bit-blocks-starter/observability/cost-monitoring/k8s-kube-cost/kube-cost.json new file mode 100644 index 0000000..9acc9d3 --- /dev/null +++ b/bit-blocks-starter/observability/cost-monitoring/k8s-kube-cost/kube-cost.json @@ -0,0 +1,133 @@ +{ + "id": null, + "uid": "k8s-cost-analytics", + "title": "Kubernetes Cost Analytics", + "tags": ["kubernetes", "cost", "kubecost", "cloud-native"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "1m", + "style": "dark", + "editable": true, + "panels": [ + { + "type": "stat", + "title": "Total K8s Spend (Last 30d)", + "id": 1, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "sum(kubecost_cluster_cost_total)", + "legendFormat": "Total Cost", + "interval": "" + } + ], + "gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 } + }, + { + "type": "bar", + "title": "K8s Cost by Namespace", + "id": 2, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "sum by (namespace) (kubecost_namespace_cost_total)", + "legendFormat": "{{namespace}}" + } + ], + "gridPos": { "x": 4, "y": 0, "w": 8, "h": 8 } + }, + { + "type": "bar", + "title": "Cost per Deployment", + "id": 3, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "sum by (deployment) (kubecost_pod_cost_total)", + "legendFormat": "{{deployment}}" + } + ], + "gridPos": { "x": 0, "y": 4, "w": 8, "h": 8 } + }, + { + "type": "stat", + "title": "Resource Efficiency ($/CPU)", + "id": 4, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "sum(kubecost_cluster_cost_total) / sum(kube_pod_container_resource_requests_cpu_cores)", + "legendFormat": "Cost per CPU" + } + ], + "gridPos": { "x": 8, "y": 8, "w": 4, "h": 4 } + }, + { + "type": "stat", + "title": "Resource Efficiency ($/RAM GB)", + "id": 5, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "sum(kubecost_cluster_cost_total) / (sum(kube_pod_container_resource_requests_memory_bytes) / 1073741824)", + "legendFormat": "Cost per GB RAM" + } + ], + "gridPos": { "x": 8, "y": 12, "w": 4, "h": 4 } + }, + { + "type": "stat", + "title": "Savings Opportunities", + "id": 6, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "sum(kubecost_savings_opportunity_total)", + "legendFormat": "Potential Savings" + } + ], + "gridPos": { "x": 0, "y": 12, "w": 4, "h": 4 } + }, + { + "type": "bar", + "title": "Top 5 Expensive Namespaces", + "id": 7, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "topk(5, sum by (namespace) (kubecost_namespace_cost_total))", + "legendFormat": "{{namespace}}" + } + ], + "gridPos": { "x": 4, "y": 12, "w": 4, "h": 4 } + }, + { + "type": "timeseries", + "title": "Cost Trend (Last 30 Days)", + "id": 8, + "datasource": "${DS_PROMETHEUS}", + "targets": [ + { + "expr": "sum(kubecost_cluster_cost_total)", + "legendFormat": "Total Cost" + } + ], + "gridPos": { "x": 0, "y": 16, "w": 8, "h": 8 } + }, + { + "type": "stat", + "title": "Anomaly Alerts (Cost Spikes)", + "id": 9, + "datasource": "${DS_PROMETHEUS}", + "description": "Alerts if cost jumps 50%+ over previous day", + "targets": [ + { + "expr": "increase(kubecost_cluster_cost_total[1d]) > (sum(kubecost_cluster_cost_total offset 1d) * 0.5)", + "legendFormat": "Anomaly Detected" + } + ], + "gridPos": { "x": 8, "y": 16, "w": 4, "h": 4 } + } + ] +} diff --git a/bit-blocks-starter/observability/logging/grafana-stack/app.log b/bit-blocks-starter/observability/logging/grafana-stack/app.log new file mode 100644 index 0000000..f7afc28 --- /dev/null +++ b/bit-blocks-starter/observability/logging/grafana-stack/app.log @@ -0,0 +1,64 @@ +2025-07-11T00:00:00Z [INFO] main.go:45 Application starting up +2025-07-11T00:00:10Z [INFO] handler.go:88 Processing request id=abcd1234 +2025-07-11T00:00:20Z [WARN] cache.go:52 Cache miss for key=user:42 +2025-07-11T00:00:25Z [ERROR] db.go:120 Failed to connect to database +2025-07-11T00:00:30Z [INFO] handler.go:94 Request complete id=abcd1234 +2025-07-11T00:01:00Z [ERROR] handler.go:100 Timeout while calling downstream service +2025-07-11T00:01:15Z [WARN] auth.go:30 Token expired for user=joe +2025-07-11T00:01:45Z [INFO] scheduler.go:75 Scheduled job=cleanup +2025-07-11T00:02:00Z [INFO] main.go:50 Application health check passed +2025-07-11T00:02:10Z [INFO] handler.go:88 Processing request id=efgh5678 +2025-07-11T00:02:20Z [WARN] cache.go:52 Cache miss for key=user:43 +2025-07-11T00:02:25Z [ERROR] db.go:120 Failed to execute query +2025-07-11T00:02:30Z [INFO] handler.go:94 Request complete id=efgh5678 +2025-07-11T00:03:00Z [ERROR] handler.go:100 Timeout while fetching data +2025-07-11T00:03:15Z [WARN] auth.go:30 Token expired for user=alice +2025-07-11T00:03:45Z [INFO] scheduler.go:75 Scheduled job=backup +2025-07-11T00:04:00Z [INFO] main.go:50 Application health check passed +2025-07-11T00:04:10Z [INFO] handler.go:88 Processing request id=ijkl9012 +2025-07-11T00:04:20Z [WARN] cache.go:52 Cache miss for key=user:44 +2025-07-11T00:04:25Z [ERROR] db.go:120 Failed to update record +2025-07-11T00:04:30Z [INFO] handler.go:94 Request complete id=ijkl9012 +2025-07-11T00:05:00Z [ERROR] handler.go:100 Timeout while calling external API +2025-07-11T00:05:15Z [WARN] auth.go:30 Token expired for user=bob +2025-07-11T00:05:45Z [INFO] scheduler.go:75 Scheduled job=cleanup +2025-07-11T00:06:00Z [INFO] main.go:50 Application health check passed +2025-07-11T00:06:10Z [INFO] handler.go:88 Processing request id=mnop3456 +2025-07-11T00:06:20Z [WARN] cache.go:52 Cache miss for key=user:45 +2025-07-11T00:06:25Z [ERROR] db.go:120 Failed to delete record +2025-07-11T00:06:30Z [INFO] handler.go:94 Request complete id=mnop3456 +2025-07-11T00:07:00Z [ERROR] handler.go:100 Timeout while retrieving data +2025-07-11T00:07:15Z [WARN] auth.go:30 Token expired for user=charlie +2025-07-11T00:07:45Z [INFO] scheduler.go:75 Scheduled job=backup +2025-07-11T00:08:00Z [INFO] main.go:50 Application health check passed +2025-07-11T00:08:10Z [INFO] handler.go:88 Processing request id=qrst7890 +2025-07-11T00:08:20Z [WARN] cache.go:52 Cache miss for key=user:46 +2025-07-11T00:08:25Z [ERROR] db.go:120 Failed to fetch record +2025-07-11T00:08:30Z [INFO] handler.go:94 Request complete id=qrst7890 +2025-07-11T00:09:00Z [ERROR] handler.go:100 Timeout while connecting to service +2025-07-11T00:09:15Z [WARN] auth.go:30 Token expired for user=dave +2025-07-11T00:09:45Z [INFO] scheduler.go:75 Scheduled job=cleanup +2025-07-11T00:10:00Z [INFO] main.go:50 Application health check passed +2025-07-11T00:10:10Z [INFO] handler.go:88 Processing request id=uvwx1234 +2025-07-11T00:10:20Z [WARN] cache.go:52 Cache miss for key=user:47 +2025-07-11T00:10:25Z [ERROR] db.go:120 Failed to insert record +2025-07-11T00:10:30Z [INFO] handler.go:94 Request complete id=uvwx1234 +2025-07-11T00:11:00Z [ERROR] handler.go:100 Timeout while processing request +2025-07-11T00:11:15Z [WARN] auth.go:30 Token expired for user=emma +2025-07-11T00:11:45Z [INFO] scheduler.go:75 Scheduled job=backup +2025-07-11T00:12:00Z [INFO] main.go:50 Application health check passed +2025-07-11T00:12:10Z [INFO] handler.go:88 Processing request id=yzab5678 +2025-07-11T00:12:20Z [WARN] cache.go:52 Cache miss for key=user:48 +2025-07-11T00:12:25Z [ERROR] db.go:120 Failed to read record +2025-07-11T00:12:30Z [INFO] handler.go:94 Request complete id=yzab5678 +2025-07-11T00:13:00Z [ERROR] handler.go:100 Timeout while accessing resource +2025-07-11T00:13:15Z [WARN] auth.go:30 Token expired for user=frank +2025-07-11T00:13:45Z [INFO] scheduler.go:75 Scheduled job=cleanup +2025-07-11T00:14:00Z [INFO] main.go:50 Application health check passed +2025-07-11T00:14:10Z [INFO] handler.go:88 Processing request id=cdef9012 +2025-07-11T00:14:20Z [WARN] cache.go:52 Cache miss for key=user:49 +2025-07-11T00:14:25Z [ERROR] db.go:120 Failed to update record +2025-07-11T00:14:30Z [INFO] handler.go:94 Request complete id=cdef9012 +2025-07-11T00:15:00Z [ERROR] handler.go:100 Timeout while fetching data +2025-07-11T00:15:15Z [WARN] auth.go:30 Token expired for user=george +2025-07-11T00:15:45Z [INFO] scheduler.go:75 Scheduled job=backup \ No newline at end of file diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/clv_overview_bitblock_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/clv_overview_bitblock_metrics.prom new file mode 100644 index 0000000..91ef052 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/clv_overview_bitblock_metrics.prom @@ -0,0 +1,70 @@ +# Sample Prometheus metrics for Customer Lifetime Value (CLV) dashboard +# Generated to match dashboard queries + +# HELP revenue_total Total revenue over time +# TYPE revenue_total gauge +revenue_total{job="clv_app"} 12500.50 +revenue_total{job="clv_app"} 13075.75 + +# HELP users_active Number of active users over time +# TYPE users_active gauge +users_active{job="clv_app"} 250 +users_active{job="clv_app"} 260 + +# HELP marketing_cost_total Total marketing cost accumulated over 30 days +# TYPE marketing_cost_total gauge +marketing_cost_total{job="clv_app"} 4200.00 +marketing_cost_total{job="clv_app"} 4300.00 + +# HELP actual_clv Observed Customer Lifetime Value +# TYPE actual_clv gauge +actual_clv{job="clv_app"} 150.25 +actual_clv{job="clv_app"} 152.75 + +# HELP forecast_clv Forecasted Customer Lifetime Value +# TYPE forecast_clv gauge +forecast_clv{job="clv_app"} 160.00 +forecast_clv{job="clv_app"} 162.50 + +# HELP churn_total Total number of churned users +# TYPE churn_total counter +churn_total{job="clv_app"} 5 +churn_total{job="clv_app"} 7 + +# HELP user_revenue_bucket Revenue distribution histogram buckets per user +# TYPE user_revenue_bucket histogram +user_revenue_bucket{job="clv_app",le="50"} 150 +user_revenue_bucket{job="clv_app",le="100"} 300 +user_revenue_bucket{job="clv_app",le="200"} 50 +user_revenue_bucket{job="clv_app",le="+Inf"} 5 + +# HELP first_purchase_total Count of first purchases in last 30 days +# TYPE first_purchase_total counter +first_purchase_total{job="clv_app"} 120 +first_purchase_total{job="clv_app"} 130 + +# HELP second_purchase_total Count of second purchases in last 30 days +# TYPE second_purchase_total counter +second_purchase_total{job="clv_app"} 60 +second_purchase_total{job="clv_app"} 65 + +# HELP time_to_second_purchase_seconds_bucket Histogram of time to second purchase (seconds) +# TYPE time_to_second_purchase_seconds_bucket histogram +time_to_second_purchase_seconds_bucket{job="clv_app",le="86400"} 40 +time_to_second_purchase_seconds_bucket{job="clv_app",le="172800"} 55 +time_to_second_purchase_seconds_bucket{job="clv_app",le="+Inf"} 65 + +# HELP cohort_retention_percent Monthly cohort retention percentage +# TYPE cohort_retention_percent gauge +cohort_retention_percent{job="clv_app",cohort="2025-01",month="0"} 100 +cohort_retention_percent{job="clv_app",cohort="2025-01",month="1"} 75 +cohort_retention_percent{job="clv_app",cohort="2025-01",month="2"} 60 +cohort_retention_percent{job="clv_app",cohort="2025-02",month="0"} 100 +cohort_retention_percent{job="clv_app",cohort="2025-02",month="1"} 80 +cohort_retention_percent{job="clv_app",cohort="2025-02",month="2"} 65 + +# HELP revenue_by_segment Total revenue broken out by customer segment +# TYPE revenue_by_segment gauge +revenue_by_segment{job="clv_app",segment="enterprise"} 8000.00 +revenue_by_segment{job="clv_app",segment="mid_market"} 3500.50 +revenue_by_segment{job="clv_app",segment="small_business"} 1550.75 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/conversion_rate_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/conversion_rate_metrics.prom new file mode 100644 index 0000000..4dcef15 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/conversion_rate_metrics.prom @@ -0,0 +1,27 @@ +# conversion_rate_metrics.prom + +# HELP business_sessions_total Total number of user sessions per day. +# TYPE business_sessions_total counter +business_sessions_total{channel="organic",campaign="spring_sale"} 12000 +business_sessions_total{channel="organic",campaign="summer_sale"} 8000 +business_sessions_total{channel="paid", campaign="spring_sale"} 15000 +business_sessions_total{channel="paid", campaign="summer_sale"} 10000 + +# HELP business_conversions_total Total number of conversion events per day. +# TYPE business_conversions_total counter +business_conversions_total{channel="organic",campaign="spring_sale"} 600 +business_conversions_total{channel="organic",campaign="summer_sale"} 400 +business_conversions_total{channel="paid", campaign="spring_sale"} 1200 +business_conversions_total{channel="paid", campaign="summer_sale"} 800 + +# Simulate next scrape (to give non-zero increase()) +business_sessions_total{channel="organic",campaign="spring_sale"} 12500 +business_sessions_total{channel="organic",campaign="summer_sale"} 9000 +business_sessions_total{channel="paid", campaign="spring_sale"} 15500 +business_sessions_total{channel="paid", campaign="summer_sale"} 11000 + +business_conversions_total{channel="organic",campaign="spring_sale"} 650 +business_conversions_total{channel="organic",campaign="summer_sale"} 450 +business_conversions_total{channel="paid", campaign="spring_sale"} 1300 +business_conversions_total{channel="paid", campaign="summer_sale"} 900 + diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/cost_billing_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/cost_billing_metrics.prom new file mode 100644 index 0000000..d9325b0 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/cost_billing_metrics.prom @@ -0,0 +1,74 @@ +# metrics.prom — sample Cost & Billing Analytics metrics + +# Total Spend Over Time +# HELP cost_total_spend Total cumulative spend in USD +# TYPE cost_total_spend gauge +cost_total_spend{cluster="cluster-01"} 12000 +cost_total_spend{cluster="cluster-01"} 13500 + +# Spend by Service +# HELP cost_service_spend Spend per service in USD +# TYPE cost_service_spend gauge +cost_service_spend{service="ec2",cluster="cluster-01"} 5000 +cost_service_spend{service="s3",cluster="cluster-01"} 3000 +cost_service_spend{service="rds",cluster="cluster-01"} 2500 +cost_service_spend{service="kubernetes",cluster="cluster-01"} 3000 + +# Daily Spend Heatmap +# HELP cost_daily_spend Daily total spend in USD +# TYPE cost_daily_spend gauge +cost_daily_spend{date="2025-07-01"} 400 +cost_daily_spend{date="2025-07-02"} 450 +cost_daily_spend{date="2025-07-03"} 500 +cost_daily_spend{date="2025-07-04"} 480 +cost_daily_spend{date="2025-07-05"} 520 + +# Budget vs Actual +# HELP cost_budget_used Amount of budget used so far in USD +# TYPE cost_budget_used gauge +cost_budget_used{month="2025-07"} 3000 +# HELP cost_budget_total Total monthly budget in USD +# TYPE cost_budget_total gauge +cost_budget_total{month="2025-07"} 5000 + +# Forecasted Spend +# HELP cost_forecasted_spend Forecasted end-of-month spend in USD +# TYPE cost_forecasted_spend gauge +cost_forecasted_spend{month="2025-07"} 5500 + +# Cost Anomaly Detection +# HELP cost_spike_anomaly Flag indicating a cost anomaly (1=true) +# TYPE cost_spike_anomaly gauge +cost_spike_anomaly{service="ec2",severity="high"} 1 +cost_spike_anomaly{service="s3",severity="low"} 0 + +# Cost by Environment/Tag +# HELP cost_env_spend Spend per environment in USD +# TYPE cost_env_spend gauge +cost_env_spend{env="prod"} 9000 +cost_env_spend{env="staging"} 3000 + +# Waste & Idle Resource Cost +# HELP cost_idle_resource Spend on idle or underutilized resources in USD +# TYPE cost_idle_resource gauge +cost_idle_resource{resource="unattached_volume"} 200 +cost_idle_resource{resource="idle_vm"} 150 + +# Top 10 Expensive Resources +# HELP cost_resource Spend per individual resource in USD +# TYPE cost_resource gauge +cost_resource{resource="vm-1234"} 1200 +cost_resource{resource="db-5678"} 800 +cost_resource{resource="bucket-9012"} 600 +# ... (add up to top 10 for testing) + +# Cost per Active User +# HELP cost_per_active_user Ratio of cost to active user ($ per MAU) +# TYPE cost_per_active_user gauge +cost_per_active_user 0.75 + +# Reserved vs On-Demand Usage +# HELP cost_instance_type_spend Spend by instance purchasing model in USD +# TYPE cost_instance_type_spend gauge +cost_instance_type_spend{type="reserved"} 7000 +cost_instance_type_spend{type="on_demand"} 5000 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/daily_active_users_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/daily_active_users_metrics.prom new file mode 100644 index 0000000..94c9ba0 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/daily_active_users_metrics.prom @@ -0,0 +1,19 @@ + +# File: dau_metrics.prom + +# HELP business_active_users_total Total number of active users per day. +# TYPE business_active_users_total counter +business_active_users_total{region="us-east",segment="free"} 5200 +business_active_users_total{region="us-east",segment="premium"} 2300 +business_active_users_total{region="us-west",segment="free"} 3100 +business_active_users_total{region="us-west",segment="premium"} 1400 +business_active_users_total{region="eu-central",segment="free"} 2700 +business_active_users_total{region="eu-central",segment="premium"} 1100 + +# (Optional) simulate a little growth throughout the day by adding higher counters later +business_active_users_total{region="us-east",segment="free"} 5500 +business_active_users_total{region="us-east",segment="premium"} 2500 +business_active_users_total{region="us-west",segment="free"} 3300 +business_active_users_total{region="us-west",segment="premium"} 1550 +business_active_users_total{region="eu-central",segment="free"} 2900 +business_active_users_total{region="eu-central",segment="premium"} 1250 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/edu_content_bitblock_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/edu_content_bitblock_metrics.prom new file mode 100644 index 0000000..b704a55 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/edu_content_bitblock_metrics.prom @@ -0,0 +1,43 @@ +# Educational Content Metrics + +# 1. Daily Active Learners (gauge) +edu_daily_active_learners 125 + +# 2. Content Completion Rate (gauge: fraction 0–1) +edu_content_completion_rate 0.72 + +# 3. Average Time Spent per Module (gauge in seconds) +edu_avg_time_spent_seconds{module="Intro to Kubernetes"} 420 +edu_avg_time_spent_seconds{module="Terraform Basics"} 585 +edu_avg_time_spent_seconds{module="Prometheus 101"} 350 + +# 4. Lesson View Counts (counter) +edu_lesson_view_total{lesson="Intro to Kubernetes"} 3421 +edu_lesson_view_total{lesson="Terraform Basics"} 2754 +edu_lesson_view_total{lesson="Prometheus 101"} 1987 +edu_lesson_view_total{lesson="Grafana Dashboards"} 1621 +edu_lesson_view_total{lesson="Go Concurrency"} 1450 +edu_lesson_view_total{lesson="Rust Ownership"} 1310 +edu_lesson_view_total{lesson="AWS IAM"} 1278 +edu_lesson_view_total{lesson="Azure DevOps"} 1189 +edu_lesson_view_total{lesson="Docker Deep Dive"} 1102 +edu_lesson_view_total{lesson="CI/CD Pipelines"} 1023 + +# 5. API Request Totals and Errors (counter) +# Used to compute error rate: rate(edu_api_request_errors_total / edu_api_request_total) +edu_api_request_total{endpoint="/content/video"} 15000 +edu_api_request_total{endpoint="/content/quiz"} 12000 +edu_api_request_errors_total{endpoint="/content/video",code="500"} 37 +edu_api_request_errors_total{endpoint="/content/quiz", code="503"} 22 + +# 6. CDN Latency (histogram) +edu_cdn_response_time_seconds_bucket{le="0.1"} 15000 +edu_cdn_response_time_seconds_bucket{le="0.3"} 23000 +edu_cdn_response_time_seconds_bucket{le="0.5"} 26000 +edu_cdn_response_time_seconds_bucket{le="1"} 28000 +edu_cdn_response_time_seconds_bucket{le="+Inf"} 30000 +edu_cdn_response_time_seconds_sum 3400 +edu_cdn_response_time_seconds_count 30000 + +# 7. Cache Hit Ratio (gauge fraction 0–1) +edu_cache_hit_ratio{cache="content_cache"} 0.87 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/market_perf_local_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/market_perf_local_metrics.prom new file mode 100644 index 0000000..e69de29 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/revenue_per_user_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/revenue_per_user_metrics.prom new file mode 100644 index 0000000..1bdb4e3 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/revenue_per_user_metrics.prom @@ -0,0 +1,35 @@ +# revenue_per_user_metrics.prom + +# HELP business_revenue_total Total revenue per day in USD. +# TYPE business_revenue_total counter +business_revenue_total{region="us-east",segment="free"} 10000 +business_revenue_total{region="us-east",segment="premium"} 20000 +business_revenue_total{region="us-west",segment="free"} 8000 +business_revenue_total{region="us-west",segment="premium"} 15000 +business_revenue_total{region="eu-central",segment="free"} 7000 +business_revenue_total{region="eu-central",segment="premium"} 12000 + +# Simulate next scrape to allow increase() over 1d +business_revenue_total{region="us-east",segment="free"} 11000 +business_revenue_total{region="us-east",segment="premium"} 22000 +business_revenue_total{region="us-west",segment="free"} 9000 +business_revenue_total{region="us-west",segment="premium"} 16000 +business_revenue_total{region="eu-central",segment="free"} 8000 +business_revenue_total{region="eu-central",segment="premium"} 13000 + +# HELP business_active_users_total Total active users per day. +# TYPE business_active_users_total counter +business_active_users_total{region="us-east",segment="free"} 5000 +business_active_users_total{region="us-east",segment="premium"} 2500 +business_active_users_total{region="us-west",segment="free"} 3000 +business_active_users_total{region="us-west",segment="premium"} 1500 +business_active_users_total{region="eu-central",segment="free"} 2800 +business_active_users_total{region="eu-central",segment="premium"} 1300 + +# Simulate next scrape to allow increase() over 1d +business_active_users_total{region="us-east",segment="free"} 5500 +business_active_users_total{region="us-east",segment="premium"} 2750 +business_active_users_total{region="us-west",segment="free"} 3300 +business_active_users_total{region="us-west",segment="premium"} 1650 +business_active_users_total{region="eu-central",segment="free"} 3080 +business_active_users_total{region="eu-central",segment="premium"} 1430 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/seven_day_retention_rate_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/seven_day_retention_rate_metrics.prom new file mode 100644 index 0000000..0ef5680 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/seven_day_retention_rate_metrics.prom @@ -0,0 +1,36 @@ +# retention_rate_metrics.prom + +# HELP business_new_users_total Total number of new users per day. +# TYPE business_new_users_total counter +business_new_users_total{region="us-east"} 100 +business_new_users_total{region="us-west"} 50 +business_new_users_total{region="eu-central"} 70 + +# Simulate next scrape (to get a non-zero increase over 1d) +business_new_users_total{region="us-east"} 200 +business_new_users_total{region="us-west"} 100 +business_new_users_total{region="eu-central"} 140 + +# HELP business_retained_users_7d_total Total number of users retained 7 days after signup. +# TYPE business_retained_users_7d_total counter +business_retained_users_7d_total{region="us-east"} 20 +business_retained_users_7d_total{region="us-west"} 10 +business_retained_users_7d_total{region="eu-central"} 14 + +# Simulate next scrape (to get a non-zero increase over 1d) +business_retained_users_7d_total{region="us-east"} 100 +business_retained_users_7d_total{region="us-west"} 50 +business_retained_users_7d_total{region="eu-central"} 84 + +# HELP business_retention_total Retention counts by cohort and days since registration. +# TYPE business_retention_total counter +business_retention_total{region="us-east",cohort_day="2025-06-28",days_since_registration="0"} 100 +business_retention_total{region="us-east",cohort_day="2025-06-28",days_since_registration="7"} 50 +business_retention_total{region="us-west",cohort_day="2025-06-28",days_since_registration="0"} 50 +business_retention_total{region="us-west",cohort_day="2025-06-28",days_since_registration="7"} 30 + +# Simulate next scrape (for increase over 1d) +business_retention_total{region="us-east",cohort_day="2025-06-28",days_since_registration="0"} 200 +business_retention_total{region="us-east",cohort_day="2025-06-28",days_since_registration="7"} 150 +business_retention_total{region="us-west",cohort_day="2025-06-28",days_since_registration="0"} 100 +business_retention_total{region="us-west",cohort_day="2025-06-28",days_since_registration="7"} 60 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/user_journey_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/user_journey_metrics.prom new file mode 100644 index 0000000..e505863 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/business/business_metrics/user_journey_metrics.prom @@ -0,0 +1,53 @@ +# user_journey_funnel.prom + +# --- Visit/Signup/Activation/Purchase Counters --- +# HELP user_journey_visits_total Total number of visits to the landing page +# TYPE user_journey_visits_total counter +user_journey_visits_total{app="demo-app",env="staging"} 1500 + +# HELP user_journey_signups_total Total number of user signups +# TYPE user_journey_signups_total counter +user_journey_signups_total{app="demo-app",env="staging"} 900 + +# HELP user_journey_activations_total Total number of user activations +# TYPE user_journey_activations_total counter +user_journey_activations_total{app="demo-app",env="staging"} 700 + +# HELP user_journey_purchases_total Total number of first-time purchases +# TYPE user_journey_purchases_total counter +user_journey_purchases_total{app="demo-app",env="staging"} 300 + +# --- Stage Counts Gauge (for bar chart) --- +# HELP user_journey_stage_count Number of users at each funnel stage +# TYPE user_journey_stage_count gauge +user_journey_stage_count{stage="Landing",app="demo-app"} 1500 +user_journey_stage_count{stage="Signup",app="demo-app"} 900 +user_journey_stage_count{stage="Activation",app="demo-app"} 700 +user_journey_stage_count{stage="Purchase",app="demo-app"} 300 + +# --- Conversion Time Histogram --- +# HELP user_conversion_duration_seconds Distribution of time (in seconds) from signup→activation +# TYPE user_conversion_duration_seconds histogram +user_conversion_duration_seconds_bucket{le="30"} 400 +user_conversion_duration_seconds_bucket{le="60"} 600 +user_conversion_duration_seconds_bucket{le="120"} 750 +user_conversion_duration_seconds_bucket{le="300"} 860 +user_conversion_duration_seconds_bucket{le="+Inf"} 900 +user_conversion_duration_seconds_sum 45000 +user_conversion_duration_seconds_count 900 + +# --- 7-Day Retention Cohorts --- +# HELP user_retention_rate Fraction of users retained on each day after signup +# TYPE user_retention_rate gauge +user_retention_rate{day="1",app="demo-app"} 0.80 +user_retention_rate{day="2",app="demo-app"} 0.65 +user_retention_rate{day="3",app="demo-app"} 0.55 +user_retention_rate{day="4",app="demo-app"} 0.50 +user_retention_rate{day="5",app="demo-app"} 0.45 +user_retention_rate{day="6",app="demo-app"} 0.42 +user_retention_rate{day="7",app="demo-app"} 0.40 + +# --- Form Abandonment Rate (gauge) --- +# HELP user_form_abandonment_ratio Ratio of users abandoning the signup form +# TYPE user_form_abandonment_ratio gauge +user_form_abandonment_ratio{form="signup",app="demo-app"} 0.25 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/prometheus.yml b/bit-blocks-starter/observability/metrics/grafana-stack/prometheus.yml new file mode 100644 index 0000000..55bd520 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/prometheus.yml @@ -0,0 +1,92 @@ +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +scrape_configs: + # … your existing jobs … + + - job_name: 'fake-cluster-overview' + metrics_path: 'technical/technical_metrics/cluster_overview_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + + - job_name: 'fake-container-insights' + metrics_path: 'technical/technical_metrics/container_insights_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + + - job_name: 'fake-tracing-summary' + metrics_path: 'technical/technical_metrics/tracing_summary_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + + - job_name: 'fake-k8s-scheduler' + metrics_path: 'technical/technical_metrics/k8s_scheduler_health_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + + - job_name: 'fake-control-plane' + metrics_path: 'technical/technical_metrics/control_plane_health_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + + - job_name: 'fake-external-api' + metrics_path: 'technical/technical_metrics/external_api_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + +#Business Metrics + - job_name: 'fake-clv-overview' + metrics_path: 'business/business_metrics/clv_overview_bitblock_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + + - job_name: 'fake-conversion-rate' + metrics_path: 'business/business_metrics/conversion_rate_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + + - job_name: 'fake-cost-billing' + metrics_path: 'business/business_metrics/cost_billing_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + + - job_name: 'fake-edu-content' + metrics_path: 'business/business_metrics/edu_content_bitblock_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + + - job_name: 'fake-/market-perf-local' + metrics_path: 'business/business_metrics/market_perf_local_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + + - job_name: 'fake-revenue-per-user' + metrics_path: 'business/business_metrics/revenue_per_user_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + + - job_name: 'fake-seven-day-retention-health' + metrics_path: 'business/business_metrics/seven_day_retention_rate_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' + + - job_name: 'fake-user-journey' + metrics_path: 'business/business_metrics/user_journey_metrics.prom' + static_configs: + - targets: ['localhost:8000'] + fallback_scrape_protocol: 'PrometheusText0.0.4' \ No newline at end of file diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/alert_summary_mttr_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/alert_summary_mttr_metrics.prom new file mode 100644 index 0000000..392dffe --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/alert_summary_mttr_metrics.prom @@ -0,0 +1,41 @@ +# sample_alert_metrics.prom + +# HELP alerts_total Total number of alerts triggered +# TYPE alerts_total gauge +alerts_total{service="auth",severity="critical"} 5 +alerts_total{service="auth",severity="warning"} 12 +alerts_total{service="payment",severity="critical"} 2 +alerts_total{service="payment",severity="warning"} 8 +alerts_total{service="db",severity="critical"} 1 +alerts_total{service="db",severity="warning"} 3 + +# HELP alerts_resolved_total Total number of alerts resolved +# TYPE alerts_resolved_total counter +alerts_resolved_total{service="auth",severity="critical"} 4 +alerts_resolved_total{service="auth",severity="warning"} 10 +alerts_resolved_total{service="payment",severity="critical"} 1 +alerts_resolved_total{service="payment",severity="warning"} 6 +alerts_resolved_total{service="db",severity="critical"} 1 +alerts_resolved_total{service="db",severity="warning"} 2 + +# HELP alerts_mttd_seconds Mean time to detect an alert +# TYPE alerts_mttd_seconds gauge +alerts_mttd_seconds 45.5 + +# HELP alerts_mtta_seconds Mean time to acknowledge an alert +# TYPE alerts_mtta_seconds gauge +alerts_mtta_seconds 30.2 + +# HELP alerts_mttr_seconds Mean time to resolve an alert +# TYPE alerts_mttr_seconds gauge +alerts_mttr_seconds 120.7 + +# HELP error_budget_remaining Remaining error budget fraction (0–1) +# TYPE error_budget_remaining gauge +error_budget_remaining 0.85 + +# HELP alerts_latest_incidents Dummy flag per latest incident +# TYPE alerts_latest_incidents gauge +alerts_latest_incidents{incident="INC-2025-001"} 1 +alerts_latest_incidents{incident="INC-2025-002"} 1 +alerts_latest_incidents{incident="INC-2025-003"} 1 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/app_observability_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/app_observability_metrics.prom new file mode 100644 index 0000000..9234692 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/app_observability_metrics.prom @@ -0,0 +1,46 @@ +# ──────────────────────────────────────────────────────────────── +# 1. HTTP request counts (for RPS & Error Rate) +# ──────────────────────────────────────────────────────────────── +# HELP http_requests_total Total HTTP requests received. +# TYPE http_requests_total counter +http_requests_total{method="GET",handler="/api/users",status="200"} 10234 +http_requests_total{method="POST",handler="/api/orders",status="200"} 5234 +http_requests_total{method="GET",handler="/api/products",status="500"} 23 + +# ──────────────────────────────────────────────────────────────── +# 2. HTTP latency histogram (for P95/P99 panels) +# ──────────────────────────────────────────────────────────────── +# HELP http_request_duration_seconds HTTP request latency histogram. +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{le="0.1"} 2400 +http_request_duration_seconds_bucket{le="0.3"} 3000 +http_request_duration_seconds_bucket{le="1"} 3200 +http_request_duration_seconds_bucket{le="+Inf"} 3250 +http_request_duration_seconds_sum 123.45 +http_request_duration_seconds_count 3250 + +# ──────────────────────────────────────────────────────────────── +# 3. CPU usage counter (for the CPU Usage stat) +# ──────────────────────────────────────────────────────────────── +# HELP process_cpu_seconds_total Total user+system CPU time spent by the app. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 12345.67 + +# ──────────────────────────────────────────────────────────────── +# 4. DB query latency histogram (DB p95 Latency) +# ──────────────────────────────────────────────────────────────── +# HELP db_query_duration_seconds DB query latency histogram. +# TYPE db_query_duration_seconds histogram +db_query_duration_seconds_bucket{le="0.01"} 500 +db_query_duration_seconds_bucket{le="0.05"} 550 +db_query_duration_seconds_bucket{le="0.1"} 580 +db_query_duration_seconds_bucket{le="+Inf"} 600 +db_query_duration_seconds_sum 5.5 +db_query_duration_seconds_count 600 + +# ──────────────────────────────────────────────────────────────── +# 5. External API errors (for External API Errors panel) +# ──────────────────────────────────────────────────────────────── +# HELP external_api_errors Counter of external API errors. +# TYPE external_api_errors counter +external_api_errors{api="payment_service"} 5 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/cluster_overview_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/cluster_overview_metrics.prom new file mode 100644 index 0000000..57469c2 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/cluster_overview_metrics.prom @@ -0,0 +1,58 @@ +# HELP node_cpu_seconds_total Total CPU seconds. +# TYPE node_cpu_seconds_total counter +node_cpu_seconds_total{mode="user",instance="instance1"} 100 +node_cpu_seconds_total{mode="system",instance="instance1"} 50 +node_cpu_seconds_total{mode="idle",instance="instance1"} 850 +node_cpu_seconds_total{mode="user",instance="instance2"} 120 +node_cpu_seconds_total{mode="system",instance="instance2"} 60 +node_cpu_seconds_total{mode="idle",instance="instance2"} 820 + +# HELP node_memory_Active_bytes Active memory in bytes. +# TYPE node_memory_Active_bytes gauge +node_memory_Active_bytes 4000000000 + +# HELP node_memory_MemTotal_bytes Total memory in bytes. +# TYPE node_memory_MemTotal_bytes gauge +node_memory_MemTotal_bytes 8000000000 + +# HELP node_filesystem_size_bytes Filesystem total size in bytes. +# TYPE node_filesystem_size_bytes gauge +node_filesystem_size_bytes{mountpoint="/",fstype="ext4"} 100000000000 +node_filesystem_size_bytes{mountpoint="/var",fstype="ext4"} 50000000000 + +# HELP node_filesystem_free_bytes Filesystem free bytes. +# TYPE node_filesystem_free_bytes gauge +node_filesystem_free_bytes{mountpoint="/",fstype="ext4"} 60000000000 +node_filesystem_free_bytes{mountpoint="/var",fstype="ext4"} 20000000000 + +# HELP node_network_receive_bytes_total Network receive bytes total. +# TYPE node_network_receive_bytes_total counter +node_network_receive_bytes_total{instance="instance1"} 1000000000 +node_network_receive_bytes_total{instance="instance2"} 2000000000 + +# HELP node_network_transmit_bytes_total Network transmit bytes total. +# TYPE node_network_transmit_bytes_total counter +node_network_transmit_bytes_total{instance="instance1"} 500000000 +node_network_transmit_bytes_total{instance="instance2"} 1000000000 + +# HELP kube_pod_info Kubernetes pod info (always 1 per pod). +# TYPE kube_pod_info gauge +kube_pod_info{namespace="default",pod="web-1"} 1 +kube_pod_info{namespace="default",pod="web-2"} 1 +kube_pod_info{namespace="kube-system",pod="kube-dns"} 1 + +# HELP kube_pod_container_status_restarts_total Total container restarts. +# TYPE kube_pod_container_status_restarts_total counter +kube_pod_container_status_restarts_total{namespace="default",pod="web-1",container="app"} 3 +kube_pod_container_status_restarts_total{namespace="default",pod="web-2",container="app"} 1 +kube_pod_container_status_restarts_total{namespace="kube-system",pod="kube-dns",container="dns"} 0 + +# HELP node_disk_read_bytes_total Disk read bytes total. +# TYPE node_disk_read_bytes_total counter +node_disk_read_bytes_total{instance="instance1"} 500000000 +node_disk_read_bytes_total{instance="instance2"} 1000000000 + +# HELP node_disk_written_bytes_total Disk written bytes total. +# TYPE node_disk_written_bytes_total counter +node_disk_written_bytes_total{instance="instance1"} 200000000 +node_disk_written_bytes_total{instance="instance2"} 400000000 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/container_insights_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/container_insights_metrics.prom new file mode 100644 index 0000000..10bbd97 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/container_insights_metrics.prom @@ -0,0 +1,47 @@ +# container_insights_metrics.prom + +# Restart counts +# HELP kube_pod_container_status_restarts_total Total container restarts. +# TYPE kube_pod_container_status_restarts_total counter +kube_pod_container_status_restarts_total{namespace="default",pod="web-1",container="app"} 3 +kube_pod_container_status_restarts_total{namespace="default",pod="web-2",container="app"} 1 +kube_pod_container_status_restarts_total{namespace="kube-system",pod="dns-1",container="dns"} 0 + +# OOM kills +# HELP kube_pod_container_status_last_terminated_reason Last termination reason count. +# TYPE kube_pod_container_status_last_terminated_reason gauge +kube_pod_container_status_last_terminated_reason{namespace="default",pod="web-1",container="app",reason="OOMKilled"} 5 +kube_pod_container_status_last_terminated_reason{namespace="default",pod="web-2",container="app",reason="OOMKilled"} 0 + +# CPU throttling +# HELP container_cpu_cfs_throttled_periods_total CFS throttled periods. +# TYPE container_cpu_cfs_throttled_periods_total counter +container_cpu_cfs_throttled_periods_total{namespace="default",pod="web-1",container="app"} 200 +container_cpu_cfs_throttled_periods_total{namespace="default",pod="web-2",container="app"} 50 + +# CPU periods +# HELP container_cpu_cfs_periods_total CFS periods. +# TYPE container_cpu_cfs_periods_total counter +container_cpu_cfs_periods_total{namespace="default",pod="web-1",container="app"} 10000 +container_cpu_cfs_periods_total{namespace="default",pod="web-2",container="app"} 10000 + +# Pod start times +# HELP kube_pod_start_time_seconds Pod start time in seconds since epoch. +# TYPE kube_pod_start_time_seconds gauge +kube_pod_start_time_seconds{namespace="default",pod="web-1"} 1698800000 +kube_pod_start_time_seconds{namespace="default",pod="web-2"} 1698800360 + +# Dependency status (up) +# HELP up Always 1 for up, 0 for down. +# TYPE up gauge +up{namespace="default",instance="web-1",job="app"} 1 +up{namespace="default",instance="web-2",job="app"} 0 + +# Throttling events (same as throttled periods rate) +# [Uses container_cpu_cfs_throttled_periods_total above] + +# Active TCP connections +# HELP container_network_tcp_connections Number of TCP connections by state. +# TYPE container_network_tcp_connections gauge +container_network_tcp_connections{namespace="default",pod="web-1",state="established"} 120 +container_network_tcp_connections{namespace="default",pod="web-2",state="established"} 80 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/control_plane_health_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/control_plane_health_metrics.prom new file mode 100644 index 0000000..08d67c5 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/control_plane_health_metrics.prom @@ -0,0 +1,25 @@ +# control_plane_health_metrics.prom + +# API Server request totals by status code +# HELP apiserver_request_total Total API server requests by response code. +# TYPE apiserver_request_total counter +apiserver_request_total{code="200",instance="apiserver-1"} 12000 +apiserver_request_total{code="500",instance="apiserver-1"} 120 +apiserver_request_total{code="503",instance="apiserver-1"} 30 +apiserver_request_total{code="404",instance="apiserver-1"} 100 +apiserver_request_total{code="200",instance="apiserver-2"} 15000 +apiserver_request_total{code="500",instance="apiserver-2"} 200 +apiserver_request_total{code="503",instance="apiserver-2"} 50 +apiserver_request_total{code="404",instance="apiserver-2"} 80 + +# etcd leader change counts +# HELP etcd_server_leader_changes_seen_total Total number of etcd leader changes observed. +# TYPE etcd_server_leader_changes_seen_total counter +etcd_server_leader_changes_seen_total{instance="etcd-1"} 2 +etcd_server_leader_changes_seen_total{instance="etcd-2"} 1 + +# etcd cluster membership (has leader) +# HELP etcd_server_has_leader Gauge indicating if this etcd instance has a leader (1 = yes, 0 = no). +# TYPE etcd_server_has_leader gauge +etcd_server_has_leader{instance="etcd-1"} 1 +etcd_server_has_leader{instance="etcd-2"} 1 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/external_api_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/external_api_metrics.prom new file mode 100644 index 0000000..5933d46 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/external_api_metrics.prom @@ -0,0 +1,36 @@ +# external_api_health_metrics.prom + +# Total external API requests by service and status +# HELP external_api_requests_total Total requests to external APIs. +# TYPE external_api_requests_total counter +external_api_requests_total{service="svcA",status="200"} 1200 +external_api_requests_total{service="svcA",status="500"} 30 +external_api_requests_total{service="svcA",status="404"} 20 + +external_api_requests_total{service="svcB",status="200"} 900 +external_api_requests_total{service="svcB",status="503"} 15 +external_api_requests_total{service="svcB",status="400"} 10 + +# Histogram of external API request durations (seconds) +# HELP external_api_request_duration_seconds_bucket External API request duration buckets. +# TYPE external_api_request_duration_seconds_bucket histogram +external_api_request_duration_seconds_bucket{le="0.05",service="svcA"} 400 +external_api_request_duration_seconds_bucket{le="0.1", service="svcA"} 800 +external_api_request_duration_seconds_bucket{le="0.3", service="svcA"} 1100 +external_api_request_duration_seconds_bucket{le="1", service="svcA"} 1180 +external_api_request_duration_seconds_bucket{le="+Inf",service="svcA"} 1250 + +external_api_request_duration_seconds_bucket{le="0.05",service="svcB"} 300 +external_api_request_duration_seconds_bucket{le="0.1", service="svcB"} 700 +external_api_request_duration_seconds_bucket{le="0.3", service="svcB"} 850 +external_api_request_duration_seconds_bucket{le="1", service="svcB"} 900 +external_api_request_duration_seconds_bucket{le="+Inf",service="svcB"} 920 + +# Count and sum for the histogram +# TYPE external_api_request_duration_seconds_count counter +external_api_request_duration_seconds_count{service="svcA"} 1250 +external_api_request_duration_seconds_count{service="svcB"} 920 + +# TYPE external_api_request_duration_seconds_sum counter +external_api_request_duration_seconds_sum{service="svcA"} 95.4 +external_api_request_duration_seconds_sum{service="svcB"} 80.7 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/incident_response_timeline_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/incident_response_timeline_metrics.prom new file mode 100644 index 0000000..63a03d8 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/incident_response_timeline_metrics.prom @@ -0,0 +1,60 @@ +# File: incident_response_metrics.prom +# ------------------------------- +# Incident events counter, labeled by severity and root cause +# ------------------------------- +# HELP incident_events_total Total number of incident events +# TYPE incident_events_total counter +incident_events_total{severity="P1",root_cause="code_bug"} 5 +incident_events_total{severity="P2",root_cause="infrastructure"} 12 +incident_events_total{severity="P3",root_cause="network"} 30 + +# ------------------------------- +# Acknowledgement latency (seconds) per incident +# ------------------------------- +# HELP incident_ack_time_seconds Time from alert firing to first acknowledgement +# TYPE incident_ack_time_seconds gauge +incident_ack_time_seconds{incident="INC-001"} 120 +incident_ack_time_seconds{incident="INC-002"} 90 +incident_ack_time_seconds{incident="INC-003"} 150 + +# ------------------------------- +# Resolution latency (seconds) per incident +# ------------------------------- +# HELP incident_resolution_time_seconds Time from incident creation to resolution +# TYPE incident_resolution_time_seconds gauge +incident_resolution_time_seconds{incident="INC-001"} 3600 +incident_resolution_time_seconds{incident="INC-002"} 2700 +incident_resolution_time_seconds{incident="INC-003"} 5400 + +# ------------------------------- +# Open vs. closed incident gauge counts +# ------------------------------- +# HELP incident_open_total Number of currently open incidents +# TYPE incident_open_total gauge +incident_open_total 3 +# HELP incident_closed_total Number of incidents closed in period +# TYPE incident_closed_total gauge +incident_closed_total 27 + +# ------------------------------- +# SLA compliance +# ------------------------------- +# HELP incident_sla_met_total Number of incidents resolved within SLA +# TYPE incident_sla_met_total gauge +incident_sla_met_total 22 + +# ------------------------------- +# On‐call assignments per engineer (counter) +# ------------------------------- +# HELP incident_assigned_total Total incident assignments per engineer +# TYPE incident_assigned_total counter +incident_assigned_total{engineer="alice"} 10 +incident_assigned_total{engineer="bob"} 8 +incident_assigned_total{engineer="carol"} 9 + +# ------------------------------- +# Raw alert volume +# ------------------------------- +# HELP alerts_total Total number of raw alerts fired +# TYPE alerts_total counter +alerts_total 100 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/infra_capacity_planning_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/infra_capacity_planning_metrics.prom new file mode 100644 index 0000000..c4d3795 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/infra_capacity_planning_metrics.prom @@ -0,0 +1,77 @@ +# Sample Prometheus metrics for Infrastructure Capacity Planning + +# CPU usage per container (in seconds) +# TYPE container_cpu_usage_seconds_total counter +container_cpu_usage_seconds_total{pod="pod1", instance="node1", cluster="cluster-A"} 15234.5 +container_cpu_usage_seconds_total{pod="pod2", instance="node1", cluster="cluster-A"} 18345.2 +container_cpu_usage_seconds_total{pod="pod3", instance="node2", cluster="cluster-A"} 11232.8 +container_cpu_usage_seconds_total{pod="pod4", instance="node2", cluster="cluster-A"} 20456.1 + +# Memory working set vs total machine memory (bytes) +# TYPE container_memory_working_set_bytes gauge +container_memory_working_set_bytes{pod="pod1", instance="node1", cluster="cluster-A"} 2147483648 +container_memory_working_set_bytes{pod="pod2", instance="node1", cluster="cluster-A"} 1073741824 +container_memory_working_set_bytes{pod="pod3", instance="node2", cluster="cluster-A"} 3221225472 +container_memory_working_set_bytes{pod="pod4", instance="node2", cluster="cluster-A"} 536870912 + +# TYPE machine_memory_bytes gauge +machine_memory_bytes{instance="node1", cluster="cluster-A"} 34359738368 +machine_memory_bytes{instance="node2", cluster="cluster-A"} 34359738368 + +# Filesystem usage (bytes) +# TYPE node_filesystem_size_bytes gauge +node_filesystem_size_bytes{instance="node1", fstype="ext4", cluster="cluster-A"} 107374182400 +node_filesystem_size_bytes{instance="node2", fstype="ext4", cluster="cluster-A"} 107374182400 + +# TYPE node_filesystem_free_bytes gauge +node_filesystem_free_bytes{instance="node1", fstype="ext4", cluster="cluster-A"} 53687091200 +node_filesystem_free_bytes{instance="node2", fstype="ext4", cluster="cluster-A"} 64424509440 + +# Disk I/O operations +# TYPE node_disk_reads_completed_total counter +node_disk_reads_completed_total{instance="node1", cluster="cluster-A"} 123456 +node_disk_reads_completed_total{instance="node2", cluster="cluster-A"} 234567 + +# TYPE node_disk_writes_completed_total counter +node_disk_writes_completed_total{instance="node1", cluster="cluster-A"} 654321 +node_disk_writes_completed_total{instance="node2", cluster="cluster-A"} 543210 + +# Network throughput (bytes) +# TYPE node_network_receive_bytes_total counter +node_network_receive_bytes_total{instance="node1", cluster="cluster-A"} 987654321 +node_network_receive_bytes_total{instance="node2", cluster="cluster-A"} 876543219 + +# TYPE node_network_transmit_bytes_total counter +node_network_transmit_bytes_total{instance="node1", cluster="cluster-A"} 123456789 +node_network_transmit_bytes_total{instance="node2", cluster="cluster-A"} 234567890 + +# Pod CPU requests vs actual usage +# TYPE kube_pod_container_resource_requests_cpu_cores gauge +kube_pod_container_resource_requests_cpu_cores{pod="pod1", cluster="cluster-A"} 0.50 +kube_pod_container_resource_requests_cpu_cores{pod="pod2", cluster="cluster-A"} 0.75 +kube_pod_container_resource_requests_cpu_cores{pod="pod3", cluster="cluster-A"} 0.25 +kube_pod_container_resource_requests_cpu_cores{pod="pod4", cluster="cluster-A"} 1.00 + +# Node info and autoscaler events +# TYPE kube_node_info gauge +kube_node_info{node="node1", cluster="cluster-A"} 1 +kube_node_info{node="node2", cluster="cluster-A"} 1 + +# TYPE cluster_autoscaler_node_scale_up_events_total counter +cluster_autoscaler_node_scale_up_events_total{cluster="cluster-A"} 3 + +# Pod scheduling density +# TYPE kube_pod_info gauge +kube_pod_info{pod="pod1", node="node1", cluster="cluster-A"} 1 +kube_pod_info{pod="pod2", node="node1", cluster="cluster-A"} 1 +kube_pod_info{pod="pod3", node="node2", cluster="cluster-A"} 1 +kube_pod_info{pod="pod4", node="node2", cluster="cluster-A"} 1 + +# Node readiness +# TYPE kube_node_status_condition gauge +kube_node_status_condition{node="node1", condition="Ready", status="true", cluster="cluster-A"} 1 +kube_node_status_condition{node="node2", condition="Ready", status="true", cluster="cluster-A"} 1 + +# Cost per vCPU (USD) +# TYPE cloud_cost_per_vcpu_usd gauge +cloud_cost_per_vcpu_usd{cluster="cluster-A"} 0.05 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/k8s_scheduler_health_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/k8s_scheduler_health_metrics.prom new file mode 100644 index 0000000..cd79ba7 --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/k8s_scheduler_health_metrics.prom @@ -0,0 +1,33 @@ +# k8s_scheduler_health_metrics.prom + +# Histogram of pod scheduling latency (seconds) +# HELP kube_pod_schedulable_duration_seconds_bucket Buckets of pod scheduling latency +# TYPE kube_pod_schedulable_duration_seconds_bucket histogram +kube_pod_schedulable_duration_seconds_bucket{le="0.01",namespace="default"} 50 +kube_pod_schedulable_duration_seconds_bucket{le="0.1",namespace="default"} 80 +kube_pod_schedulable_duration_seconds_bucket{le="1",namespace="default"} 90 +kube_pod_schedulable_duration_seconds_bucket{le="+Inf",namespace="default"} 100 + +kube_pod_schedulable_duration_seconds_bucket{le="0.01",namespace="kube-system"} 30 +kube_pod_schedulable_duration_seconds_bucket{le="0.1",namespace="kube-system"} 60 +kube_pod_schedulable_duration_seconds_bucket{le="1",namespace="kube-system"} 75 +kube_pod_schedulable_duration_seconds_bucket{le="+Inf",namespace="kube-system"} 80 + +# (Optional) count & sum for histogram +# TYPE kube_pod_schedulable_duration_seconds_count counter +kube_pod_schedulable_duration_seconds_count{namespace="default"} 100 +kube_pod_schedulable_duration_seconds_count{namespace="kube-system"} 80 +# TYPE kube_pod_schedulable_duration_seconds_sum counter +kube_pod_schedulable_duration_seconds_sum{namespace="default"} 2.5 +kube_pod_schedulable_duration_seconds_sum{namespace="kube-system"} 1.8 + +# Pod evictions (terminated reason = Evicted) +# HELP kube_pod_container_status_terminated_reason Total pod terminations by reason +# TYPE kube_pod_container_status_terminated_reason counter +kube_pod_container_status_terminated_reason{reason="Evicted",namespace="default",pod="web-1",node="node1"} 5 +kube_pod_container_status_terminated_reason{reason="Evicted",namespace="default",pod="web-2",node="node1"} 2 +kube_pod_container_status_terminated_reason{reason="Evicted",namespace="kube-system",pod="dns-1",node="node2"} 1 + +# Other terminations (ignored by our queries) +kube_pod_container_status_terminated_reason{reason="OOMKilled",namespace="default",pod="web-1",node="node1"} 1 + diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/logging_overview_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/logging_overview_metrics.prom new file mode 100644 index 0000000..e69de29 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/sec_compliance_prometheus_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/sec_compliance_prometheus_metrics.prom new file mode 100644 index 0000000..db13beb --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/sec_compliance_prometheus_metrics.prom @@ -0,0 +1,54 @@ +# HELP vulnerabilities_total Total number of vulnerabilities by severity +# TYPE vulnerabilities_total gauge +vulnerabilities_total{severity="critical"} 5 +vulnerabilities_total{severity="high"} 12 +vulnerabilities_total{severity="medium"} 30 +vulnerabilities_total{severity="low"} 100 + +# HELP patch_compliance_percent Percentage of nodes with up-to-date patches +# TYPE patch_compliance_percent gauge +patch_compliance_percent 85 + +# HELP auth_success_total Total successful authentication events +# TYPE auth_success_total counter +auth_success_total 10567 + +# HELP auth_fail_total Total failed authentication events +# TYPE auth_fail_total counter +auth_fail_total 125 + +# HELP role_changes_total Total privilege change events +# TYPE role_changes_total counter +role_changes_total 8 + +# HELP net_conn_count Number of network connections by port +# TYPE net_conn_count gauge +net_conn_count{port="22"} 150 +net_conn_count{port="3389"} 5 +net_conn_count{port="8080"} 350 + +# HELP tls_cert_expiry_days Days until TLS certificate expiry +# TYPE tls_cert_expiry_days gauge +tls_cert_expiry_days 14 + +# HELP intrusion_alerts_total Number of intrusion/malware alerts per host +# TYPE intrusion_alerts_total gauge +intrusion_alerts_total{host="host1"} 2 +intrusion_alerts_total{host="host2"} 0 +intrusion_alerts_total{host="host3"} 5 + +# HELP log_ingest_rate Log ingestion rate per second +# TYPE log_ingest_rate gauge +log_ingest_rate 250 + +# HELP policy_violations_total Total number of policy violations +# TYPE policy_violations_total gauge +policy_violations_total 7 + +# HELP mttd_seconds Mean time to detect in seconds +# TYPE mttd_seconds gauge +mttd_seconds 360 + +# HELP mttr_seconds Mean time to remediate in seconds +# TYPE mttr_seconds gauge +mttr_seconds 180 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/sli_slo_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/sli_slo_metrics.prom new file mode 100644 index 0000000..16b447b --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/sli_slo_metrics.prom @@ -0,0 +1,20 @@ +# error_budget_metrics.prom + +# Total HTTP requests by status +# TYPE http_requests_total counter +http_requests_total{status="200"} 9500 +http_requests_total{status="404"} 300 +http_requests_total{status="500"} 200 + +# Histogram of HTTP request durations (in seconds) +# TYPE http_request_duration_seconds_bucket histogram +http_request_duration_seconds_bucket{le="0.1"} 8000 +http_request_duration_seconds_bucket{le="0.5"} 9500 +http_request_duration_seconds_bucket{le="1"} 9800 +http_request_duration_seconds_bucket{le="+Inf"} 10000 + +# Total count and sum for the duration histogram +# TYPE http_request_duration_seconds_count counter +http_request_duration_seconds_count 10000 +# TYPE http_request_duration_seconds_sum counter +http_request_duration_seconds_sum 500.0 diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/tracing_summary_metrics.prom b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/tracing_summary_metrics.prom new file mode 100644 index 0000000..c58931b --- /dev/null +++ b/bit-blocks-starter/observability/metrics/grafana-stack/technical/technical_metrics/tracing_summary_metrics.prom @@ -0,0 +1,48 @@ +# tracing_summary_metrics.prom + +# Histogram of span durations (seconds) +# HELP tempo_span_duration_seconds_bucket Bucketed span durations +# TYPE tempo_span_duration_seconds_bucket histogram +tempo_span_duration_seconds_bucket{le="0.05",service="svc1"} 10 +tempo_span_duration_seconds_bucket{le="0.1",service="svc1"} 25 +tempo_span_duration_seconds_bucket{le="0.5",service="svc1"} 80 +tempo_span_duration_seconds_bucket{le="1",service="svc1"} 100 +tempo_span_duration_seconds_bucket{le="+Inf",service="svc1"} 120 + +tempo_span_duration_seconds_bucket{le="0.05",service="svc2"} 5 +tempo_span_duration_seconds_bucket{le="0.1",service="svc2"} 15 +tempo_span_duration_seconds_bucket{le="0.5",service="svc2"} 60 +tempo_span_duration_seconds_bucket{le="1",service="svc2"} 80 +tempo_span_duration_seconds_bucket{le="+Inf",service="svc2"} 100 + +# Add endpoint-level buckets for “Slowest Endpoints” +tempo_span_duration_seconds_bucket{le="0.05",service="svc1",endpoint="/login"} 4 +tempo_span_duration_seconds_bucket{le="0.1",service="svc1",endpoint="/login"} 10 +tempo_span_duration_seconds_bucket{le="0.5",service="svc1",endpoint="/login"} 30 +tempo_span_duration_seconds_bucket{le="1",service="svc1",endpoint="/login"} 40 +tempo_span_duration_seconds_bucket{le="+Inf",service="svc1",endpoint="/login"} 50 + +tempo_span_duration_seconds_bucket{le="0.05",service="svc1",endpoint="/orders"} 6 +tempo_span_duration_seconds_bucket{le="0.1",service="svc1",endpoint="/orders"} 15 +tempo_span_duration_seconds_bucket{le="0.5",service="svc1",endpoint="/orders"} 50 +tempo_span_duration_seconds_bucket{le="1",service="svc1",endpoint="/orders"} 60 +tempo_span_duration_seconds_bucket{le="+Inf",service="svc1",endpoint="/orders"} 70 + +# COUNT and SUM for histogram (optional, but good practice) +# TYPE tempo_span_duration_seconds_count counter +tempo_span_duration_seconds_count{service="svc1"} 120 +tempo_span_duration_seconds_count{service="svc2"} 100 +# TYPE tempo_span_duration_seconds_sum counter +tempo_span_duration_seconds_sum{service="svc1"} 15.3 +tempo_span_duration_seconds_sum{service="svc2"} 9.8 + +# Total spans received +# TYPE tempo_spans_received_total counter +tempo_spans_received_total{service="svc1"} 1000 +tempo_spans_received_total{service="svc2"} 600 +tempo_spans_received_total{trace_id="traceA"} 50 +tempo_spans_received_total{trace_id="traceB"} 30 + +# Error spans per trace +tempo_spans_received_total{trace_id="traceA",result="error"} 5 +tempo_spans_received_total{trace_id="traceB",result="error"} 3 From 1a0a9e39f72f0e62442d6f15982574ef07a6ea14 Mon Sep 17 00:00:00 2001 From: or4dx <19908496+or4dx@users.noreply.github.com> Date: Sat, 19 Jul 2025 15:27:58 +0400 Subject: [PATCH 2/3] Added observability tracing and logging files --- bit-blocks-starter/observability/README.md | 85 +++++++++++++++++++ .../cloud-cost/cloud-cost-alerts.yaml | 29 +++++++ .../k8s-kube-cost/k8s-kube-cost-alerts.yaml | 38 +++++++++ .../logging/grafana-stack/README.md | 24 ++++++ .../dashboards/grafana-logs-dashboard.json | 48 +++++++++++ .../dashboards}/logging_overview.json | 0 .../grafana-stack/loki/loki-config.yaml | 46 ++++++++++ .../grafana-stack/loki/promtail-config.yaml | 36 ++++++++ .../grafana-stack/loki/sample-logback.xml | 11 +++ .../grafana-stack/loki/sample-nginx.conf | 12 +++ .../grafana-stack/parsers/grok-patterns.md | 15 ++++ .../parsers/json-parse-pattern.md | 10 +++ .../observability/tracing/README.md | 32 +++++++ .../tracing/collector/docker-compose.yaml | 0 .../collector/otel-collector-config.yaml | 19 +++++ .../tracing/collector/tempo.yaml | 16 ++++ .../dashboards/tracing-overview-grafana.json | 44 ++++++++++ .../dashboards}/tracing_summary.json | 0 .../exporters/tempo-exporter-config.yaml | 0 .../parsers/example-attribute-parsing.md | 12 +++ 20 files changed, 477 insertions(+) create mode 100644 bit-blocks-starter/observability/README.md create mode 100644 bit-blocks-starter/observability/cost-monitoring/cloud-cost/cloud-cost-alerts.yaml create mode 100644 bit-blocks-starter/observability/cost-monitoring/k8s-kube-cost/k8s-kube-cost-alerts.yaml create mode 100644 bit-blocks-starter/observability/logging/grafana-stack/README.md create mode 100644 bit-blocks-starter/observability/logging/grafana-stack/dashboards/grafana-logs-dashboard.json rename bit-blocks-starter/observability/{dashboards/grafana-stack/technical => logging/grafana-stack/dashboards}/logging_overview.json (100%) create mode 100644 bit-blocks-starter/observability/logging/grafana-stack/loki/loki-config.yaml create mode 100644 bit-blocks-starter/observability/logging/grafana-stack/loki/promtail-config.yaml create mode 100644 bit-blocks-starter/observability/logging/grafana-stack/loki/sample-logback.xml create mode 100644 bit-blocks-starter/observability/logging/grafana-stack/loki/sample-nginx.conf create mode 100644 bit-blocks-starter/observability/logging/grafana-stack/parsers/grok-patterns.md create mode 100644 bit-blocks-starter/observability/logging/grafana-stack/parsers/json-parse-pattern.md create mode 100644 bit-blocks-starter/observability/tracing/README.md create mode 100644 bit-blocks-starter/observability/tracing/collector/docker-compose.yaml create mode 100644 bit-blocks-starter/observability/tracing/collector/otel-collector-config.yaml create mode 100644 bit-blocks-starter/observability/tracing/collector/tempo.yaml create mode 100644 bit-blocks-starter/observability/tracing/dashboards/tracing-overview-grafana.json rename bit-blocks-starter/observability/{dashboards/grafana-stack/technical => tracing/dashboards}/tracing_summary.json (100%) create mode 100644 bit-blocks-starter/observability/tracing/exporters/tempo-exporter-config.yaml create mode 100644 bit-blocks-starter/observability/tracing/parsers/example-attribute-parsing.md diff --git a/bit-blocks-starter/observability/README.md b/bit-blocks-starter/observability/README.md new file mode 100644 index 0000000..7097137 --- /dev/null +++ b/bit-blocks-starter/observability/README.md @@ -0,0 +1,85 @@ +# Observability 📊 | BitBlock Starter Pack + +The **Observability** module of the BitBlock Starter Pack gives you instant, production-ready visibility into your cloud-native apps and infrastructure—right out of the box. + +Designed for DevOps, SREs, solo founders, and platform teams, this folder helps you quickly add world-class monitoring, cost transparency, and logging with minimal setup. + + +## 📂 Folder Structure + +``` +observability/ +├── cost-monitoring/ +├── dashboards/ +├── metrics/ +└── logging/ +``` + +### What’s Inside + +- **cost-monitoring/** + Pre-built workflows, configs, and queries for cloud and Kubernetes cost visibility. + *Use this to track spend by service, namespace, team, or project. Get actionable insights and alerts for smarter budgeting and resource allocation.* + +- **dashboards/** + Ready-to-import Grafana dashboards for key SRE, infrastructure, business, and cost metrics. + *Jumpstart your monitoring: Copy, customize, and connect to your data sources for instant observability.* + +- **metrics/** + Sample Prometheus metrics and starter queries. + *Ideal for demos, onboarding, or building your first dashboards. Replace with real exporters in production environments.* + +- **logging/** + Example configs and patterns for collecting, parsing, and visualizing logs. + *Integrate with popular tools (e.g., Loki) for end-to-end visibility from application to infrastructure.* + +--- + +## 🚀 Quick Start + +1. **Set Up Your Stack:** + Deploy Prometheus, Grafana, and your preferred logging backend (e.g., Loki, ELK). + +2. **Import Dashboards:** + Load dashboards from `/dashboards` into Grafana to get immediate visibility. + +3. **Connect Cost & Metrics Data:** + - Point `/metrics` samples to Prometheus for demo data, or wire up your exporters for live environments. + - Use `/cost-monitoring` configs for real-time cost insights. + +4. **Enable Logging:** + Apply patterns and configs in `/logging` to ship, parse, and search logs across your environment. + +5. **Customize & Extend:** + Adapt dashboards, queries, and alerts to your team’s needs. + +--- + +## 🛠️ Use Cases + +- Track Kubernetes and cloud costs across teams and projects. +- Monitor SRE metrics: reliability, performance, error budgets, incidents. +- Visualize infrastructure, application, and business KPIs. +- Centralize and search logs for rapid troubleshooting and audits. + +--- + +## 💡 Best Practices + +- Use the sample metrics and configs for demos or quick starts—swap in production data as you go. +- Version-control all dashboards and observability configs to keep your monitoring stack consistent. +- Review and customize alerts, thresholds, and variables to fit your unique environment. + +--- + +## 🤝 Contributions + +Have a killer dashboard or cost query? +Want to help others monitor smarter? +Open a PR or issue—let’s make cloud-native observability effortless for everyone. + +--- + +Happy shipping & scaling 🚀 +*— BitBlock Team* + diff --git a/bit-blocks-starter/observability/cost-monitoring/cloud-cost/cloud-cost-alerts.yaml b/bit-blocks-starter/observability/cost-monitoring/cloud-cost/cloud-cost-alerts.yaml new file mode 100644 index 0000000..1769d9a --- /dev/null +++ b/bit-blocks-starter/observability/cost-monitoring/cloud-cost/cloud-cost-alerts.yaml @@ -0,0 +1,29 @@ +groups: + - name: cloud-cost-alerts + rules: + - alert: CloudMonthlyBudgetExceeded + expr: sum(cloud_cost_total) > 5000 + for: 10m + labels: + severity: critical + annotations: + summary: "Cloud spend above budget" + description: "Cloud spend this month has exceeded $5,000. Review your cloud resources and billing details." + + - alert: CloudDailySpendSpike + expr: increase(cloud_cost_total[1d]) > 1.2 * avg_over_time(increase(cloud_cost_total[1d])[7d:1d]) + for: 10m + labels: + severity: warning + annotations: + summary: "Cloud daily spend anomaly" + description: "Today's cloud spend is over 20% higher than the 7-day daily average. Investigate recent deployments or spikes." + + - alert: TopCloudServiceOverspend + expr: topk(1, sum(cloud_cost_total) by (service)) > 2000 + for: 10m + labels: + severity: warning + annotations: + summary: "Top cloud service overspending" + description: "The top cloud service ({{ $labels.service }}) is spending over $2,000 this month." diff --git a/bit-blocks-starter/observability/cost-monitoring/k8s-kube-cost/k8s-kube-cost-alerts.yaml b/bit-blocks-starter/observability/cost-monitoring/k8s-kube-cost/k8s-kube-cost-alerts.yaml new file mode 100644 index 0000000..c17c72c --- /dev/null +++ b/bit-blocks-starter/observability/cost-monitoring/k8s-kube-cost/k8s-kube-cost-alerts.yaml @@ -0,0 +1,38 @@ +groups: + - name: kubecost-k8s-cost-alerts + rules: + - alert: K8sClusterMonthlySpendHigh + expr: sum(kubecost_cluster_cost_total) > 2000 + for: 10m + labels: + severity: warning + annotations: + summary: "K8s cluster monthly spend high" + description: "The total K8s cluster spend this month is over $2,000. Investigate workloads and resource allocations." + + - alert: K8sNamespaceOverspend + expr: sum(kubecost_namespace_cost_total) by (namespace) > 1000 + for: 10m + labels: + severity: warning + annotations: + summary: "K8s namespace overspending" + description: "Namespace {{ $labels.namespace }} is spending over $1,000 this month. Review for cost optimization." + + - alert: K8sCostPerCPUHigh + expr: sum(kubecost_cluster_cost_total) / sum(kubecost_cluster_cpu_usage_cores) > 5 + for: 10m + labels: + severity: warning + annotations: + summary: "K8s cost per CPU too high" + description: "K8s cost per CPU usage is above $5. Check for resource over-provisioning or underutilization." + + - alert: K8sCostAnomaly + expr: increase(kubecost_cluster_cost_total[1d]) > 1.2 * avg_over_time(increase(kubecost_cluster_cost_total[1d])[7d:1d]) + for: 10m + labels: + severity: critical + annotations: + summary: "K8s cost anomaly detected" + description: "Today's K8s spend is over 20% higher than the last 7-day daily average. Investigate recent changes." diff --git a/bit-blocks-starter/observability/logging/grafana-stack/README.md b/bit-blocks-starter/observability/logging/grafana-stack/README.md new file mode 100644 index 0000000..2997a46 --- /dev/null +++ b/bit-blocks-starter/observability/logging/grafana-stack/README.md @@ -0,0 +1,24 @@ +# BitBlock Logging Observability + +This folder contains starter templates and patterns for collecting, parsing, and visualizing logs across your infrastructure and applications. +**Powered by Grafana Loki** for efficient, scalable, and cost-effective log aggregation. + +## Included +- Loki and Promtail config files (K8s-ready & bare metal) +- Example log parsing patterns (JSON, Grok) +- Sample Logback (Java) & NGINX structured logging configs +- Grafana dashboard template for logs + +## How To Use + +1. **Deploy Loki & Promtail** using provided configs. +2. Point Promtail to your log files/containers or use sidecar deployment in Kubernetes. +3. Use parsing patterns to extract fields for querying and alerting. +4. Import the Grafana dashboard for instant log visibility. + +See `loki/`, `parsers/`, and `dashboards/` for details. + +--- + +> For advanced use cases, integrate with OpenTelemetry Collector or Fluent Bit as log forwarders. +> See Grafana docs for more advanced alerting and pipeline stages. diff --git a/bit-blocks-starter/observability/logging/grafana-stack/dashboards/grafana-logs-dashboard.json b/bit-blocks-starter/observability/logging/grafana-stack/dashboards/grafana-logs-dashboard.json new file mode 100644 index 0000000..d5a6405 --- /dev/null +++ b/bit-blocks-starter/observability/logging/grafana-stack/dashboards/grafana-logs-dashboard.json @@ -0,0 +1,48 @@ +{ + "id": null, + "uid": "bitblock-logs-dashboard", + "title": "BitBlock Logs Overview", + "tags": ["logs", "observability"], + "timezone": "browser", + "schemaVersion": 36, + "version": 1, + "refresh": "30s", + "panels": [ + { + "type": "logs", + "title": "All Application Logs", + "datasource": "${DS_LOKI}", + "targets": [ + { + "expr": "{job=\"varlogs\"}", + "refId": "A" + } + ], + "gridPos": { "x": 0, "y": 0, "w": 24, "h": 8 } + }, + { + "type": "logs", + "title": "NGINX Logs", + "datasource": "${DS_LOKI}", + "targets": [ + { + "expr": "{app=\"nginx\"}", + "refId": "B" + } + ], + "gridPos": { "x": 0, "y": 8, "w": 24, "h": 8 } + }, + { + "type": "logs", + "title": "Error Logs", + "datasource": "${DS_LOKI}", + "targets": [ + { + "expr": "{level=\"error\"}", + "refId": "C" + } + ], + "gridPos": { "x": 0, "y": 16, "w": 24, "h": 8 } + } + ] +} diff --git a/bit-blocks-starter/observability/dashboards/grafana-stack/technical/logging_overview.json b/bit-blocks-starter/observability/logging/grafana-stack/dashboards/logging_overview.json similarity index 100% rename from bit-blocks-starter/observability/dashboards/grafana-stack/technical/logging_overview.json rename to bit-blocks-starter/observability/logging/grafana-stack/dashboards/logging_overview.json diff --git a/bit-blocks-starter/observability/logging/grafana-stack/loki/loki-config.yaml b/bit-blocks-starter/observability/logging/grafana-stack/loki/loki-config.yaml new file mode 100644 index 0000000..12d49ac --- /dev/null +++ b/bit-blocks-starter/observability/logging/grafana-stack/loki/loki-config.yaml @@ -0,0 +1,46 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +ingester: + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 3m + chunk_retain_period: 1m + max_transfer_retries: 0 + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/cache + shared_store: filesystem + filesystem: + directory: /loki/chunks + +limits_config: + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: true + retention_period: 168h diff --git a/bit-blocks-starter/observability/logging/grafana-stack/loki/promtail-config.yaml b/bit-blocks-starter/observability/logging/grafana-stack/loki/promtail-config.yaml new file mode 100644 index 0000000..2b02c04 --- /dev/null +++ b/bit-blocks-starter/observability/logging/grafana-stack/loki/promtail-config.yaml @@ -0,0 +1,36 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: system + static_configs: + - targets: + - localhost + labels: + job: varlogs + __path__: /var/log/*.log + + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app] + target_label: app + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - action: replace + source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - action: replace + source_labels: [__meta_kubernetes_pod_container_name] + target_label: container + - action: replace + replacement: /var/log/pods/*/*/*.log + target_label: __path__ diff --git a/bit-blocks-starter/observability/logging/grafana-stack/loki/sample-logback.xml b/bit-blocks-starter/observability/logging/grafana-stack/loki/sample-logback.xml new file mode 100644 index 0000000..ceec2dd --- /dev/null +++ b/bit-blocks-starter/observability/logging/grafana-stack/loki/sample-logback.xml @@ -0,0 +1,11 @@ + + + + + http://localhost:3100/loki/api/v1/push + + + + + + diff --git a/bit-blocks-starter/observability/logging/grafana-stack/loki/sample-nginx.conf b/bit-blocks-starter/observability/logging/grafana-stack/loki/sample-nginx.conf new file mode 100644 index 0000000..084a5d8 --- /dev/null +++ b/bit-blocks-starter/observability/logging/grafana-stack/loki/sample-nginx.conf @@ -0,0 +1,12 @@ +log_format main_json escape=json + '{' + '"time":"$time_iso8601",' + '"remote_addr":"$remote_addr",' + '"request":"$request",' + '"status":$status,' + '"bytes":$body_bytes_sent,' + '"referer":"$http_referer",' + '"user_agent":"$http_user_agent"' + '}'; + +access_log /var/log/nginx/access.log main_json; diff --git a/bit-blocks-starter/observability/logging/grafana-stack/parsers/grok-patterns.md b/bit-blocks-starter/observability/logging/grafana-stack/parsers/grok-patterns.md new file mode 100644 index 0000000..d98c5a9 --- /dev/null +++ b/bit-blocks-starter/observability/logging/grafana-stack/parsers/grok-patterns.md @@ -0,0 +1,15 @@ +## Common Log Formats and Grok Patterns + +### NGINX Access Log Example + +``` +$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" +``` + +**Grok Pattern:** + +``` +%{IPORHOST:remote_addr} - %{USERNAME:remote_user} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{DATA:http_referer}" "%{DATA:http_user_agent}" +``` + +> **Note:** Loki supports grok parsing via LogQL. \ No newline at end of file diff --git a/bit-blocks-starter/observability/logging/grafana-stack/parsers/json-parse-pattern.md b/bit-blocks-starter/observability/logging/grafana-stack/parsers/json-parse-pattern.md new file mode 100644 index 0000000..9bfd5fb --- /dev/null +++ b/bit-blocks-starter/observability/logging/grafana-stack/parsers/json-parse-pattern.md @@ -0,0 +1,10 @@ +## Parsing JSON Application Logs + +Promtail automatically detects JSON logs if the log line starts with a `{`. + +**Example log:** +```json +{"level":"info","ts":1689825600,"msg":"User signup","user":"jane","status":201} +``` + +In Grafana, use **Log labels** and **JSON parsing** to extract fields from your logs. \ No newline at end of file diff --git a/bit-blocks-starter/observability/tracing/README.md b/bit-blocks-starter/observability/tracing/README.md new file mode 100644 index 0000000..dfd48af --- /dev/null +++ b/bit-blocks-starter/observability/tracing/README.md @@ -0,0 +1,32 @@ + +# Tracing: Collection, Parsing & Visualization + +This folder contains patterns and ready-to-use configs to enable distributed tracing in cloud-native stacks, with end-to-end visibility from application to infrastructure using OpenTelemetry and Grafana Tempo. + +## Components + +- **collector/**: OpenTelemetry Collector config for ingesting, processing, and exporting traces. +- **exporters/**: Example configs for app instrumentation/exporters. +- **parsers/**: Guidelines for attribute parsing and semantic conventions. +- **dashboards/**: Grafana dashboards for Tempo/Tracing. +- **How it works**: + - Apps are instrumented using OpenTelemetry SDKs and send traces to the Collector. + - The Collector exports traces to Tempo. + - Grafana (with Tempo as datasource) provides trace search, filtering, and visualization. + +## Quick Start + +1. `cd tracing/collector && docker-compose up` +2. Instrument your application(s) with OpenTelemetry, pointing to `localhost:4317`. +3. Add the provided dashboard to your Grafana instance and select the Tempo datasource. + +### **Summary Table** + +| Area | Tool | Example File(s) | Key Config/Pattern | +| ---------- | -------------- | ----------------------------- | ------------------------------ | +| Collection | Otel Collector | otel-collector-config.yaml | OTLP receiver + Tempo exporter | +| Export | OpenTelemetry | tempo-exporter-config.yaml | OTLP exporter in app code | +| Parsing | Otel SemConv | example-attribute-parsing.md | Service/span semantic tags | +| Storage | Grafana Tempo | tempo.yaml | Local (or object storage) | +| Dashboard | Grafana | tracing-overview-grafana.json | Trace panel, topk services | + diff --git a/bit-blocks-starter/observability/tracing/collector/docker-compose.yaml b/bit-blocks-starter/observability/tracing/collector/docker-compose.yaml new file mode 100644 index 0000000..e69de29 diff --git a/bit-blocks-starter/observability/tracing/collector/otel-collector-config.yaml b/bit-blocks-starter/observability/tracing/collector/otel-collector-config.yaml new file mode 100644 index 0000000..296e7da --- /dev/null +++ b/bit-blocks-starter/observability/tracing/collector/otel-collector-config.yaml @@ -0,0 +1,19 @@ +receivers: + otlp: + protocols: + grpc: + http: + +exporters: + logging: + loglevel: debug + otlp: + endpoint: tempo:4317 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [otlp, logging] diff --git a/bit-blocks-starter/observability/tracing/collector/tempo.yaml b/bit-blocks-starter/observability/tracing/collector/tempo.yaml new file mode 100644 index 0000000..b03e026 --- /dev/null +++ b/bit-blocks-starter/observability/tracing/collector/tempo.yaml @@ -0,0 +1,16 @@ +auth_enabled: false +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + grpc: + http: + zipkin: + endpoint: 9411 +storage: + trace: + backend: local + local: + path: /tmp/tempo/traces diff --git a/bit-blocks-starter/observability/tracing/dashboards/tracing-overview-grafana.json b/bit-blocks-starter/observability/tracing/dashboards/tracing-overview-grafana.json new file mode 100644 index 0000000..a7be8d7 --- /dev/null +++ b/bit-blocks-starter/observability/tracing/dashboards/tracing-overview-grafana.json @@ -0,0 +1,44 @@ +{ + "id": null, + "uid": "tracing-overview-bitblock", + "title": "Tracing Overview (BitBlock)", + "tags": ["tracing", "tempo", "observability"], + "schemaVersion": 36, + "version": 1, + "refresh": "1m", + "panels": [ + { + "type": "trace", + "title": "Trace Search", + "datasource": "Tempo", + "gridPos": { "x": 0, "y": 0, "w": 24, "h": 10 }, + "targets": [ + { + "datasource": "Tempo", + "expr": "" + } + ], + "options": { + "defaultQuery": { + "serviceName": "", + "spanKind": "", + "minDuration": "", + "maxDuration": "", + "tags": [] + } + } + }, + { + "type": "table", + "title": "Top Services by Trace Count", + "datasource": "Tempo", + "gridPos": { "x": 0, "y": 10, "w": 12, "h": 8 }, + "targets": [ + { + "expr": "topk(5, count_over_time({service_name=~\".+\"}[1h]))", + "legendFormat": "{{service_name}}" + } + ] + } + ] +} diff --git a/bit-blocks-starter/observability/dashboards/grafana-stack/technical/tracing_summary.json b/bit-blocks-starter/observability/tracing/dashboards/tracing_summary.json similarity index 100% rename from bit-blocks-starter/observability/dashboards/grafana-stack/technical/tracing_summary.json rename to bit-blocks-starter/observability/tracing/dashboards/tracing_summary.json diff --git a/bit-blocks-starter/observability/tracing/exporters/tempo-exporter-config.yaml b/bit-blocks-starter/observability/tracing/exporters/tempo-exporter-config.yaml new file mode 100644 index 0000000..e69de29 diff --git a/bit-blocks-starter/observability/tracing/parsers/example-attribute-parsing.md b/bit-blocks-starter/observability/tracing/parsers/example-attribute-parsing.md new file mode 100644 index 0000000..8c941d3 --- /dev/null +++ b/bit-blocks-starter/observability/tracing/parsers/example-attribute-parsing.md @@ -0,0 +1,12 @@ +### Standard Trace Attribute Naming + +Use the following standard attributes to ensure consistent and effective trace data: + +- **`service.name`**: Logical application or service name (**required**) +- **`span.kind`**: Type of span (e.g., `client`, `server`, `internal`) — enables filtering in observability tools like Grafana +- **`http.method`**, **`http.route`**, **`http.status_code`**: For web request spans +- **`db.system`**, **`db.statement`**: For database spans +- **`error`**: Indicates error traces (`true`/`false`) + +> **Best Practice:** +> Follow [OpenTelemetry Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/) for improved parsing, querying, and dashboarding. From 188d94fc0daa49205e0317af2fe9919393412b03 Mon Sep 17 00:00:00 2001 From: or4dx <19908496+or4dx@users.noreply.github.com> Date: Sat, 19 Jul 2025 15:33:08 +0400 Subject: [PATCH 3/3] Fixed error in job_name --- .../observability/metrics/grafana-stack/prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bit-blocks-starter/observability/metrics/grafana-stack/prometheus.yml b/bit-blocks-starter/observability/metrics/grafana-stack/prometheus.yml index 55bd520..eb4df0c 100644 --- a/bit-blocks-starter/observability/metrics/grafana-stack/prometheus.yml +++ b/bit-blocks-starter/observability/metrics/grafana-stack/prometheus.yml @@ -67,7 +67,7 @@ scrape_configs: - targets: ['localhost:8000'] fallback_scrape_protocol: 'PrometheusText0.0.4' - - job_name: 'fake-/market-perf-local' + - job_name: 'fake-market-perf-local' metrics_path: 'business/business_metrics/market_perf_local_metrics.prom' static_configs: - targets: ['localhost:8000']