From 4855fe70fa38654cf959c46936b592ef49b334a9 Mon Sep 17 00:00:00 2001 From: Umesh Date: Sat, 19 Sep 2026 17:27:16 +0530 Subject: [PATCH] docs: add issue/PR templates, expanded contributing guidelines, and docs portal for GitHub Pages --- .../ISSUE_TEMPLATE/alert_rule_enhancement.yml | 118 ++ .github/ISSUE_TEMPLATE/bug_report.yml | 84 + .github/ISSUE_TEMPLATE/config.yml | 11 + .github/ISSUE_TEMPLATE/feature_request.yml | 72 + .github/PULL_REQUEST_TEMPLATE.md | 50 + .github/workflows/pages.yml | 42 + CONTRIBUTING.md | 285 +++- docs/index.html | 1472 +++++++++++++++++ 8 files changed, 2107 insertions(+), 27 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/alert_rule_enhancement.yml create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/pages.yml create mode 100644 docs/index.html diff --git a/.github/ISSUE_TEMPLATE/alert_rule_enhancement.yml b/.github/ISSUE_TEMPLATE/alert_rule_enhancement.yml new file mode 100644 index 0000000..8051b26 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/alert_rule_enhancement.yml @@ -0,0 +1,118 @@ +name: "Alert Rule / SLO Enhancement" +description: "Propose, modify, or tune a Prometheus alert rule, recording rule, or SLO definition" +title: "[Alert/SLO]: " +labels: ["alerts", "slo", "enhancement"] +body: + - type: markdown + attributes: + value: | + ### SRE Alert & SLO Governance Notice + `cloud-native-observability` enforces strict automated validation for all alerting rules: + 1. **Truth-Gate Protocol**: All queries must use verified telemetry metrics and bounded cardinality. + 2. **Canonical SLI AST Equivalence**: Service availability error rates must match the single canonical AST in `deploy/slo/canonical_sli.promql` (excluding 4xx client faults). + 3. **Promtool Test Suite**: Every alert rule must include Promtool unit tests in `deploy/*/alerts/tests/`. + 4. **Runbook Requirement**: Every alert must map directly to an actionable SOP in `docs/RUNBOOKS.md`. + - type: dropdown + id: alert_category + attributes: + label: Alert Category + description: Select the classification of the alert or recording rule. + options: + - Canonical Availability SLI / Burn-Rate Alert (Multi-Window 14.4x / 6x / 1x) + - Golden Signal Alert (Latency, Traffic, Errors, Saturation) + - Kubernetes Infrastructure Alert (Node, Pod, Workload) + - Middleware / Component Alert (Loki, Tempo, OTel Collector) + - Precomputed SLO Recording Rule + - Other Alert Tuning + validations: + required: true + - type: dropdown + id: severity + attributes: + label: Severity & Notification Routing + description: Desired alert severity and Alertmanager receiver tier. + options: + - "critical (Immediate Pager / PagerDuty - breaches error budget or service down)" + - "warning (Slack Alert / Channel Notification - potential degradation)" + - "info (Ticketing / Daily Standup Triage)" + validations: + required: true + - type: input + id: target_file + attributes: + label: Target File Path + description: File to be added or modified. + placeholder: deploy/kubernetes/alerts/slo-alerts.yaml, deploy/docker-compose/prometheus/alerts.yml, etc. + validations: + required: true + - type: textarea + id: promql_rule + attributes: + label: PromQL Expression & Rule Definition + description: Provide the complete YAML rule definition including `alert`, `expr`, `for`, `labels`, and `annotations`. + placeholder: | + - alert: ServiceHighLatencyP99 + expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)) > 1.0 + for: 5m + labels: + severity: warning + team: platform + annotations: + summary: "P99 latency above 1s for {{ $labels.service }}" + runbook_url: "https://github.com/umesh0492/cloud-native-observability/blob/main/docs/RUNBOOKS.md#highlatencyp99" + render: yaml + validations: + required: true + - type: textarea + id: promtool_test_spec + attributes: + label: Promtool Unit Test Case Specification + description: Provide the input series fixtures and expected evaluation states to be added to the Promtool test file. + placeholder: | + evaluation_interval: 1m + input_series: + - series: 'http_request_duration_seconds_bucket{le="1.0",service="demo"}' + values: '100+10x10' + alert_rule_test: + - eval_time: 10m + alertname: ServiceHighLatencyP99 + exp_alerts: + - exp_labels: + severity: warning + service: demo + render: yaml + validations: + required: true + - type: input + id: runbook_url_mapping + attributes: + label: Runbook URL Mapping in docs/RUNBOOKS.md + description: Specify the exact anchor and heading in docs/RUNBOOKS.md documenting triage & mitigation procedures. + placeholder: docs/RUNBOOKS.md#servicehighlatencyp99 + validations: + required: true + - type: textarea + id: ast_cardinality_impact + attributes: + label: AST Equivalence & Metric Cardinality Analysis + description: | + If modifying availability rules, confirm compatibility with `deploy/slo/canonical_sli.promql`. + List all metric label dimensions (`by (...)`) and verify no high-cardinality values (e.g. user IDs, raw URLs) are introduced. + placeholder: | + - Canonical SLI AST equivalent: Yes / N/A + - Grouping dimensions: (service, namespace) + - Cardinality bounded to low-cardinality route templates + validations: + required: true + - type: checkboxes + id: preflight_checks + attributes: + label: Pre-Submission Verification + description: Verify the following before submitting this proposal. + options: + - label: I have verified that 4xx client errors are NOT included in availability SLIs (server 5xx errors only). + required: true + - label: I have verified this rule has a designated runbook entry in `docs/RUNBOOKS.md`. + required: true + - label: I have run or will run `bash ./scripts/test_alerts.sh` to validate Promtool test passes. + required: true diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..f306dd5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,84 @@ +name: "Bug Report" +description: "Report a reproducible defect, broken dashboard, failing alert rule, or configuration error" +title: "[Bug]: " +labels: ["bug", "triage"] +body: + - type: markdown + attributes: + value: | + Thank you for reporting an issue with `cloud-native-observability`! + Please fill out this form as completely as possible. Remember to omit or mask all proprietary secrets, tokens, or credentials. + - type: dropdown + id: component + attributes: + label: Affected Component + description: Select the telemetry layer or repository component where the issue occurs. + options: + - docker-compose + - kubernetes + - terraform + - slo-alerts + - runbooks + - otel-collector + - other + validations: + required: true + - type: textarea + id: description + attributes: + label: Problem Description + description: Provide a clear and concise description of the bug. + placeholder: What went wrong? What behavior was unexpected? + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Steps to Reproduce + description: How can maintainers reproduce this issue? + placeholder: | + 1. Deploy stack using '...' + 2. Execute query '...' + 3. Observe error '...' + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected Behavior + description: What did you expect to happen? + placeholder: The alert should fire when... / The container should start... + validations: + required: true + - type: textarea + id: logs + attributes: + label: Relevant Logs, PromQL Expressions, or Error Messages + description: Paste any terminal logs, container logs, Promtool error messages, or PromQL outputs. + render: shell + - type: textarea + id: system_info + attributes: + label: Environment & Tooling Versions + description: Please share relevant environment details. + value: | + - OS: [e.g. macOS 14.5, Ubuntu 22.04] + - Docker / Docker Compose Version: [e.g. 24.0 / 2.24] + - Kubernetes / Helm Version: [e.g. k8s 1.28, Helm 3.14] + - Terraform Version: [e.g. 1.5.7] + - Promtool Version: [e.g. 2.51.0] + - OpenTelemetry Collector Version: [e.g. 0.98.0] + validations: + required: false + - type: checkboxes + id: checklist + attributes: + label: Verification Checklist + description: Please confirm the following before submitting. + options: + - label: I have searched existing issues and verified this is not a duplicate. + required: true + - label: I have verified this report contains ZERO hardcoded secrets, API tokens, or internal credentials. + required: true + - label: If this is an alert rule issue, I have tested locally using `bash ./scripts/test_alerts.sh`. + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..df710ed --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: Community Discussions & Architecture QA + url: https://github.com/umesh0492/cloud-native-observability/discussions + about: Ask architectural questions, discuss Grafana dashboards, or consult on SLO design. + - name: Incident Response Runbooks + url: https://github.com/umesh0492/cloud-native-observability/blob/main/docs/RUNBOOKS.md + about: Consult on-call SRE runbooks, triage procedures, and alert mitigation guides. + - name: Security Vulnerabilities & Disclosure + url: https://github.com/umesh0492/cloud-native-observability/security/advisories/new + about: Privately report security concerns, credential leaks, or vulnerability findings under our responsible disclosure policy. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..52d252b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,72 @@ +name: "Feature Request / Telemetry Enhancement" +description: "Suggest an idea, new exporter, Grafana dashboard, OpenTelemetry pipeline, or architectural enhancement" +title: "[Feature]: " +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Thank you for suggesting an enhancement to `cloud-native-observability`! + We welcome proposals for new exporters, Grafana panels, OpenTelemetry pipelines, and cloud-native blueprints that uphold our zero-cardinality-explosion and strict AST-verification standards. + - type: dropdown + id: category + attributes: + label: Enhancement Category + description: Select the telemetry domain for your proposal. + options: + - New Exporter / Metric Ingestion + - Grafana Dashboard Addition / Enhancement + - OpenTelemetry Collector Pipeline + - Alert Rule / SLO Enhancement + - Infrastructure / Terraform Blueprint + - Documentation / Runbook + - Other + validations: + required: true + - type: textarea + id: motivation + attributes: + label: Motivation / Problem Statement + description: Is your feature request related to a problem or observability gap? Please describe. + placeholder: A clear and concise description of the engineering challenge or gap you are addressing... + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed Solution & Technical Specification + description: Describe your proposed design, PromQL queries, pipeline configurations, or dashboard layouts. + placeholder: | + - Pipeline components (receivers, processors, exporters) + - Specific PromQL or LogQL queries + - Architecture integration points + validations: + required: true + - type: textarea + id: cardinality + attributes: + label: Cardinality & Resource Impact Analysis + description: If introducing new metrics or labels, how will TSDB cardinality and memory be bounded? + placeholder: | + - Expected series count + - Label normalization strategy (e.g. stripping IDs from paths) + - Memory or storage overhead + validations: + required: false + - type: textarea + id: alternatives + attributes: + label: Alternatives Considered + description: What alternative solutions, tools, or configurations have you considered? + placeholder: Other exporters, query patterns, or architectures explored... + validations: + required: false + - type: checkboxes + id: contribution_intent + attributes: + label: Contribution Readiness + options: + - label: I am willing to submit a pull request implementing this feature. + required: false + - label: I have verified this proposal complies with our ground rules (zero hardcoded secrets, canonical SLI standards). + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..709a868 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,50 @@ +## Summary + + + +## Affected Components + +- [ ] Docker Compose Local Sandbox (`deploy/docker-compose/`) +- [ ] Kubernetes Manifests / Helm Charts (`deploy/kubernetes/`) +- [ ] ArgoCD GitOps Definitions (`deploy/kubernetes/argocd/`) +- [ ] Terraform Infrastructure as Code (`deploy/terraform/`) +- [ ] Prometheus Alert Rules & SLO Definitions (`deploy/*/alerts/`, `deploy/slo/`) +- [ ] Grafana Dashboards (`deploy/*/grafana/dashboards/`) +- [ ] OpenTelemetry Collector Pipeline (`deploy/*/otel-collector/`) +- [ ] Incident Runbooks & Architecture Docs (`docs/`, `README.md`) + +--- + +## Strict Quality & Truth-Gate Checklist + +Before requesting maintainer review, please verify that all automated verification gates pass locally: + +- [ ] **Version Synchronization**: `./scripts/check_version.sh` passes with exit code 0 (validating version tags, ArgoCD sync, and truthfulness gates). +- [ ] **Promtool Alert Unit Tests**: `bash ./scripts/test_alerts.sh` passes with exit code 0 (all test fixtures succeed). +- [ ] **Alertmanager Config Lint**: `amtool check-config deploy/kubernetes/alerts/alertmanager.yaml` passes without syntax or route errors. +- [ ] **Terraform Formatting & Validation**: `terraform -chdir=deploy/terraform fmt -check -recursive` and `terraform -chdir=deploy/terraform validate` succeed. +- [ ] **Terraform Linting**: `tflint --recursive` passes inside `deploy/terraform` with zero rule violations. +- [ ] **Canonical Availability SLI AST Equivalence**: Any new or updated availability error-rate expression matches `deploy/slo/canonical_sli.promql` and strictly excludes 4xx client status codes (`status=~"5.."`). +- [ ] **Runbook Coverage**: All new or modified alerts have a dedicated triage and mitigation SOP documented in [docs/RUNBOOKS.md](file:///Users/umesh/Documents/go-backend-libraries/cloud-native-observability/docs/RUNBOOKS.md). +- [ ] **Zero Hardcoded Credentials**: No secrets, tokens, or plaintext credentials exist in manifests or configuration files. + +--- + +## Verification Evidence + + + +```shell +# Paste output of: +# ./scripts/check_version.sh +# bash ./scripts/test_alerts.sh +# terraform -chdir=deploy/terraform fmt -check -recursive +# cd deploy/terraform && tflint --recursive +``` + +--- + +## Related Issues + + +Fixes # diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..91547ab --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,42 @@ +name: Deploy GitHub Pages Documentation + +on: + push: + branches: + - main + paths: + - "docs/**" + - ".github/workflows/pages.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + deploy: + name: Publish Documentation Portal + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup GitHub Pages + uses: actions/configure-pages@v5 + + - name: Upload Pages Artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2d571c2..70d15e7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,58 +2,289 @@ Thank you for your interest in contributing to `cloud-native-observability`! -This repository provides production-ready, standardized observability infrastructure blueprints across Docker Compose, Kubernetes (Helm), ArgoCD GitOps, and Terraform. +This repository provides production-ready, standardized observability infrastructure blueprints across Docker Compose, Kubernetes (Helm), ArgoCD GitOps, and Terraform. We welcome contributions ranging from bug fixes and documentation enhancements to new OpenTelemetry pipelines and alert rules. + +To maintain production reliability across heterogeneous platforms, this project enforces automated verification gates and rigorous engineering standards. Please review this guide before submitting issues or pull requests. --- ## 1. Ground Rules & Maintainer Expectations -`cloud-native-observability` adheres to strict automated verification gates: -- **Truth-Gate Protocol**: Every metric referenced in Grafana dashboards, every alert query in Prometheus/Alertmanager, and every SLI formula must be validated against actual telemetry sources. -- **Canonical SLI Equivalence**: The availability error-rate SLI is single-sourced in `deploy/slo/canonical_sli.promql` and AST-verified across Compose alerts, Kubernetes SLO rules, and Terraform alert resources. -- **Zero Hardcoded Secrets**: All credentials and sensitive tokens must use Kubernetes secrets or environment variables. +Every contribution must satisfy four foundational principles: + +### A. Truth-Gate Protocol +- **Zero Phantom Telemetry**: Every metric referenced in Grafana dashboards, every alert query in Prometheus or Alertmanager, and every SLI formula must be validated against real telemetry sources exported by services or OpenTelemetry collectors. +- **Fact-Based Claims**: Documentation and commit messages must avoid unverified superlatives, unmeasured recovery time claims, or speculative architecture topologies. +- **Dashboard Integrity**: All Grafana panel queries are statically validated against the repository's metric catalog during CI. + +### B. Canonical SLI AST Equivalence +- The availability error-rate Service Level Indicator (SLI) is single-sourced in `deploy/slo/canonical_sli.promql`: + ```promql + (sum(rate(http_requests_total{status=~"5.."}[{{WINDOW}}])) or vector(0)) / (sum(rate(http_requests_total[{{WINDOW}}])) > 0) or vector(0) + ``` +- **Abstract Syntax Tree (AST) Validation**: Any modification to availability alerts in Docker Compose (`deploy/docker-compose/prometheus/alerts.yml`), Kubernetes PrometheusRules (`deploy/kubernetes/alerts/slo-alerts.yaml`), or Terraform alert definitions (`deploy/terraform/modules/grafana_provisioning/alerts.tf`) must strictly match the canonical AST structure parsed by `./scripts/check_version.sh`. +- **Exclusion of 4xx Client Faults**: Availability error rates evaluate server unreliability only (`status=~"5.."`). Client-side errors (`4xx`) must never consume the service error budget. + +### C. Zero Hardcoded Credentials +- Manifests, Compose files, and Terraform modules must contain **zero plaintext secrets**, API tokens, or hardcoded passwords. +- All secrets must be externalized using Kubernetes Secrets, Helm values overrides, `.env` files (gitignored), or secret managers (e.g. AWS Secrets Manager, HashiCorp Vault). + +### D. Runbook Completeness +- Every newly introduced or modified alert rule must map directly to an actionable Standard Operating Procedure (SOP) in `docs/RUNBOOKS.md`, providing triage steps, diagnostic queries, and mitigation actions for on-call engineers. --- -## 2. Development Setup & Prerequisites +## 2. Repository Structure + +Understanding where files live helps you target your contributions accurately: + +```text +cloud-native-observability/ +├── .github/ +│ ├── ISSUE_TEMPLATE/ # GitHub issue forms (bugs, features, alert rules) +│ ├── workflows/ # GitHub Actions CI/CD (ci.yml, release.yml, pages.yml) +│ └── PULL_REQUEST_TEMPLATE.md # Quality & truth-gate verification checklist +├── deploy/ +│ ├── docker-compose/ # Local developer sandbox (LGTM stack + OTel + Demo App) +│ │ ├── grafana/ # Provisioned datasources and dashboards +│ │ ├── loki/ # Local Loki configuration +│ │ ├── otel-collector/ # OTel collector pipeline (OTLP receiver, processors, exporters) +│ │ ├── prometheus/ # Prometheus configuration, alert rules, and alert tests +│ │ └── tempo/ # Tempo tracing engine configuration +│ ├── kubernetes/ # Production Kubernetes deployment blueprints +│ │ ├── alerts/ # PrometheusRule CRDs and Alertmanager configs +│ │ │ ├── rules/ # Pure Prometheus rules synced from CRDs +│ │ │ └── tests/ # Promtool unit test suites +│ │ ├── argocd/ # GitOps Application & ApplicationSet definitions +│ │ └── helm/ # Helm chart values (kube-prometheus-stack, etc.) +│ ├── slo/ # Canonical SLI PromQL source of truth +│ └── terraform/ # Infrastructure as Code (AWS EKS, Helm, Grafana) +│ ├── environments/ # Environment specifications (dev, prod) +│ └── modules/ # Reusable modules (eks, kubernetes_stack, grafana_provisioning) +├── docs/ # Architecture documentation, ADRs, and runbooks +│ ├── adr/ # Architectural Decision Records +│ ├── CLUSTERED_OBSERVABILITY.md # Clustered architecture and scaling topologies +│ ├── RUNBOOKS.md # Incident response runbooks for all alerts +│ ├── SLO_DESIGN.md # SRE error budget and multi-burn-rate mathematical framework +│ └── TERRAFORM_GUIDE.md # Terraform provisioning workflows +└── scripts/ # Verification gates and test runners + ├── check_tag_readiness.sh # Pre-release tag readiness verification + ├── check_version.sh # Core truth-gate & AST synchronization verifier + └── test_alerts.sh # Promtool alert test runner & CRD sync +``` + +--- + +## 3. Development Setup & Required Tooling + +To run the local observability stack and execute verification suites, install the following tools: + +| Tool | Minimum Version | Purpose | +|---|---|---| +| **Docker & Docker Compose** | Compose v2.20+ | Running local LGTM stack and demo workloads | +| **Prometheus (`promtool`)** | v2.50+ | Unit testing Prometheus alerting and recording rules | +| **Terraform** | v1.5+ | Infrastructure as Code linting, formatting, and validation | +| **TFLint** | v0.51.1+ | Static analysis and best practice enforcement for Terraform | +| **yamllint** | Python 3.10+ | YAML formatting and schema compliance | +| **kubeconform** | v0.6.0+ | Strict Kubernetes schema validation for manifests and CRDs | +| **amtool** (optional) | v0.27+ | Alertmanager configuration syntax and routing verification | + +### Installing Tooling on macOS (Homebrew) +```bash +brew install docker docker-compose +brew install prometheus +brew install terraform +brew install tflint +brew install yamllint +brew install kubeconform +``` + +### Installing Tooling on Linux (Debian/Ubuntu) +```bash +# Docker Compose +sudo apt-get update && sudo apt-get install -y docker-compose-plugin + +# Terraform +sudo apt-get install -y gnupg software-properties-common +curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp.gpg +echo "deb [signed-by=/usr/share/keyrings/hashicorp.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list +sudo apt-get update && sudo apt-get install -y terraform -### Required Tooling -- **Docker & Docker Compose**: Compose v2.20+ -- **Prometheus Tool (`promtool`)**: v2.50+ -- **Terraform**: v1.5+ -- **yamllint**: Python yamllint -- **kubeconform**: Strict Kubernetes schema validator +# TFLint & Kubeconform +curl -s https://raw.githubusercontent.com/terraform-linters/tflint/master/install_linux.sh | bash +curl -L -s https://github.com/yannh/kubeconform/releases/latest/download/kubeconform-linux-amd64.tar.gz | tar xz -C /usr/local/bin kubeconform -### Verification Suite -Before submitting any changes, run the full verification suite locally: +# Yamllint +pip install yamllint +``` + +--- +## 4. Step-by-Step Local Verification Commands + +Before opening a pull request or pushing tags, run the complete local verification suite to ensure zero errors. + +### Step 1: Version & Truthfulness Gate +Validates version synchronizations between `CHANGELOG.md`, `README.md`, Git tags, and ArgoCD manifests. Checks for unverified claims, validates dashboard PromQL against the metric catalog, verifies Alertmanager route parity, and executes AST equivalence parsing: ```bash -# 1. Version & Telemetry Synchronization Gate ./scripts/check_version.sh +``` -# 2. Prometheus Alert Rule Validation +### Step 2: Prometheus Alert Rule Unit Tests +Synchronizes Kubernetes `PrometheusRule` CRD definitions into pure YAML rule sets, validates Kubernetes schemas with `kubeconform`, and executes `promtool test rules` across all test fixtures: +```bash bash ./scripts/test_alerts.sh +``` -# 3. Docker Compose Configuration Lint -docker compose -f deploy/docker-compose/docker-compose.yml config --quiet - -# 4. Terraform Formatting & Validation +### Step 3: Terraform Formatting, Validation & TFLint +Ensures all HCL files follow standard canonical formatting, validates provider blocks, and runs static analysis across all modules: +```bash +# Check formatting terraform -chdir=deploy/terraform fmt -check -recursive + +# Validate syntax terraform -chdir=deploy/terraform init -backend=false terraform -chdir=deploy/terraform validate -# 5. YAML Linting +# Run TFLint static analysis +cd deploy/terraform && tflint --init && tflint --recursive +``` + +### Step 4: Docker Compose Configuration Validation +Validates Compose service dependencies, volume bindings, and port configurations: +```bash +docker compose -f deploy/docker-compose/docker-compose.yml config --quiet +``` + +### Step 5: YAML Linting +Validates YAML formatting and indentation against repository rules: +```bash python3 -m yamllint . ``` +### Step 6: Release Readiness Preflight +When preparing a new release tag (e.g. `v0.2.3`), run the pre-release readiness script or make target to verify working tree cleanliness, changelog alignment, and end-to-end verification: +```bash +# Direct script execution +./scripts/check_tag_readiness.sh v0.2.3 + +# Or via Makefile target +make check-release-readiness TAG=v0.2.3 +``` + --- -## 3. Pull Request Guidelines +## 5. Working with Alerts & SLOs + +When modifying or adding alerts: -1. Create a feature branch from `main`: +1. **Adhere to the Google SRE Multi-Window Multi-Burn-Rate Model**: + - Critical Pager 1h window: **14.4x burn rate** (consumes 2% budget in 1 hour). + - Critical Pager 6h window: **6.0x burn rate** (consumes 5% budget in 6 hours). + - Warning Ticket 3d window: **1.0x burn rate** (consumes 10% budget in 3 days). +2. **Always exclude 4xx client errors**: + - Only HTTP 5xx codes (`status=~"5.."`) reflect genuine platform faults. +3. **Write Promtool Unit Tests**: + - For Kubernetes rules, add fixtures to `deploy/kubernetes/alerts/tests/`. + - For Compose rules, add fixtures to `deploy/docker-compose/prometheus/alerts-test.yaml`. +4. **Update Documentation**: + - Add a triage and mitigation entry in `docs/RUNBOOKS.md`. + - Ensure the alert contains an annotation: `runbook_url: "https://.../docs/RUNBOOKS.md#"`. + +--- + +## 6. Git Workflow & Conventional Commits + +### Branching Strategy +1. Always base your work on the latest `main` branch: ```bash - git checkout -b feat/your-feature-name + git checkout main + git pull origin main ``` -2. Ensure all 5 verification checks above pass with exit code 0. -3. Keep commits atomic with conventional commit messages (`feat:`, `fix:`, `docs:`, `refactor:`, `ci:`). -4. Open a pull request describing the changes and linking relevant issues. +2. Create a focused feature branch using standard prefixes: + - `feat/`: New capabilities, dashboards, exporters, or blueprints + - `fix/`: Defect repairs or alert threshold calibrations + - `docs/`: Documentation updates, runbooks, or guides + - `refactor/`: HCL, Helm, or pipeline refactoring without behavior change + - `ci/`: CI/CD workflow updates + +### Conventional Commit Standards +We follow the [Conventional Commits](https://www.conventionalcommits.org/) specification: +```text +(): + +[optional body providing technical rationale] + +[optional footer referencing issues, e.g. Fixes #123] +``` + +**Allowed Types**: +- `feat`: A new feature or capability +- `fix`: A bug fix +- `docs`: Documentation updates +- `style`: Formatting changes that do not affect code logic +- `refactor`: Code or configuration change that neither fixes a bug nor adds a feature +- `perf`: Performance optimization (e.g. reducing TSDB ingestion overhead) +- `test`: Adding or correcting tests (e.g. Promtool test cases) +- `ci`: CI/CD workflow changes or build script updates +- `chore`: Maintenance tasks, dependency bumps, or tool configuration + +**Allowed Scopes**: +- `compose`: Docker Compose sandbox +- `k8s`: Kubernetes manifests +- `helm`: Helm charts and value overrides +- `argocd`: ArgoCD GitOps applications +- `terraform`: Terraform HCL modules and configurations +- `alerts`: Prometheus alerts and recording rules +- `slo`: Service Level Objectives and canonical SLI expressions +- `otel`: OpenTelemetry Collector configuration +- `dashboards`: Grafana dashboard JSON models +- `runbooks`: Incident response runbooks +- `scripts`: Local and CI verification scripts + +**Examples**: +- `feat(alerts): add KubeDeploymentReplicasMismatch warning alert` +- `fix(otel): add memory_limiter processor before batch processor` +- `docs(runbooks): add triage steps for ServiceErrorBudgetBurnRateHigh1h` +- `ci(actions): add pages deployment workflow for documentation portal` + +--- + +## 7. Branch Protection & CI-Gated Release Tags + +### Branch Protection Rules +The `main` branch is protected by strict GitHub repository settings: +- Direct pushes to `main` are disabled; all changes must arrive via pull request. +- All status checks defined in `.github/workflows/ci.yml` must pass before merging: + - Version & claim truthfulness gate + - Promtool alert rules unit testing + - Docker Compose syntax validation + - Terraform fmt, validate, and TFLint + - Kubeconform Kubernetes schema validation + - Yamllint YAML syntax validation +- Pull requests require at least one approving code review from a maintainer. +- Commits must maintain a linear history (Squash and Merge or Rebase and Merge). + +### Release Tag Protocol +Releases are strictly tied to semantic versioning (`vX.Y.Z`): +1. Maintainers update `CHANGELOG.md` with the new version section and release notes. +2. Update version headers in `README.md` and ArgoCD target revisions as applicable. +3. Validate repository release readiness: + ```bash + make check-release-readiness TAG=vX.Y.Z + ``` +4. Once merged to `main`, push the annotated Git tag: + ```bash + git tag -a vX.Y.Z -m "Release vX.Y.Z" + git push origin vX.Y.Z + ``` +5. Pushing a `v*` tag triggers `.github/workflows/release.yml`, which runs preflight verification, builds release tarballs, generates SHA256 checksums, and publishes an official GitHub Release. + +--- + +## 8. Community & Communication + +- **Discussions**: Use GitHub Discussions for architectural questions, telemetry design patterns, or troubleshooting. +- **Issue Tracker**: Use GitHub Issues for reproducible bugs and actionable feature proposals. +- **Code of Conduct**: All participants are expected to adhere to our [Code of Conduct](CODE_OF_CONDUCT.md). +- **Security Inquiries**: Report security vulnerabilities privately per our [Security Policy](SECURITY.md). diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..3b31756 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,1472 @@ + + + + + + Cloud-Native Observability Platform · Documentation & Architecture Portal + + + + + + +
+
+ +
⚡
+ cloud-native-observability + + +
+
+ + +
+
+
+ 🛡️ Production-Grade Observability Blueprints +
+

Cloud-Native Observability Platform

+

+ Unified LGTM (Loki, Grafana, Tempo, Prometheus/Thanos) and OpenTelemetry architecture with automated cross-layer AST verification, Google SRE Multi-Burn-Rate alerting, and GitOps blueprints. +

+ +
+
● Release: v0.2.3
+
✓ CI: Passing
+
⬡ Terraform: 1.5+
+
⚡ OTel Collector: v0.98+
+
+ + +
+
+
+
+
+
+
+
bash · quickstart
+
+
+
+
+ $git clone https://github.com/umesh0492/cloud-native-observability.git + +
+
+ $cd cloud-native-observability +
+
+ $docker compose up -d +
+
+
+ Grafana UI (admin/admin) + localhost:3000 +
+
+ Prometheus Web + localhost:9090 +
+
+ Tempo Tracing + localhost:3200 +
+
+ OTel OTLP Endpoints + 4317 (gRPC) / 4318 (HTTP) +
+
+
+
+ +
+
+ + +
+
+
+ Topology +

Telemetry Flow & Architecture

+

+ Unified telemetry ingestion: distributed traces, log streams, and system metrics are normalized by OpenTelemetry Collectors and routed to dedicated storage backends. +

+
+ +
+
+
+ Microservices / Workloads + Go, Python, Java, Node.js +
+
→
+
+ OTel Collector + OTLP gRPC (4317) / HTTP (4318) +
+
→
+
+ Processors & Connectors + Memory Limiter, Batch, Spanmetrics +
+
→
+
+ LGTM Storage Backends + Prometheus/Thanos, Loki, Tempo +
+
→
+
+ Grafana Unified UI + Single Pane of Glass +
+
+
+
+
+ + +
+
+
+ Components +

LGTM Platform Matrix

+

+ Engineered for production resilience across local Docker Compose sandboxes, Kubernetes Helm/ArgoCD clusters, and AWS EKS via Terraform. +

+
+ +
+ +
+
+
Loki
+ Logs +
+

Horizontally scalable, multi-tenant log aggregation system optimized for cost efficiency through index-free metadata labeling.

+
    +
  • Structured metadata & LogQL streaming queries
    • +
  • Zero-indexing design avoids high-cardinality crashes
    • +
  • Direct trace-to-logs linking via `trace_id` correlation
    • +
+
+ + +
+
+
Grafana
+ Visualization +
+

Production dashboard visualization engine with pre-provisioned data sources and single-pane-of-glass operational telemetry.

+
    +
  • Pre-built Platform Overview & Microservice APM dashboards
    • +
  • Automatic drill-down from metrics to Tempo trace spans
    • +
  • AST-verified alert rule queries & unified alert panel
    • +
+
+ + +
+
+
Tempo
+ Distributed Tracing +
+

High-scale distributed tracing backend designed for massive ingestion throughput with zero-dependency object storage persistence.

+
    +
  • Native OTLP receiver integration
    • +
  • Spanmetrics connector derives latency histograms from spans
    • +
  • Service graph connector calculates request dependencies
    • +
+
+ + +
+
+
Prometheus / Thanos
+ Metrics & SLOs +
+

High-reliability metric TSDB with Thanos sidecar/store/querier integration for long-term historical retention and multi-cluster deduplication.

+
    +
  • Precomputed recording rules for rolling 5m/30m/1h/6h/3d error rates
    • +
  • Google SRE Multi-Burn-Rate alerting engine
    • +
  • 100% unit-tested alert definitions via Promtool
    • +
+
+ + +
+
+
OpenTelemetry Collector
+ Telemetry Gateway +
+

Vendor-agnostic telemetry proxy providing memory-bounded pipeline buffering, attribute redaction, and multi-destination dispatching.

+
    +
  • Memory limiter processor prevents container OOMKills
    • +
  • Batch processor buffers high-throughput trace and metric events
    • +
  • Route path normalization eliminates TSDB cardinality explosion
    • +
+
+
+
+
+ + +
+
+
+ SRE Governance +

Google SRE Multi-Window Burn-Rate Alerting

+

+ Mathematical framework for high-precision, low-noise error budget consumption alerting adhering to Chapter 5 of the Google Site Reliability Workbook. +

+
+ +
+

Mathematical Foundation

+

+ Given an availability target of 99.9% (SLO), the allowable unreliability is the Error Budget: +

+
+ Error Budget = 1 - SLO = 1 - 0.999 = 0.001 (0.1% of total requests) +
+

+ A Burn Rate of 1.0x exhausts exactly 100% of the budget over 30 days (720 hours). Faster consumption triggers multi-window paging: +

+
+ Burn Rate = Observed Error Rate / Allowed Error Rate (0.001) +
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Alert NameSeverity TierShort WindowLong WindowBurn RateBudget ConsumedRouting Target
ServiceErrorBudgetBurnRateHigh1hCRITICAL PAGER5 minutes1 hour14.4x2% in 1 hourPagerDuty (Immediate Page)
ServiceErrorBudgetBurnRateHigh6hCRITICAL PAGER30 minutes6 hours6.0x5% in 6 hoursPagerDuty (Immediate Page)
ServiceErrorBudgetBurnRateSlow3dWARNING TICKET6 hours3 days1.0x10% in 3 daysSlack / Jira Backlog
+
+ + +
+

Interactive SRE Burn-Rate Simulator

+

+ Adjust the parameters below to compute the instantaneous error budget burn rate and assess which alert tier triggers: +

+ +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+
Calculated Burn Rate
+
14.4x
+
+
+
1h Budget Consumption
+
2.0% / hr
+
+
+
Alertmanager Evaluation
+
FIRING (Critical Pager)
+
+
+
+ +
+
+ + +
+
+
+ Cross-Layer Consistency +

Canonical Availability SLI AST Equivalence

+

+ Single-sourced mathematical logic in deploy/slo/canonical_sli.promql is statically parsed and AST-verified across all deployment blueprints to eliminate metric drift. +

+
+ +
+

Canonical AST Formula

+
+ (sum(rate(http_requests_total{status=~"5.."}[{{WINDOW}}])) or vector(0)) / (sum(rate(http_requests_total[{{WINDOW}}])) > 0) or vector(0) +
+

+ Security & Cardinality Guardrail: Excludes 4xx client errors (`status=~"5.."` server-only) to protect against false pages triggered by bad clients, unauthorized scans, or user typos. +

+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Observability LayerTarget Configuration FileRule / Metric TargetWindowAST Verification Status
Layer 1: Local Sandboxdeploy/docker-compose/prometheus/alerts.ymlHighErrorRate5mVerified Equivalent ✅
Layer 2: Production GitOpsdeploy/kubernetes/alerts/slo-alerts.yamljob:http_requests:error_rate_5m5mVerified Equivalent ✅
Layer 2: Production GitOpsdeploy/kubernetes/alerts/slo-alerts.yamljob:http_requests:error_rate_30m30mVerified Equivalent ✅
Layer 2: Production GitOpsdeploy/kubernetes/alerts/slo-alerts.yamljob:http_requests:error_rate_1h1hVerified Equivalent ✅
Layer 2: Production GitOpsdeploy/kubernetes/alerts/slo-alerts.yamljob:http_requests:error_rate_6h6hVerified Equivalent ✅
Layer 2: Production GitOpsdeploy/kubernetes/alerts/slo-alerts.yamljob:http_requests:error_rate_3d3dVerified Equivalent ✅
Layer 3: Infrastructure as Codedeploy/terraform/modules/grafana_provisioning/alerts.tfHighErrorRateP15mVerified Equivalent ✅
+
+
+
+ + +
+
+
+ Implementation +

Telemetry Configurations & Rules

+

+ Browse verified PromQL alert rules, OpenTelemetry pipeline definitions, and Alertmanager routing hierarchies. +

+
+ +
+
+ + + + +
+ + +
+ + 
# Canonical Service Level Indicator (SLI) - Availability Error Rate
+# deploy/slo/canonical_sli.promql
+(
+  sum(rate(http_requests_total{status=~"5.."}[{{WINDOW}}]))
+  or vector(0)
+)
+/
+(
+  sum(rate(http_requests_total[{{WINDOW}}])) > 0
+)
+or vector(0)
+
+ + +
+ + 
# Google SRE Multiwindow Burn-Rate Alert (14.4x consumption)
+# deploy/kubernetes/alerts/slo-alerts.yaml
+- alert: ServiceErrorBudgetBurnRateHigh1h
+  expr: |
+    (
+      job:http_requests:error_rate_1h{job="demo-app"} > (1 - 0.999) * 14.4
+    )
+    and
+    (
+      job:http_requests:error_rate_5m{job="demo-app"} > (1 - 0.999) * 14.4
+    )
+  for: 2m
+  labels:
+    severity: critical
+    tier: p1-pager
+  annotations:
+    summary: "High error budget burn rate (14.4x) on {{ $labels.service }}"
+    description: "Service {{ $labels.service }} is consuming 2% of its monthly error budget in 1 hour."
+    runbook_url: "https://github.com/umesh0492/cloud-native-observability/blob/main/docs/RUNBOOKS.md#serviceerrorbudgetburnratehigh1h"
+
+ + +
+ + 
# OpenTelemetry Collector Pipeline Configuration
+# deploy/docker-compose/otel-collector/otel-collector-config.yaml
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  memory_limiter:
+    check_interval: 1s
+    limit_percentage: 75
+    spike_limit_percentage: 20
+
+  batch:
+    send_batch_size: 8192
+    timeout: 500ms
+
+connectors:
+  spanmetrics:
+    histogram:
+      explicit:
+        buckets: [2ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s]
+    dimensions:
+      - name: http.status_code
+      - name: http.method
+      - name: http.route
+
+exporters:
+  prometheus:
+    endpoint: "0.0.0.0:8889"
+  otlp/tempo:
+    endpoint: "tempo:4317"
+    tls:
+      insecure: true
+  otlphttp/loki:
+    endpoint: "http://loki:3100/otlp"
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [memory_limiter, batch]
+      exporters: [otlp/tempo, spanmetrics]
+    metrics:
+      receivers: [otlp, spanmetrics]
+      processors: [memory_limiter, batch]
+      exporters: [prometheus]
+    logs:
+      receivers: [otlp]
+      processors: [memory_limiter, batch]
+      exporters: [otlphttp/loki]
+
+ + +
+ + 
# Alertmanager Multi-Tier Routing Configuration
+# deploy/kubernetes/alerts/alertmanager.yaml
+global:
+  resolve_timeout: 5m
+
+route:
+  group_by: ['alertname', 'namespace', 'service']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+  receiver: 'slack-notifications'
+  routes:
+    # Critical Tier -> PagerDuty Immediate Page
+    - match:
+        severity: critical
+      receiver: 'pagerduty-critical'
+      continue: true
+
+    # Warning Tier -> Slack Monitoring Channel
+    - match:
+        severity: warning
+      receiver: 'slack-notifications'
+
+inhibit_rules:
+  # Inhibit warning alerts if the entire service is down
+  - source_match:
+      alertname: 'ServiceDown'
+    target_match_re:
+      alertname: 'HighErrorRate|HighLatencyP99'
+    equal: ['service', 'namespace']
+
+
+
+
+ + +
+
+
+ Guides & SOPs +

Architecture & Operations Library

+

+ Comprehensive operational documentation, incident runbooks, and cloud infrastructure guides. +

+
+ +
+ +
+
+
Incident Runbooks
+
Actionable SOPs for on-call engineers covering Multi-Burn-Rate breaches, pod crash looping, worker pool exhaustion, and circuit breaker trip events.
+
+ Read docs/RUNBOOKS.md → +
+ + +
+
+
SLO & Error Budget Design
+
Theoretical derivations, rolling multi-window alerting mathematics, and the canonical availability SLI specification.
+
+ Read docs/SLO_DESIGN.md → +
+ + +
+
+
Clustered Observability
+
Production topologies for multi-tenant, high-volume metrics, logs, and distributed trace architectures at enterprise scale.
+
+ Read docs/CLUSTERED_OBSERVABILITY.md → +
+ + +
+
+
Terraform IaC Guide
+
Automated provisioning workflows for AWS EKS clusters, Helm monitoring releases, and Grafana alert rule synchronization.
+
+ Read docs/TERRAFORM_GUIDE.md → +
+ + +
+
+
Integration Guide
+
Step-by-step instrumentations for Go, Python, and Java services exporting OTLP traces and Prometheus golden signals.
+
+ Read docs/INTEGRATION_GUIDE.md → +
+ + +
+
+
Sampling Strategies
+
Head-based and tail-based distributed tracing sampling algorithms to preserve critical error spans while containing storage costs.
+
+ Read docs/SAMPLING_STRATEGIES.md → +
+
+
+
+ + + + + + + +