Add focused alerting and durable incident evidence

.github/workflows/ci.yml

@@ -102,3 +102,36 @@ jobs:
           flags: unittests
           token: ${{ secrets.CODECOV_TOKEN }}
           slug: platformlabs-co/fetchr.sh
+
+  alerting:
+    name: Alert Rules and Lifecycle
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.24'
+      - name: Download verified monitoring tools
+        run: |
+          mkdir -p "$RUNNER_TEMP/monitoring"
+          cd "$RUNNER_TEMP/monitoring"
+          for spec in prometheus:3.14.0 alertmanager:0.34.1; do
+            tool="${spec%:*}"
+            version="${spec#*:}"
+            archive="$tool-$version.linux-amd64.tar.gz"
+            curl -fsSL "https://github.com/prometheus/$tool/releases/download/v$version/$archive" -o "$archive"
+            curl -fsSL "https://github.com/prometheus/$tool/releases/download/v$version/sha256sums.txt" -o "$tool.sums"
+            sha256sum --check --ignore-missing "$tool.sums"
+            tar -xzf "$archive"
+          done
+      - name: Validate rules and routing
+        run: |
+          "$RUNNER_TEMP/monitoring/prometheus-3.14.0.linux-amd64/promtool" check config examples/alerting/prometheus.yml
+          "$RUNNER_TEMP/monitoring/prometheus-3.14.0.linux-amd64/promtool" test rules examples/alerting/rules.test.yml
+          "$RUNNER_TEMP/monitoring/alertmanager-0.34.1.linux-amd64/amtool" check-config examples/alerting/alertmanager.yml
+      - name: Validate real notification lifecycle
+        run: |
+          go test -race ./internal/alerting ./internal/proxy
+          go build -o "$RUNNER_TEMP/netkit-alert-test" ./cmd/netkit
+          python3 scripts/test-alerting-integration.py --netkit "$RUNNER_TEMP/netkit-alert-test" --alertmanager "$RUNNER_TEMP/monitoring/alertmanager-0.34.1.linux-amd64/alertmanager"

.gitignore

@@ -9,3 +9,10 @@ bin/
 
 # Embedded dashboard files (generated during build)
 internal/dashboard/out/
+
+# Local alerting examples: credentials and retained evidence
+examples/alerting/webhook-token
+examples/alerting/notification-token
+examples/alerting/incidents/
+examples/alerting/alertmanager-data/
+examples/alerting/prometheus-data/

README.md

@@ -372,3 +372,7 @@ Captured history is sanitized before retention. The dashboard separates HTTP sta
 ### Investigation workspace
 
 Traffic is the landing page, with preserved filters, adjacent request inspection, buffered Live/Pause, and an Overview that drills into matching retained evidence. Compose is available at `/compose/`. See [investigation workspace semantics](docs/investigation-workspace.md) for freshness, URL scope, latency populations, and retention limits.
+
+### Focused alerting
+
+Use the [Prometheus and Alertmanager example](examples/alerting/README.md) for sustained upstream/transport failure alerts, opt-in ordinary-workload header latency, external availability and insufficient-sample signals. The separate `netkit alerts` process retains bounded metadata-only incident snapshots with authenticated webhook writes. Alert evaluation and notification ownership remain external; no native monitor editor or production notification destination is enabled automatically.

cmd/netkit/alerts.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/biancarosa/netkit/internal/alerting"
+)
+
+func runAlerts(args []string) error {
+	flags := flag.NewFlagSet("alerts", flag.ContinueOnError)
+	listen := flags.String("listen", "127.0.0.1:8090", "Incident receiver address; protect with authenticated ingress before exposing")
+	dir := flags.String("dir", "", "Private incident directory (required)")
+	admin := flags.String("admin-url", "http://127.0.0.1:8081", "Fixed Netkit admin URL for metadata evidence")
+	dashboard := flags.String("dashboard-url", "http://127.0.0.1:3000", "Public Netkit dashboard URL including any prefix")
+	instance := flags.String("instance", "netkit-local", "Expected Prometheus instance alias")
+	capacity := flags.Int("capacity", 100, "Maximum stored incidents (1–1000)")
+	ttl := flags.Duration("retention", 7*24*time.Hour, "Incident retention (1 minute–30 days)")
+	if err := flags.Parse(args); err != nil {
+		return err
+	}
+	if *dir == "" {
+		return fmt.Errorf("--dir is required")
+	}
+	receiver, err := alerting.New(alerting.Config{Dir: *dir, Token: os.Getenv("NETKIT_ALERT_TOKEN"), Instance: *instance, AdminURL: *admin, DashboardURL: *dashboard, Capacity: *capacity, TTL: *ttl})
+	if err != nil {
+		return err
+	}
+	if err := receiver.Prune(); err != nil {
+		return err
+	}
+	server := &http.Server{Addr: *listen, Handler: receiver.Handler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second, WriteTimeout: 45 * time.Second, IdleTimeout: 30 * time.Second}
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+	go func() {
+		ticker := time.NewTicker(time.Minute)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+				_ = server.Shutdown(shutdown)
+				cancel()
+				return
+			case <-ticker.C:
+				if err := receiver.Prune(); err != nil {
+					log.Print("incident retention maintenance failed; check private storage")
+				}
+			}
+		}
+	}()
+	if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+		return err
+	}
+	return nil
+}

cmd/netkit/main.go

@@ -14,13 +14,17 @@ import (
 func main() {
 	// Parse command
 	if len(os.Args) < 2 {
-		log.Fatal("Please specify a command: serve, request, or ca")
+		log.Fatal("Please specify a command: serve, request, ca, or alerts")
 	}
 
 	command := os.Args[1]
 	os.Args = os.Args[1:] // Remove command from args for flag parsing
 
 	switch command {
+	case "alerts":
+		if err := runAlerts(os.Args[1:]); err != nil {
+			log.Fatal(err)
+		}
 	case "ca":
 		if err := runCA(os.Args[1:]); err != nil {
 			log.Fatal(err)
@@ -32,7 +36,7 @@ func main() {
 			log.Fatal(err)
 		}
 	default:
-		log.Fatalf("Unknown command: %s. Use 'serve', 'request', or 'ca'", command)
+		log.Fatalf("Unknown command: %s. Use 'serve', 'request', 'ca', or 'alerts'", command)
 	}
 }
 

dashboard/src/components/InvestigationWorkspace.tsx

@@ -302,18 +302,24 @@ export function InvestigationWorkspace({
         >
           Clear filters
         </Button>
-        {(state.scope.route || state.scope.from || state.scope.to) && (
+        {(state.scope.signal ||
+          state.scope.route ||
+          state.scope.from ||
+          state.scope.to) && (
           <p className="w-full break-all text-xs">
+            {state.scope.signal && `Alert signal: ${state.scope.signal} · `}
             {state.scope.route && `Route: ${state.scope.route} · `}
             {state.scope.from &&
               `From ${new Date(state.scope.from).toLocaleString()} `}
             {state.scope.to &&
               `to ${new Date(state.scope.to).toLocaleString()}`}{" "}
             <button
               className="underline"
-              onClick={() => setScope({ route: "", from: "", to: "" })}
+              onClick={() =>
+                setScope({ signal: "", route: "", from: "", to: "" })
+              }
             >
-              Remove route/time scope
+              Remove alert/route/time scope
             </button>
           </p>
         )}

dashboard/src/lib/investigation.test.ts

@@ -22,6 +22,8 @@ const record = (
     outcome: "complete",
     success: true,
     response_source: "upstream",
+    upstream_start_time: "2026-09-22T12:00:00Z",
+    upstream_end_time: "2026-09-22T12:00:00.001Z",
     upstream_latency_us: 1000,
     total_duration_us: 30000000,
     ...overrides,
@@ -101,3 +103,43 @@ test("trend bucket boundaries partition records exactly and produce matching dri
   assert.equal(buckets([record("a"), record("b")]).length, 1);
   assert.equal(summary([]).p95, null);
 });
+
+test("alert links use exact signal populations and exclude unrelated failures", () => {
+  const records = [
+    record("http5xx", { response_status: 503 }),
+    record("401", { response_status: 401 }),
+    record("transport", {
+      outcome: "upstream_error",
+      response_status: 502,
+      response_source: "proxy",
+    }),
+    record("canceled", {
+      outcome: "client_canceled",
+      response_status: 0,
+      response_source: "proxy",
+    }),
+  ];
+  assert.deepEqual(
+    records
+      .filter((r) => matches(r, { ...emptyScope, signal: "http5xx" }))
+      .map((r) => r.id),
+    ["http5xx"],
+  );
+  assert.deepEqual(
+    records
+      .filter((r) => matches(r, { ...emptyScope, signal: "transport" }))
+      .map((r) => r.id),
+    ["transport"],
+  );
+  assert.deepEqual(
+    records
+      .filter((r) => matches(r, { ...emptyScope, signal: "headers" }))
+      .map((r) => r.id),
+    ["http5xx", "401"],
+  );
+  assert.equal(
+    records.filter((r) => matches(r, { ...emptyScope, signal: "unknown" }))
+      .length,
+    0,
+  );
+});

dashboard/src/lib/investigation.ts

@@ -1,5 +1,6 @@
 import type { BackendRequestRecord as Record } from "../services/api";
 export type Scope = {
+  signal: string;
   host: string;
   route: string;
   method: string;
@@ -8,6 +9,7 @@ export type Scope = {
   to: string;
 };
 export const emptyScope: Scope = {
+  signal: "",
   host: "",
   route: "",
   method: "",
@@ -42,6 +44,16 @@ export function matches(record: Record, scope: Scope) {
   const target = endpoint(record);
   const time = Date.parse(record.timestamp);
   return (
+    (!scope.signal ||
+      (scope.signal === "http5xx"
+        ? record.response_source === "upstream" &&
+          record.response_status >= 500 &&
+          record.response_status <= 599
+        : scope.signal === "transport"
+          ? ["upstream_error", "proxy_error"].includes(record.outcome)
+          : scope.signal === "headers"
+            ? headersEligible(record)
+            : false)) &&
     (!scope.host || scope.host === target.host) &&
     (!scope.route || scope.route === target.route) &&
     (!scope.method || scope.method === record.method) &&
@@ -58,6 +70,10 @@ export function headersEligible(record: Record) {
     record.response_source === "upstream" &&
     record.response_status > 0 &&
     record.method !== "CONNECT" &&
+    Number.isFinite(Date.parse(record.upstream_start_time)) &&
+    Date.parse(record.upstream_start_time) > 0 &&
+    Date.parse(record.upstream_end_time) >=
+      Date.parse(record.upstream_start_time) &&
     record.upstream_latency_us >= 0
   );
 }

examples/alerting/README.md

@@ -0,0 +1,94 @@
+# Focused alerting
+
+Prometheus evaluates rules; Alertmanager groups, silences and delivers notifications. Netkit exports HTTP signals and provides investigation and bounded incident metadata. This example does not configure a real email/chat destination or deploy any service.
+
+## Rules and populations
+
+| Rule | Condition | Gate / hold | Interpretation |
+| --- | --- | --- | --- |
+| NetkitUpstreamFailureRate | Upstream HTTP 5xx / non-tunnel HTTP observations > 10% over 5m | At least 20 observations; 3m sustained | Does not call 401/429 or inspection rejection an upstream outage. |
+| NetkitTransportFailureRate | Proxy errors or upstream connection failures / HTTP observations > 10% over 5m | At least 20 observations; 3m sustained | Client cancellation and inspection rejection are excluded from the numerator. |
+| NetkitHeaderLatencyHigh | p95 upstream response-header latency > 1 second over 5m | At least 20 header samples; 5m sustained; **ordinary profile only** | Excludes response body lifetime, tunnels and failed attempts without headers. |
+| NetkitUnavailable | External Prometheus scrape `up == 0` | 2m sustained | Reachability, not proof of capture completeness. |
+| NetkitInsufficientSamples | Fewer than 20 HTTP observations over 5m | 5m sustained; informational | No data/low volume is unknown, not healthy or a missing-traffic failure. |
+
+Thresholds are examples, not universal defaults. The sample `netkit_latency_profile: unclassified` deliberately disables the latency rule. Enable `ordinary` only on an isolated proxy serving a comparable ordinary HTTP workload. Netkit does not infer request types, dependencies or route templates. Never enable a blanket latency rule for mixed long-polling/streaming traffic. Existing metrics support instance-level scope, not host/route alerts; the UI must not imply otherwise.
+
+Each rule includes a value, threshold, window and investigation link. Static `netkit_dashboard` and `netkit_incidents` external labels must contain public URLs without credentials or query secrets. Set the dashboard URL to include any deployment prefix, without a trailing slash. Metric/alert labels must never contain request URLs, credentials or arbitrary payload fields.
+
+## Local setup
+
+Use Prometheus 3.14.0 and Alertmanager 0.34.1 (the versions validated in CI), plus a current Netkit build. Start Netkit normally with admin port 8081 and dashboard port 3000. Run the incident receiver as a **separate process**, ideally on a different host or failure domain from the monitored proxy in a real deployment.
+
+From this directory:
+
+```sh
+umask 077
+openssl rand -hex 32 > webhook-token
+mkdir -m 700 incidents
+NETKIT_ALERT_TOKEN="$(cat webhook-token)" netkit alerts \
+  --dir "$PWD/incidents" \
+  --listen 127.0.0.1:8090 \
+  --admin-url http://127.0.0.1:8081 \
+  --dashboard-url http://localhost:3000 \
+  --instance netkit-local
+```
+
+In separate terminals, from this same directory:
+
+```sh
+alertmanager --config.file=alertmanager.yml --storage.path=./alertmanager-data
+prometheus --config.file=prometheus.yml --storage.tsdb.path=./prometheus-data
+```
+
+The webhook token is mandatory (at least 16 characters), provided only through the receiver environment and Alertmanager's credentials file. Keep it out of Git. The receiver binds loopback by default. Its read-only HTML interface has **no built-in authentication**: use authenticated private ingress before exposing it. The fixed admin URL is trusted configuration; redirects and alert-supplied target URLs are not followed. Keep the existing Netkit admin endpoint private as well.
+
+The example expects one proxy with the safe Prometheus instance alias `netkit-local`. For multiple instances, use one receiver and directory per instance and route Alertmanager by instance; mismatched instances are rejected rather than snapshotting unrelated traffic.
+
+Open Prometheus's Alerts page to see pending/firing evaluation, Alertmanager to manage silences, and `http://localhost:8090/` for received incident evidence. Absence of an incident is not a health signal: pending or silenced alerts may never notify this receiver.
+
+## Notification routing
+
+The supplied receiver writes local evidence only. Add your operator-owned notification integration to the same Alertmanager receiver, using its documented email/chat/webhook schema. For a generic endpoint that accepts Alertmanager's webhook JSON, add another entry under `webhook_configs`:
+
+```yaml
+- url: https://notifications.example.invalid/alertmanager
+  send_resolved: true
+  http_config:
+    authorization:
+      credentials_file: ./notification-token
+```
+
+Replace the `.invalid` placeholder and create the private token file before enabling it. Keep grouping by alertname/job/instance, 30s initial group delay, 5m group interval and 4h repeat interval unless your response policy calls for different values. A NetkitUnavailable alert inhibits the traffic rules for the same job/instance. Alertmanager silences affect both evidence delivery and other integrations; snapshots therefore reflect **delivered notifications**, not an exhaustive incident history. Monitor Prometheus, Alertmanager notification failures, and the receiver independently; Netkit cannot report its own total monitoring-stack failure.
+
+## Evidence contract and retention
+
+`GET /requests/evidence?signal=http5xx|transport|headers&from=<RFC3339>&to=<RFC3339>` is an admin-only metadata boundary. It returns at most 20 matching retained records and the total matching retained count. It excludes URLs, host/path names, headers, bodies and error text entirely. The receiver validates a fixed schema and discards all arbitrary alert annotations/labels, storing only known rule identity, lifecycle timestamps, finite numeric value/threshold and typed evidence.
+
+The first delivered notification snapshots the available evidence from five minutes before condition onset through delivery. This contextual window includes evaluation and pending time; it is not an immutable copy of every metric sample. Later retries and recovery update lifecycle metadata without replacing the initial evidence. An out-of-order firing retry cannot reopen a resolved incident with the same start time. A resolved notification received without an earlier firing delivery is recorded as such, with whatever evidence is available at delivery.
+
+Snapshots are atomic mode-0600 JSON files in a required private mode-0700 directory. Defaults are 100 incidents and seven days, configurable with `--capacity` (1–1000) and `--retention` (1m–30d). TTL is measured from snapshot creation, not renewed on each retry. Pruning runs on reads/writes and once per minute while the process is running; offline expiry is enforced on restart. The receiver never deletes files outside its incident naming convention. Disk errors return non-success so Alertmanager can retry.
+
+Evidence states distinguish captured, no retained matches, unavailable (including failed fetch or invalid schema), and not applicable (availability/insufficient-sample alerts). A snapshot can survive request-history clearing, eviction and proxy/receiver restarts. Incident links return explicit 404/expired state after retention. Investigation links apply the signal and snapshot time bounds to the **current** Traffic buffer; that buffer may no longer contain the saved evidence. Resolved means the alert expression stopped firing, not proof of health—scrape failure and insufficient data have separate signals.
+
+The receiver preserves metadata only. Payload reproduction, durable full traffic storage, per-route metrics, rate-limiting/stream-interruption rules and native monitor editing remain later work.
+
+## Validation
+
+```sh
+promtool check config prometheus.yml
+promtool check rules rules.yml
+promtool test rules rules.test.yml
+amtool check-config alertmanager.yml
+# From repository root, with real binaries available:
+go test -race ./internal/alerting ./internal/proxy
+go build -o /tmp/netkit-alert-test ./cmd/netkit
+python3 scripts/test-alerting-integration.py \
+  --netkit /tmp/netkit-alert-test --alertmanager /path/to/alertmanager
+```
+
+Rule tests use synthetic clocks/series to verify brief spikes, sustained failures, sample gates, low/no-data, external downtime/recovery, cancellation/inspection exclusions and latency profile gating. The integration test runs a real Alertmanager and real receiver on disposable loopback ports, verifying grouping, repeat delivery, silence suppression, recovery, deduplication and safe evidence persistence. It sends no external notifications.
+
+A scrape target removed from configuration produces no `up` series; `NetkitUnavailable` monitors configured targets, not inventory deletion. Maintain target inventory and independently monitor the monitoring stack. Missing expected traffic requires an explicit cadence policy and is not inferred from ordinary silence.
+
+References: [Prometheus alerting rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/), [rule unit tests](https://prometheus.io/docs/prometheus/latest/configuration/unit_testing_rules/), [Alertmanager configuration](https://prometheus.io/docs/alerting/latest/configuration/).