diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7dc9d11..fc29850 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,26 @@ jobs: - run: npx tsc --noEmit - run: npx eslint src - run: npm run build + - uses: actions/setup-go@v5 + with: + go-version: '1.24' + - name: Verify real demo traffic + working-directory: . + run: | + mkdir -p bin + go build -o bin/netkit-demo ./cmd/netkit + python3 examples/demo/demo.py run --skip-build > "$RUNNER_TEMP/netkit-demo.log" 2>&1 & + demo_pid=$! + trap 'kill "$demo_pid" 2>/dev/null || true; wait "$demo_pid" || true; cat "$RUNNER_TEMP/netkit-demo.log"' EXIT + for attempt in $(seq 1 60); do + kill -0 "$demo_pid" || exit 1 + if grep -q 'Demo ready:' "$RUNNER_TEMP/netkit-demo.log"; then + python3 examples/demo/demo.py verify + exit 0 + fi + sleep 1 + done + exit 1 test: name: Test and Lint diff --git a/.gitignore b/.gitignore index 7b9f377..a737d4a 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,8 @@ examples/alerting/notification-token examples/alerting/incidents/ examples/alerting/alertmanager-data/ examples/alerting/prometheus-data/ + +# Local browser recordings and demo Python cache +output/playwright/ +.playwright-cli/ +examples/demo/__pycache__/ diff --git a/Makefile b/Makefile index 4077bc8..948b872 100644 --- a/Makefile +++ b/Makefile @@ -737,3 +737,11 @@ release: # Default target all: help + +.PHONY: demo demo-verify +# Build and run the deterministic local proxy/dashboard demo. +demo: + python3 examples/demo/demo.py run + +demo-verify: + python3 examples/demo/demo.py verify diff --git a/README.md b/README.md index 3776314..f76904f 100644 --- a/README.md +++ b/README.md @@ -1,378 +1,135 @@ -# netkit +# ⚑ netkit -A modern HTTP proxy and request capture tool with a sleek web dashboard. +See what your HTTP traffic is doing. Netkit is a Go proxy with a web dashboard +for inspecting requests, finding slow responses, and investigating failures. - +## πŸš€ Try it locally -## Features +You’ll need **Go 1.24.1+, Node.js 20+, npm, and Python 3.9+** for the demo. -- **HTTP Proxy Server**: Forward HTTP requests through the proxy -- **Request History**: Comprehensive tracking of all proxied requests -- **Web Dashboard**: Modern React-based interface for managing requests -- **Real-time Statistics**: Monitor proxy performance and request metrics -- **Admin API**: RESTful endpoints for health checks, metrics, and history -- **Request Replay**: Load and replay requests from history -- **Timing Metrics**: Detailed breakdown of proxy overhead and upstream latency - -## Quick Start - -### 1. Build the Project - -```bash -make dev -``` - -Open http://localhost:3000 to access the dashboard. - -### Docker dashboard - -The published `biancarosa/netkit:0.1.0` image does not embed the dashboard. -Build the corrected image from this checkout: - -```bash -docker build -t netkit:local . -docker run --rm -p 127.0.0.1:3000:3000 -p 127.0.0.1:8080:8080 netkit:local -``` - -Open http://localhost:3000. The dashboard accesses its admin API through the -same port. Run `make test-docker-dashboard` to verify the image before publishing. - -## Dashboard Features - -### Request Builder -- **HTTP Method Selection**: GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS -- **URL Input**: Full URL with validation -- **Headers Management**: Dynamic header pairs with enable/disable toggles -- **Request Body**: JSON editor with syntax highlighting -- **Send Requests**: Execute requests through the proxy or directly - -### Request History -The dashboard provides two types of request history: - -#### Local History -- Requests made from the dashboard interface -- Stored in browser localStorage -- Includes request/response data and timing -- Replay functionality to load previous requests - -#### Proxy History -- All HTTP requests that pass through the proxy server -- Stored in server memory (configurable size) -- Detailed timing metrics (proxy overhead, upstream latency) -- Request/response size tracking -- Success/error status tracking - -### Statistics Panel -Real-time statistics showing: -- **Request Counts**: Success vs error rates -- **Timing Metrics**: Average durations and latency breakdown -- **Data Transfer**: Total request/response sizes -- **Status Codes**: Distribution of HTTP status codes -- **Methods**: Usage breakdown by HTTP method - -## API Endpoints - -### Proxy Server (Port 8080) -- **HTTP Proxy**: All HTTP traffic - -### Admin Server (Port 8081) -- `GET /healthz` - Health check -- `GET /metrics` - Prometheus-style metrics -- `GET /requests` - Request history (JSON) -- `GET /requests/stats` - Request statistics -- `POST /requests/clear` - Clear request history - -## Configuration - -### Command Line Flags - -```bash -./bin/netkit serve [flags] - -Flags: - --port int Proxy server port (default 8080) - --admin-port int Admin server port (enables admin endpoints) - --history-size int Maximum requests to keep in history (default 1000) - --dashboard Enable web dashboard (default true) - --dashboard-port int Dashboard server port (default 3000) - --dashboard-base-path Public dashboard URL prefix, for example /netkit - --log-level string Log level: debug, info, warn, error (default "info") -``` - -### Environment Variables - -The embedded production dashboard uses same-origin API routes by default: -`/api/proxy` for proxied requests and `/api/admin/*` for health, history, and metrics. - -For standalone dashboard development, you can still point the browser at separate proxy/admin ports in `dashboard/.env.local`: -```bash -NEXT_PUBLIC_PROXY_HOST=localhost -NEXT_PUBLIC_PROXY_PORT=8080 -NEXT_PUBLIC_ADMIN_PORT=8081 -``` - -For path-based reverse proxies, build and run with the same base path: - -```bash -docker build --build-arg NETKIT_DASHBOARD_BASE_PATH=/netkit -t netkit . -docker run -e NETKIT_DASHBOARD_BASE_PATH=/netkit -p 3000:3000 -p 8080:8080 -p 8081:8081 netkit -``` - -## Usage Examples - -### Using as HTTP Proxy - -```bash -# Configure your application to use the proxy -curl -x http://localhost:8080 http://httpbin.org/get - -# Or set environment variables -export HTTP_PROXY=http://localhost:8080 -export HTTPS_PROXY=http://localhost:8080 -``` - -### Request History API - -```bash -# Get request history -curl http://localhost:8081/requests - -# Get statistics -curl http://localhost:8081/requests/stats - -# Clear history -curl -X POST http://localhost:8081/requests/clear +```sh +git clone https://github.com/chaosslabs/netkit.git +cd netkit +make demo ``` -## Development - -### Prerequisites -- Go 1.21+ -- Node.js 18+ -- npm +Open **[localhost:13000](http://localhost:13000)**. The demo builds Netkit and its +dashboard, starts a local store API, and sends real traffic through the proxy: -### Development Commands +- A successful catalog request: **200** +- A JSON order submission: **201** +- A slow upstream response: **~800 ms** +- An unavailable service: **503** -```bash -# Install all development tools (Go, Node.js, git-cliff) -make install +Run `make demo-verify` to send another batch and check the captures. **Ctrl-C** +stops the demo. No external API or credentials needed. -# Install specific toolsets -make install-backend # Go tools only -make install-dashboard # Node.js dependencies only -make install-changelog-tools # git-cliff and conventional commit tools +[🎬 Watch the 48-second demo](examples/demo/netkit-demo.mp4) Β· +[Demo setup and ports](examples/demo/README.md) -# Run tests -make test -make e2e +## πŸ‘€ See it in action -# Run linting -make lint -make check +These screenshots come directly from the recorded demo at 1440Γ—1000. -# Start development servers -make dev - -# Build for production -make build -make build-dashboard -``` +**Traffic:** filter captured requests and inspect status, headers, JSON bodies, +and timing side by side. -### Conventional Commits and Changelog +![Netkit Traffic showing a captured POST order and its HTTP 201 JSON response](docs/images/demo-traffic.png) -This project uses [Conventional Commits](https://www.conventionalcommits.org/) for structured commit messages and [git-cliff](https://git-cliff.org/) for automated changelog generation. +**Compose:** send a request through the proxy and inspect its response. -#### Quick Start with Conventional Commits +![Netkit Compose showing a live catalog request with an HTTP 200 JSON response](docs/images/demo-compose.png) -```bash -# Install tools -make install-changelog-tools +**Overview** groups retained traffic by host and path, with failure counts and +p95 time to response headers. Drill into a group to inspect its requests. +[More about the investigation workflow β†’](docs/investigation-workspace.md) -# Initialize changelog configuration -make changelog-init +## πŸ”Œ Use it with your app -# Make conventional commits -make commit-feat msg="add user authentication" -make commit-fix msg="resolve proxy timeout issue" -make commit-docs msg="update API documentation" +Build a standalone binary with the dashboard included: -# Create releases -make release-patch # 1.0.0 -> 1.0.1 (bug fixes) -make release-minor # 1.0.0 -> 1.1.0 (new features) -make release-major # 1.0.0 -> 2.0.0 (breaking changes) +```sh +make install-dashboard +make build-embedded +./bin/netkit serve ``` -#### Available Commit Types - -- `feat` - New features -- `fix` - Bug fixes -- `docs` - Documentation changes -- `style` - Code formatting -- `refactor` - Code restructuring -- `test` - Test additions/modifications -- `chore` - Maintenance tasks -- `perf` - Performance improvements -- `security` - Security fixes +The normal server uses **3000** for the dashboard, **8080** for the proxy, and +**8081** for the admin API. These differ from the demo’s ports. +Run `./bin/netkit serve --help` for all configuration flags. -#### Changelog Commands +```sh +curl --proxy http://localhost:8080 http://example.com -```bash -make changelog # Generate full changelog -make changelog-update # Update with latest commits -make check-conventional-commits # Validate commit format +# For applications that support proxy environment variables: +export HTTP_PROXY=http://localhost:8080 +export HTTPS_PROXY=http://localhost:8080 ``` -For detailed information, see [docs/CHANGELOG_GUIDE.md](docs/CHANGELOG_GUIDE.md). +Prefer Docker? Build from this checkout: -### Project Structure - -``` -netkit/ -β”œβ”€β”€ cmd/netkit/ # Main application entry point -β”œβ”€β”€ internal/ -β”‚ β”œβ”€β”€ proxy/ # Proxy server implementation -β”‚ └── api/ # Admin API handlers -β”œβ”€β”€ dashboard/ # React dashboard -β”‚ β”œβ”€β”€ src/ -β”‚ β”‚ β”œβ”€β”€ components/ # React components -β”‚ β”‚ β”œβ”€β”€ services/ # API service layer -β”‚ β”‚ └── hooks/ # Custom hooks and state management -β”‚ └── public/ # Static assets -└── Makefile # Development commands +```sh +docker build -t netkit:local . +docker run --rm -p 127.0.0.1:3000:3000 -p 127.0.0.1:8080:8080 netkit:local ``` -## Request History Details - -### HTTP vs HTTPS Requests - -- **HTTP Requests**: Fully captured with complete request/response data -- **HTTPS Requests**: Only CONNECT tunnel establishment is visible (encrypted content cannot be captured) - -### Data Captured - -For each HTTP request through the proxy: -- Request method, URL, headers, body -- Response status, headers, body -- Timing breakdown: - - Proxy overhead (time spent in proxy code) - - Upstream latency (time waiting for target server) - - Total duration -- Data sizes (request and response bytes) -- Success/error status - -### Performance Considerations - -- Request history is stored in memory -- Configurable maximum size (default: 1000 requests) -- Automatic cleanup of oldest requests when limit reached -- Minimal performance impact on proxy operations - -## Contributing +Open [localhost:3000](http://localhost:3000). The dashboard accesses its admin API +through the same port. -1. Fork the repository -2. Create a feature branch -3. Make your changes -4. Run `make check` to ensure all tests pass -5. Submit a pull request +## πŸ” What gets captured? -## License +- **HTTP:** method, sanitized URL and headers, eligible JSON bodies, status, + byte counts, and timing. +- **HTTPS by default:** CONNECT tunnel setup only. Encrypted requests stay opaque. +- **Optional HTTPS inspection:** decrypted exchanges when clients trust your + inspection CA. See [HTTPS setup](docs/setup.md#https-inspection). +- **History:** a bounded in-memory buffer, cleared on restart. Default capacity: + 1,000 records; change it with `--history-size`. -MIT License - see LICENSE file for details. +Sensitive fields are masked before retention. Only complete JSON bodies up to +64 KiB are retained; other bodies show omission markers. Redaction is not a +general secret detector. See the [capture policy](docs/trustworthy-capture.md). -### HTTPS tunnel history +Netkit has no proxy/dashboard authentication and binds its listeners to all +interfaces. Keep it within a trusted development environment. -When an application uses Netkit through `HTTPS_PROXY`, history records each -`CONNECT host:port` tunnel as soon as its setup succeeds or fails. Long-lived -connections appear immediately. The status and timing describe tunnel setup, -not the encrypted upstream API response or the lifetime of the connection. -A reused tunnel can carry multiple HTTPS requests but creates one history entry. +## πŸ“Š API and monitoring -Netkit does not decrypt TLS: HTTPS paths, headers, bodies, and application status -codes are not captured. CONNECT records leave payload sizes at zero (unmeasured) -and omit request headers to avoid retaining proxy credentials. Transfer errors -after establishment do not change the setup result. Requests made through the -dashboard's destination-header API follow its separate HTTP forwarding path. +The admin API uses port **8081** by default. The dashboard also exposes these +routes under `/api/admin`. -## Maintainer operations +| Endpoint | Purpose | +| --- | --- | +| `GET /healthz` | Health and capture mode | +| `GET /requests` | Retained request history | +| `GET /requests/stats` | Statistics for retained history | +| `GET /metrics` | Prometheus counters and histograms | +| `POST /requests/clear` | Clear retained history | -See [RUNBOOK.md](RUNBOOK.md) for release validation, local troubleshooting, -CLI/container rollback, and the applicability of cost checks to this distributed tool. +History statistics cover the current buffer. Prometheus counters survive history +clearing and eviction, but reset when Netkit restarts. -### Inspect HTTPS request and response bodies +For alerts and retained incident evidence, use the +[Prometheus + Alertmanager example](examples/alerting/README.md). -Netkit can decrypt HTTPS for **every destination** when clients explicitly trust -its inspection CA. There is no domain allowlist. Without `--inspect-https`, CONNECT -continues to work as an encrypted tunnel. - -Generate a CA once and keep it across proxy restarts: +## πŸ› οΈ Development ```sh -mkdir -p "$HOME/.config/netkit" -netkit ca --cert "$HOME/.config/netkit/ca.pem" --key "$HOME/.config/netkit/ca-key.pem" -netkit serve --inspect-https \ - --ca-cert "$HOME/.config/netkit/ca.pem" \ - --ca-key "$HOME/.config/netkit/ca-key.pem" +make install-dashboard # Install frontend dependencies +make dev # Start backend and dashboard +make test # Go unit tests +make e2e # Go end-to-end tests +make check # Backend tests, lint, and build ``` -The `ca` command refuses to overwrite existing files and creates the private key -with mode `0600`. Only the public `ca.pem` belongs in client trust stores. The -private key stays with Netkit. Existing signing CA PEM files can also be loaded; -an invalid, expired, or mismatched CA fails startup. - -For Node.js, make the public certificate available to the application and restart -its process with: - -```sh -NODE_EXTRA_CA_CERTS=/path/to/ca.pem \ -NODE_OPTIONS=--use-env-proxy \ -HTTP_PROXY=http://127.0.0.1:8080 \ -HTTPS_PROXY=http://127.0.0.1:8080 \ -node your-app.js -``` - -For curl: - -```sh -curl --proxy http://127.0.0.1:8080 --cacert /path/to/ca.pem \ - https://example.com/ -``` - -Python requests uses `REQUESTS_CA_BUNDLE=/path/to/ca.pem`; Chromium requires the -CA in its own applicable trust store and an explicit proxy configuration. -Certificate-pinned applications need their pinning configuration adjusted. -Netkit still verifies upstream certificates using its system trust roots; it -does not disable TLS verification. Private upstream CAs must also be trusted by -Netkit itself. - -The existing request dashboard and `/requests` API show each decrypted HTTP -exchange, including its full URL, headers, and both bodies. Headers and bodies -are captured as received, including credentials. Keep access within your trusted -debugging environment. History remains in memory and is cleared on restart. - -Responses are streamed to clients immediately, including SSE; their history -record is saved when the exchange finishes or disconnects. Long-lived streams -therefore appear after closing. Capture stores complete bodies in memory, so -choose `--history-size` to suit payload sizes. Binary or compressed payloads are -not decoded for display. Inspected connections negotiate HTTP/1.1 (including -keep-alive); HTTP/2-only clients are not supported. WebSocket and other protocol -upgrades return 501 in inspection mode; frame inspection is not implemented. -Plain HTTP carried inside CONNECT (as used by Node fetch) is also captured. -Only traffic configured to pass through Netkit can be inspected. - -Inspection has no upstream response-header deadline by default, so long-polling -requests can wait for a response. Client cancellation still cancels the upstream -request. To impose a deadline, use `--inspection-response-header-timeout=90s` -with a duration longer than the upstream polling interval. TLS handshake and -idle-connection timeouts remain in effect. - -## Trustworthy capture and metrics - -Captured history is sanitized before retention. The dashboard separates HTTP status from transfer outcome, discloses capture/retention scope, and preserves stale data when refresh fails. Real cumulative counters and histograms survive history eviction and clearing. See [capture safety and signal semantics](docs/trustworthy-capture.md) for the redaction policy, `--redact-fields`, compatibility changes, timing definitions, and metrics. - -### Investigation workspace - -Traffic is the landing page, with preserved filters, adjacent request inspection, buffered Live/Pause, and an Overview that drills into matching retained evidence. Compose is available at `/compose/`. See [investigation workspace semantics](docs/investigation-workspace.md) for freshness, URL scope, latency populations, and retention limits. +Dashboard checks: `cd dashboard && npm run test:unit && npm run build`. +Use [Conventional Commits](docs/CHANGELOG_GUIDE.md) and open a PR with your change +and validation results. -### Focused alerting +## πŸ“š More details -Use the [Prometheus and Alertmanager example](examples/alerting/README.md) for sustained upstream/transport failure alerts, opt-in ordinary-workload header latency, external availability and insufficient-sample signals. The separate `netkit alerts` process retains bounded metadata-only incident snapshots with authenticated webhook writes. Alert evaluation and notification ownership remain external; no native monitor editor or production notification destination is enabled automatically. +- [Dashboard configuration and HTTPS setup](docs/setup.md) +- [Capture safety, outcomes, and metric definitions](docs/trustworthy-capture.md) +- [Investigation workspace](docs/investigation-workspace.md) +- [Release and troubleshooting runbook](RUNBOOK.md) +- [Changelog and release workflow](docs/CHANGELOG_GUIDE.md) diff --git a/docs/images/demo-compose.png b/docs/images/demo-compose.png new file mode 100644 index 0000000..f01d0d3 Binary files /dev/null and b/docs/images/demo-compose.png differ diff --git a/docs/images/demo-traffic.png b/docs/images/demo-traffic.png new file mode 100644 index 0000000..0b35bb6 Binary files /dev/null and b/docs/images/demo-traffic.png differ diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 0000000..be17712 --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,94 @@ +# Netkit setup + +[← Back to the README](../README.md) + +## Dashboard configuration + +The embedded production dashboard uses same-origin API routes by default: +`/api/proxy` for proxied requests and `/api/admin/*` for health, history, and metrics. + +For standalone dashboard development, set separate proxy/admin ports in +`dashboard/.env.local`: + +```bash +NEXT_PUBLIC_PROXY_HOST=localhost +NEXT_PUBLIC_PROXY_PORT=8080 +NEXT_PUBLIC_ADMIN_PORT=8081 +``` + +For path-based reverse proxies, build and run with the same base path: + +```bash +docker build --build-arg NETKIT_DASHBOARD_BASE_PATH=/netkit -t netkit . +docker run -e NETKIT_DASHBOARD_BASE_PATH=/netkit -p 127.0.0.1:3000:3000 -p 127.0.0.1:8080:8080 netkit +``` + +## HTTPS inspection + +Netkit can decrypt HTTPS for **every destination** when clients explicitly trust +its inspection CA. There is no domain allowlist. Without `--inspect-https`, CONNECT +continues to work as an encrypted tunnel. + +The commands below assume `netkit` is on your PATH. For a local build, use +`./bin/netkit` instead. Generate a CA once and keep it across proxy restarts: + +```sh +mkdir -p "$HOME/.config/netkit" +netkit ca --cert "$HOME/.config/netkit/ca.pem" --key "$HOME/.config/netkit/ca-key.pem" +netkit serve --inspect-https \ + --ca-cert "$HOME/.config/netkit/ca.pem" \ + --ca-key "$HOME/.config/netkit/ca-key.pem" +``` + +The `ca` command refuses to overwrite existing files and creates the private key +with mode `0600`. Only the public `ca.pem` belongs in client trust stores. The +private key stays with Netkit. Existing signing CA PEM files can also be loaded; +an invalid, expired, or mismatched CA fails startup. + +For Node.js, make the public certificate available to the application and restart +its process with: + +```sh +NODE_EXTRA_CA_CERTS=/path/to/ca.pem \ +NODE_OPTIONS=--use-env-proxy \ +HTTP_PROXY=http://127.0.0.1:8080 \ +HTTPS_PROXY=http://127.0.0.1:8080 \ +node your-app.js +``` + +For curl: + +```sh +curl --proxy http://127.0.0.1:8080 --cacert /path/to/ca.pem \ + https://example.com/ +``` + +Python requests uses `REQUESTS_CA_BUNDLE=/path/to/ca.pem`; Chromium requires the +CA in its own applicable trust store and an explicit proxy configuration. +Certificate-pinned applications need their pinning configuration adjusted. +Netkit still verifies upstream certificates using its system trust roots; it +does not disable TLS verification. Private upstream CAs must also be trusted by +Netkit itself. + +The dashboard and `/requests` API show sanitized decrypted exchanges. Sensitive +headers, URL values, and JSON fields are masked before retention. Only complete +JSON bodies up to 64 KiB are retained; other bodies have omission markers. This +is a field/pattern policy, not a general secret detector. See the +[capture policy](trustworthy-capture.md) for details. History is in memory and +is cleared on restart. + +Responses are streamed to clients immediately, including SSE; their history +record is saved when the exchange finishes or disconnects. Long-lived streams +therefore appear after closing. Capture buffers are bounded, and byte counts track the observed stream. +Binary or compressed payloads are not decoded for display. Inspected connections negotiate HTTP/1.1 (including +keep-alive); HTTP/2-only clients are not supported. WebSocket and other protocol +upgrades return 501 in inspection mode; frame inspection is not implemented. +Plain HTTP carried inside CONNECT (as used by Node fetch) is also captured. +Only traffic configured to pass through Netkit can be inspected. + +Inspection has no upstream response-header deadline by default, so long-polling +requests can wait for a response. Client cancellation still cancels the upstream +request. To impose a deadline, use `--inspection-response-header-timeout=90s` +with a duration longer than the upstream polling interval. TLS handshake and +idle-connection timeouts remain in effect. + diff --git a/examples/demo/README.md b/examples/demo/README.md new file mode 100644 index 0000000..beb5f0b --- /dev/null +++ b/examples/demo/README.md @@ -0,0 +1,84 @@ +# 🎬 Local Netkit demo + +Run the real proxy and dashboard against a small local store API. No external API, +credentials, or Docker needed. The first build downloads dependencies; all demo +traffic stays on your machine. + +[Watch the recorded walkthrough](netkit-demo.mp4) (48 seconds, 1440Γ—1000 at 30 fps, real browser capture). + +## πŸš€ Start + +Requirements: Go 1.24.1+, Node.js 20+, npm, and Python 3.9+. + +From the repository root: + +```sh +make demo +``` + +Open **http://localhost:13000**. The launcher builds the dashboard and Netkit, starts +both servers, sends four real HTTP exchanges through the forward proxy, and verifies +the resulting captures. It refuses to start if a demo port is occupied. Ctrl-C stops +both servers; history is in memory and disappears on shutdown. + +Netkit currently binds its proxy, admin, and dashboard listeners to all interfaces. +Run this on a trusted development machine; do not expose these unauthenticated ports +to the Internet. The synthetic upstream binds only to loopback. + +| Service | Address | +| --- | --- | +| Dashboard and same-origin API | http://localhost:13000 | +| HTTP forward proxy | http://localhost:18080 | +| Admin API and metrics | http://localhost:18081 | +| Synthetic store API | http://127.0.0.1:18090 | + +To reuse your last build, run `python3 examples/demo/demo.py run --skip-build`. +To send another batch and recheck captures while the environment runs: + +```sh +make demo-verify +``` + +## πŸ‘€ Walkthrough + +1. In **Traffic**, inspect `GET /products`: HTTP 200 with a JSON catalog. +2. Inspect `POST /orders`: HTTP 201, the captured JSON request, and the upstream's + response containing `order_id: demo-1042` and the original request payload. +3. Inspect `GET /slow`: HTTP 200 after an intentional 800 ms upstream delay. +4. Inspect `GET /unavailable`: an actual upstream HTTP 503 with a JSON error. +5. Open **Overview** to see the same real exchanges aggregated. +6. In the request builder, send `GET http://127.0.0.1:18090/products` and return to + Traffic to inspect the new capture. + +You can also use a normal proxy client. `--noproxy ''` is deliberate: curl otherwise +may bypass the proxy for loopback destinations. + +```sh +curl --noproxy '' --proxy http://localhost:18080 http://127.0.0.1:18090/products +curl http://localhost:18081/metrics +``` + +[See screenshots from the video β†’](../../README.md#-see-it-in-action) + +This demo covers plain HTTP capture. It does not demonstrate HTTPS interception, +transport failures, alert delivery, or production deployment. + +## πŸŽ₯ Record the dashboard + +With [Playwright CLI](https://github.com/microsoft/playwright-cli) installed: + +```sh +mkdir -p output/playwright +npx playwright install ffmpeg +playwright-cli -s=netkit-demo open http://localhost:13000 --headed +playwright-cli -s=netkit-demo resize 1440 1000 +playwright-cli -s=netkit-demo video-start output/playwright/netkit-demo.webm --size=1440x1000 --fps=30 --cursor +playwright-cli -s=netkit-demo snapshot +# Use the displayed element refs with click/fill to follow the walkthrough above. +playwright-cli -s=netkit-demo video-stop +``` + +Set `--size` explicitly to match the viewport; otherwise Playwright downscales +the video to fit 800Γ—800. The recording captures the running browser. It is not a mock UI or a replay of +fabricated responses. Working recordings under `output/playwright/` are ignored by Git. The reviewed +`netkit-demo.mp4` walkthrough is included here for convenient playback. diff --git a/examples/demo/demo.py b/examples/demo/demo.py new file mode 100644 index 0000000..d443253 --- /dev/null +++ b/examples/demo/demo.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Run a real Netkit demo with a deterministic, local-only upstream.""" +import argparse +import http.client +import json +from pathlib import Path +import signal +import socket +import subprocess +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +ROOT = Path(__file__).resolve().parents[2] +PROXY, ADMIN, DASHBOARD, UPSTREAM = 18080, 18081, 13000, 18090 + + +class Upstream(BaseHTTPRequestHandler): + def do_GET(self): + status = 200 + payload = {"service": "demo-store", "products": ["keyboard", "headphones"]} + if self.path == "/slow": + time.sleep(0.8) + payload = {"message": "Slow upstream completed", "delay_ms": 800} + elif self.path == "/unavailable": + status, payload = 503, {"error": "Demo inventory unavailable", "retryable": True} + elif self.path not in ("/products", "/healthz"): + status, payload = 404, {"error": "Not found"} + self.respond(status, payload) + + def do_POST(self): + if self.path != "/orders": + self.respond(404, {"error": "Not found"}) + return + try: + if self.headers.get("Transfer-Encoding", "").lower() == "chunked": + chunks = [] + while True: + size = int(self.rfile.readline().split(b";", 1)[0], 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + break + chunks.append(self.rfile.read(size)) + if self.rfile.read(2) != b"\r\n": + raise ValueError("Invalid chunk terminator") + raw = b"".join(chunks) + else: + raw = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + body = json.loads(raw) + except (ValueError, json.JSONDecodeError): + self.respond(400, {"error": "Expected JSON"}) + return + self.respond(201, {"order_id": "demo-1042", "received": body}) + + def respond(self, status, payload): + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + +def request(port, method, path, body=None, headers=None): + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + try: + connection.request(method, path, body, headers or {}) + response = connection.getresponse() + return response.status, response.read() + finally: + connection.close() + + +def verify(): + """Send actual forward-proxy traffic, then check captured records and metrics.""" + scenarios = [("GET", "/products", None, 200), + ("POST", "/orders", json.dumps({"item": "keyboard", "quantity": 2}), 201), + ("GET", "/slow", None, 200), + ("GET", "/unavailable", None, 503)] + for method, path, body, expected in scenarios: + url = f"http://127.0.0.1:{UPSTREAM}{path}" + previous = {r["id"] for r in json.loads(request(ADMIN, "GET", "/requests")[1])["records"]} + status, response = request(PROXY, method, url, body, {"Content-Type": "application/json"}) + assert status == expected, (path, status, response) + # Confirm the response really came from our upstream. + payload = json.loads(response) + if path == "/orders": + assert payload["received"] == json.loads(body) + record = None + for _ in range(40): + records = json.loads(request(ADMIN, "GET", "/requests")[1])["records"] + record = next((r for r in records if r["url"] == url and + r["method"] == method and r["id"] not in previous), None) + if record is not None: + break + time.sleep(0.05) + assert record is not None, f"No fresh capture for {method} {url}" + assert record["response_status"] == expected, record + assert json.loads(record["response_body"]) == payload, record + if body: + assert json.loads(record["request_body"]) == json.loads(body), record + if path == "/slow": + assert record["upstream_latency_us"] >= 700_000, record + print(f"PASS {method:4} {path:14} HTTP {status} captured; {record['total_duration_us'] / 1000:.1f} ms", flush=True) + status, metrics = request(ADMIN, "GET", "/metrics") + assert status == 200 and b"netkit" in metrics + stats = json.loads(request(ADMIN, "GET", "/requests/stats")[1]) + assert stats["total_requests"] >= len(scenarios), stats + print("PASS history, JSON bodies, upstream latency, statistics and Prometheus metrics", flush=True) + + +def run(skip_build): + # Fail before building or seeding if another process owns a demo port. + for port in (PROXY, ADMIN, DASHBOARD, UPSTREAM): + with socket.socket() as probe: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + probe.bind(("", port)) + binary = ROOT / "bin/netkit-demo" + if not skip_build: + subprocess.run(["npm", "ci"], cwd=ROOT / "dashboard", check=True) + subprocess.run(["npm", "run", "build"], cwd=ROOT / "dashboard", check=True) + binary.parent.mkdir(exist_ok=True) + subprocess.run(["go", "build", "-o", str(binary), "./cmd/netkit"], cwd=ROOT, check=True) + if not binary.is_file() or not (ROOT / "dashboard/out/index.html").is_file(): + raise RuntimeError("Demo build missing; run make demo first") + server = ThreadingHTTPServer(("127.0.0.1", UPSTREAM), Upstream) + threading.Thread(target=server.serve_forever, daemon=True).start() + process = None + try: + process = subprocess.Popen([str(binary), "serve", "--port", str(PROXY), + "--admin-port", str(ADMIN), "--dashboard-port", str(DASHBOARD), + "--dashboard-dir", str(ROOT / "dashboard/out"), "--history-size", "200", + "--log-level", "error"], cwd=ROOT) + for _ in range(100): + if process.poll() is not None: + raise RuntimeError("Netkit exited before becoming ready") + try: + if request(DASHBOARD, "GET", "/api/admin/healthz")[0] == 200: + break + except OSError: + pass + time.sleep(0.1) + else: + raise RuntimeError("Netkit readiness timed out") + verify() + print(f"\nDemo ready: http://localhost:{DASHBOARD}\nCtrl-C stops both servers.\n" + "Netkit listeners bind all interfaces; use a trusted development machine.", flush=True) + if process.wait() != 0: + raise RuntimeError("Netkit exited unexpectedly") + finally: + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + server.shutdown() + server.server_close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("run", "verify"), nargs="?", default="run") + parser.add_argument("--skip-build", action="store_true", help="Reuse the last local demo build") + args = parser.parse_args() + def stop(*_args): + raise KeyboardInterrupt + + signal.signal(signal.SIGTERM, stop) + try: + run(args.skip_build) if args.action == "run" else verify() + except KeyboardInterrupt: + pass diff --git a/examples/demo/netkit-demo.mp4 b/examples/demo/netkit-demo.mp4 new file mode 100644 index 0000000..71d8a17 Binary files /dev/null and b/examples/demo/netkit-demo.mp4 differ