diff --git a/.github/scripts/cli-e2e.sh b/.github/scripts/cli-e2e.sh index 27ed5fc..8ec2cb1 100755 --- a/.github/scripts/cli-e2e.sh +++ b/.github/scripts/cli-e2e.sh @@ -153,4 +153,16 @@ esac lustro_to net-get-short.json --json net get "$short" check transaction "$RULES" < "$OUT/net-get-short.json" +# net export takes the short id too, and writes the same entry that a full export has. +step "lustro net export --har: the mocked request, then every transaction" +lustro net export --har "$OUT/export-one.har" --ids "$short" +check har "$RULES" < "$OUT/export-one.har" +entries="$("$OUT/venv/bin/python" -c 'import json,sys; print(len(json.load(sys.stdin)["log"]["entries"]))' < "$OUT/export-one.har")" +if [ "$entries" != "1" ]; then + echo "error: --ids $short exported $entries entries" >&2 + exit 1 +fi +lustro net export --har "$OUT/export-all.har" +check har "$RULES" < "$OUT/export-all.har" + step "The CLI end to end passed" diff --git a/.github/scripts/cli_e2e_check.py b/.github/scripts/cli_e2e_check.py index 61aa401..ae47b31 100755 --- a/.github/scripts/cli_e2e_check.py +++ b/.github/scripts/cli_e2e_check.py @@ -129,6 +129,19 @@ def check_transaction(text: str, rules: List[dict]) -> None: expect(tx["responseBody"] == rule["responseBody"], "body {!r}, the rule sets {!r}".format(tx["responseBody"], rule["responseBody"])) +def check_har(text: str, rules: List[dict]) -> None: + """``net export --har`` writes a HAR document; the request the first rule served is in it.""" + har = json.loads(text) + component("HarDocument").validate(har) + rule = rules[0] + entries = [entry for entry in har["log"]["entries"] if rule["urlPattern"] in entry["request"]["url"]] + expect(bool(entries), "no entry matches {}".format(rule["urlPattern"])) + for entry in entries: + expect(entry["_lustro"]["isMocked"] is True, "the rule did not serve {}".format(entry["_lustro"]["id"])) + expect(entry["response"]["status"] == rule["statusCode"], "status {}".format(entry["response"]["status"])) + expect(entry["response"]["content"].get("text") == rule["responseBody"], "body {!r}".format(entry["response"]["content"].get("text"))) + + CHECKS: Dict[str, Callable[[str, List[dict]], None]] = { "open": check_open, "meta": check_meta, @@ -139,6 +152,7 @@ def check_transaction(text: str, rules: List[dict]) -> None: "state": check_state, "find": find, "transaction": check_transaction, + "har": check_har, } diff --git a/CHANGELOG.md b/CHANGELOG.md index 171ff1d..c036acf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -197,6 +197,28 @@ see [DECISIONS.md](DECISIONS.md). `docs/STYLEGUIDE.md`. The JavaScript tests cover the escaping of each viewer. The sample has a request for each viewer, a PNG and a BMP that it sends as request bodies, and a request that sends a 250 KB JSON body. +- **Export captured traffic as HAR and Markdown.** A transaction left Lustro + only as JSON from the API, as a cURL command, or as plain text, one at a + time. `GET network/transactions/_/export?format=har&ids=...` returns the + transactions as a HAR 1.2 document, oldest first, and every transaction when + `ids` is left out (wire protocol 1.2). It is built from the store, so it has + the redacted values. Each entry spends its `durationMs` in `timings.wait`, + `_resourceType` lets Chrome DevTools list it under Fetch/XHR or Img, a body + kept as bytes is base64, and `_lustro` carries the transaction id, + `isMocked`, `categories`, the truncation flags, `responseComplete`, and + `error`. `lustro net export --har FILE [--ids ID ...]` saves it, and sends + many ids in batches, because a request line must stay under 8 KB. In the + Network tab, **Select** adds a checkbox to each row, a checkbox in the header + that selects every row the filters show, and a bar with the count, **Export + HAR**, which saves a file, and **Copy Markdown**. A filter change keeps only + the selected rows it still shows. The detail's new **Markdown** button copies + one transaction: the method and URL as a heading, the headers in `http` + blocks, and each body in a block tagged with a language for its content type, + JSON indented without changing a value. Both formats mark a body that the + capture cut short. The OpenAPI document describes the route and the HAR + shape, the golden fixture `export-har.json` shows it, a unit test checks that + the export writes exactly that fixture, and the CLI end-to-end run exports + the sample's mocked request. ### Fixed diff --git a/README.md b/README.md index e1af7de..f8f63c2 100644 --- a/README.md +++ b/README.md @@ -248,6 +248,16 @@ The panel reports only the status and outcome, so the sender reads at most download or an endless stream cannot exhaust the app's heap. A send still running when the per-request timeout answers `504` has its call cancelled right after. +## Export + +The Network tab's **Select** button adds a checkbox to each row. **Export HAR** saves the selected +requests as a HAR 1.2 file, which browser devtools and most HTTP tools import, and +**Copy Markdown** copies them as one Markdown document for a bug report, a pull request, or a +chat. The detail's **Markdown** button copies one request. A filter change keeps only the selected +requests that it still shows, so an export has the rows on screen. Both formats have the redacted +values that the tab shows, and say when the capture cut a body short. Agents and scripts get the +HAR from `GET /api/v1/network/transactions/_/export` or `lustro net export --har FILE`. + ## Mock rules The Network tab's **Mock Rules** panel short-circuits matching requests with a synthetic diff --git a/docs/AGENTS.md b/docs/AGENTS.md index a62fb56..aeed4cc 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -38,10 +38,11 @@ lustro open --print-only # forward the port; exits 1 when adb forward f lustro net list # the newest 50 transactions, newest first lustro net get --no-body # one transaction, without its bodies lustro net wait --url /v1/orders -- adb shell input tap 540 1200 +lustro net export --har out.har # every transaction as a HAR file ``` Each row of `net list` starts with a short transaction id: the first 8 characters of the id, or more -when two listed ids start with the same 8. `net get` and `net body` take the short id, the full id, +when two listed ids start with the same 8. `net get`, `net body`, and `net export --ids` take the short id, the full id, or any start of an id that only one transaction has. When more than one id starts with it, they print the matches and exit 2; when no id does, they exit 1. JSON output has the full id, which the wire routes in the curl examples need. The filters `--url`, `--method`, `--status` (a code such as @@ -179,6 +180,7 @@ below summarizes it. All routes are token-authenticated and use the shared error | Poll transactions | `GET transactions?cursor=&search=` | Cursor envelope. First poll (no/invalid cursor) → `reset` with the full list; cursor unchanged → `unchanged` (items omitted); after a change → `delta`. `search` filters case-insensitively over URL, method, and bodies. Carries a top-level `state` `{ paused, overwriteMode, throttleDelayMs, captureFilter }`. | | Transaction detail | `GET transactions/{id}` | Full object (headers + bodies, with truncation flags). Enveloped `404` if missing. | | Transaction body | `GET transactions/{id}/body/{request\|response}` | One body as it is stored: the bytes of an image, or the redacted text as UTF-8. Not JSON. Enveloped `404` when no body was kept. See below. | +| Export as HAR | `GET transactions/_/export?format=har&ids=` | A HAR 1.2 document of the transactions in `ids` (comma-separated), or of every one. See below. | | Clear | `POST clear` | Clears the captured list and starts the capture filter's counts again; mock rules and settings are preserved. | | List rules | `GET rules` | `{ items: [MockRule...] }`. | | Add / upsert rule | `POST rules` | Body `MockRuleInput` (`urlPattern` required). Supplying a stable `id` makes the write **idempotent** (upsert by id); omitting it generates one. Returns `{ status: "ok", id }`. | @@ -242,6 +244,18 @@ body, or a binary type other than an image. The redactor never sees an image, so unredacted. `lustro net body [request|response] -o FILE` wraps the route. These fields and the route are part of protocol 1.2. +**Export as HAR.** `GET transactions/_/export?format=har&ids=,` returns the transactions as a +HAR 1.2 document, oldest first, which browser devtools, proxies, and HAR viewers import. It is +built from the store, so it has the same redacted values as the detail. Without `ids` it has every +transaction; an id the app no longer has is left out. Each entry's `_lustro` has the transaction +`id`, `isMocked`, `categories`, `requestBodyTruncated`, `responseBodyTruncated`, +`responseComplete`, and `error`. The capture has no phase timings, so each entry spends its whole +`durationMs` in `timings.wait`, and `_resourceType` (`fetch`, or `image` for an image) tells +Chrome DevTools how to file it. A body kept as bytes is base64: `content.encoding` says so for a +response, and `postData._encoding` for a request. The request line and its headers must stay +under 8 KB, so send many ids in batches; `lustro net export --har FILE [--ids ID ...]` does that, +and `--har -` writes the document to stdout. This route is part of protocol 1.2. + **Compressed bodies.** A body sent or received with `Content-Encoding: gzip`, `x-gzip`, or `deflate` is stored inflated: `requestBody` and `responseBody` hold the text, and `...BodyTruncated` says whether the inflated body passed the capture cap. The headers still name diff --git a/llms.txt b/llms.txt index 99e54cc..689347d 100644 --- a/llms.txt +++ b/llms.txt @@ -27,7 +27,7 @@ Facts to get right when you add Lustro to an app: ## Tools - [lustro CLI](https://raw.githubusercontent.com/Twinsen81/Lustro/main/lustro-cli/README.md): a Python command-line client for the HTTP API. `lustro open` forwards the port and opens the console with the token -- Agents: use the `lustro` CLI rather than curl, and keep its output small with `lustro net list --errors --last 20`, `lustro net get --no-body`, and `lustro net body | head -c 4000`. `lustro net wait --url -- ` runs the command and waits for the request it causes +- Agents: use the `lustro` CLI rather than curl, and keep its output small with `lustro net list --errors --last 20`, `lustro net get --no-body`, and `lustro net body | head -c 4000`. `lustro net wait --url -- ` runs the command and waits for the request it causes, and `lustro net export --har out.har [--ids ...]` saves transactions as a HAR file ## Optional diff --git a/lustro-cli/README.md b/lustro-cli/README.md index e8e30e1..68faec0 100644 --- a/lustro-cli/README.md +++ b/lustro-cli/README.md @@ -65,6 +65,7 @@ discovery), `--json` (JSON output). They go before or after the command: | `lustro net wait [-- COMMAND]` | cursor loop; runs `COMMAND`, prints the first matching request that finishes, and exits | | `lustro net get ` | `GET network/transactions/`, with each body cut at 2 KB | | `lustro net body [request\|response] [-o FILE]` | `GET network/transactions//body/`: saves a body as it is stored, the response by default; stdout without `-o` | +| `lustro net export --har FILE [--ids ID ...]` | `GET network/transactions/_/export`: saves the transactions, or only those in `--ids`, as a HAR file; stdout for `--har -` | | `lustro net clear` | `POST network/clear` | | `lustro net pause` | `POST network/pause` | | `lustro net overwrite` | `POST network/overwrite-mode` | @@ -111,8 +112,8 @@ The output is small by default, because an agent reads all of it. `ERR` for a failed request), the duration, and the URL. The short id is the first 8 characters of the id, or more characters when two listed ids start with the same 8. JSON output has the full id. -- **Ids.** `net get` and `net body` take a full id, or the start of one, such as - the short id of a row. They find the id that starts with it in the transaction +- **Ids.** `net get`, `net body`, and `net export --ids` take a full id, or the + start of one, such as the short id of a row. They find the id that starts with it in the transaction list, which is one more request. When more than one id starts with it, they print the matches and exit 2. When no id starts with it, they exit 1. The wire routes take only the full id. diff --git a/lustro-cli/src/lustro_cli/cli.py b/lustro-cli/src/lustro_cli/cli.py index 15d9639..1b71440 100644 --- a/lustro-cli/src/lustro_cli/cli.py +++ b/lustro-cli/src/lustro_cli/cli.py @@ -37,6 +37,7 @@ NETWORK = "/api/v1/network" TRANSACTIONS = NETWORK + "/transactions" +EXPORT = TRANSACTIONS + "/_/export" # The flags that every command takes, before or after its name, and their values # when they are not given. See _global_options for why main() sets these. @@ -55,6 +56,9 @@ # How many of the matches an ambiguous id prints. AMBIGUOUS_ROWS = 10 ID_HELP = "a transaction id, or its start, such as the short id of a row" +# How many ids one export request names. The server takes a request line and +# headers of up to 8 KB, and a full id is 36 characters. +EXPORT_BATCH = 50 # The Lustro runtime makes each transaction id with UUID.randomUUID(). An id of # this shape goes to the server as it is, without the list request that a @@ -233,11 +237,27 @@ def _resolve_id(client: LustroClient, given: str) -> str: """The full id of the transaction that ``given`` names: the id itself, or the start of only one id, such as the short id of a row. A start costs one list request, because the wire routes take only a full id.""" - if _FULL_ID.fullmatch(given): - return given if not given: raise UsageError("expected a transaction id, or the start of one") - items = _list_items(client.get(TRANSACTIONS)) + return _resolve_ids(client, [given])[0] + + +def _resolve_ids(client: LustroClient, given: List[str]) -> List[str]: + """The full id of each transaction in ``given``, as _resolve_id finds it. + One list request covers every start.""" + items: Optional[List[dict]] = None + resolved = [] + for start in given: + if _FULL_ID.fullmatch(start): + resolved.append(start) + continue + if items is None: + items = _list_items(client.get(TRANSACTIONS)) + resolved.append(_match_id(items, start)) + return resolved + + +def _match_id(items: List[dict], given: str) -> str: matches = [tx for tx in items if isinstance(tx.get("id"), str) and tx["id"].startswith(given)] # An id of another shape can also be the start of a longer one. if any(tx["id"] == given for tx in matches): @@ -274,6 +294,13 @@ def _status_range(value: str) -> Tuple[int, int]: raise argparse.ArgumentTypeError("expected a status code such as 404, or a class such as 4xx") +def _id_list(value: str) -> List[str]: + ids = [part.strip() for part in value.split(",") if part.strip()] + if not ids: + raise argparse.ArgumentTypeError("expected transaction ids, or their starts, separated by commas") + return ids + + def _field_names(value: str) -> List[str]: names = [name.strip() for name in value.split(",") if name.strip()] if not names: @@ -727,6 +754,71 @@ def cmd_net_body(args: argparse.Namespace) -> int: return 0 +def _export_entries(har: Any) -> List[Any]: + entries = har.get("log", {}).get("entries") if isinstance(har, dict) else None + if not isinstance(entries, list): + raise LustroError("invalid_response", "the export is not a HAR document: it has no log.entries") + return entries + + +def _export_har(client: LustroClient, ids: Optional[List[str]]) -> Any: + """The HAR document of the transactions that ``ids`` names, or of every one. + Many ids go in batches, and the entries of each batch join the first one's.""" + if ids is None: + har = client.get(EXPORT, params={"format": "har"}) + _export_entries(har) + return har + har = None + for start in range(0, len(ids), EXPORT_BATCH): + batch = client.get(EXPORT, params={"format": "har", "ids": ",".join(ids[start : start + EXPORT_BATCH])}) + entries = _export_entries(batch) + if har is None: + har = batch + else: + _export_entries(har).extend(entries) + # Each batch is oldest first, and the times are ISO 8601 in UTC, so they sort as text. + _export_entries(har).sort(key=lambda entry: str(entry.get("startedDateTime", ""))) + return har + + +def cmd_net_export(args: argparse.Namespace) -> int: + """Save captured transactions as a HAR file, or write it to stdout.""" + client = _build_client(args) + ids = None + if args.ids is not None: + # In the order given, each id once. + ids = list(dict.fromkeys(_resolve_ids(client, [given for group in args.ids for given in group]))) + har = _export_har(client, ids) + entries = _export_entries(har) + # Indented, as browser devtools write a HAR file, and as the console saves one. + data = (json.dumps(har, indent=2, ensure_ascii=False) + "\n").encode("utf-8") + if args.har == "-": + sys.stdout.buffer.write(data) + sys.stdout.flush() + else: + try: + with open(args.har, "wb") as fh: + fh.write(data) + except OSError as exc: + print("could not write HAR file {}: {}".format(args.har, exc), file=sys.stderr) + return 2 + exported = {entry.get("_lustro", {}).get("id") for entry in entries if isinstance(entry, dict)} + missing = [tx_id for tx_id in ids or [] if tx_id not in exported] + if missing: + print( + "warning: the app no longer has {} of the transactions, so the file leaves them out: {}".format( + len(missing), ", ".join(missing) + ), + file=sys.stderr, + ) + if args.har != "-": + if args.json: + _emit({"path": args.har, "entries": len(entries), "bytes": len(data), "missing": missing}, raw_json=True) + else: + print("saved {} transactions ({} bytes) to {}".format(len(entries), len(data), args.har)) + return 0 + + def cmd_net_clear(args: argparse.Namespace) -> int: client = _build_client(args) _emit(client.post(NETWORK + "/clear"), raw_json=args.json) @@ -1016,6 +1108,27 @@ def build_parser() -> argparse.ArgumentParser: n_body.add_argument("-o", "--output", default=None, metavar="FILE", help="write the body to FILE (default: stdout)") n_body.set_defaults(func=cmd_net_body) + n_export = net_sub.add_parser( + "export", + parents=[common], + help="GET transactions/_/export: save transactions as a HAR file", + description="Save the captured transactions as a HAR 1.2 file, which browser devtools and HTTP tools " + "import. Headers and bodies are redacted as in the app. Each entry has the transaction id, and what HAR " + "has no field for, in _lustro.", + ) + n_export.add_argument( + "--har", required=True, metavar="FILE", help="write the HAR document to FILE, or to stdout for -" + ) + n_export.add_argument( + "--ids", + nargs="+", + type=_id_list, + default=None, + metavar="ID", + help="only these transactions, separated by spaces or commas; each is {} (default: every one)".format(ID_HELP), + ) + n_export.set_defaults(func=cmd_net_export) + n_clear = net_sub.add_parser("clear", parents=[common], help="POST clear") n_clear.set_defaults(func=cmd_net_clear) diff --git a/lustro-cli/src/lustro_cli/wire/golden/export-har.json b/lustro-cli/src/lustro_cli/wire/golden/export-har.json new file mode 100644 index 0000000..c0a7ff0 --- /dev/null +++ b/lustro-cli/src/lustro_cli/wire/golden/export-har.json @@ -0,0 +1,339 @@ +{ + "log": { + "version": "1.2", + "creator": { + "name": "Lustro", + "version": "0.1.0" + }, + "entries": [ + { + "startedDateTime": "2026-09-28T14:22:09.003Z", + "time": 322, + "request": { + "method": "POST", + "url": "https://api.example.com/v1/orders", + "httpVersion": "h2", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "name": "Authorization", + "value": "[REDACTED]" + } + ], + "queryString": [], + "postData": { + "mimeType": "application/json; charset=utf-8", + "text": "{\"sku\":\"A-1\",\"quantity\":2}" + }, + "headersSize": -1, + "bodySize": 26 + }, + "response": { + "status": 201, + "statusText": "", + "httpVersion": "h2", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "name": "Location", + "value": "/v1/orders/9c2e" + } + ], + "content": { + "size": 54, + "mimeType": "application/json; charset=utf-8", + "text": "{\"id\":\"9c2e\",\"sku\":\"A-1\",\"quantity\":2,\"status\":\"open\"}" + }, + "redirectURL": "/v1/orders/9c2e", + "headersSize": -1, + "bodySize": 54 + }, + "cache": {}, + "timings": { + "blocked": -1, + "dns": -1, + "connect": -1, + "ssl": -1, + "send": 0, + "wait": 322, + "receive": 0 + }, + "_resourceType": "fetch", + "_lustro": { + "id": "tx_4b1d77a0", + "isMocked": false, + "categories": [ + "api" + ], + "requestBodyTruncated": false, + "responseBodyTruncated": false, + "responseComplete": true, + "error": null + } + }, + { + "startedDateTime": "2026-09-28T14:22:11.781Z", + "time": 41, + "request": { + "method": "GET", + "url": "https://api.example.com/v1/orders/77e2", + "httpVersion": "", + "cookies": [], + "headers": [ + { + "name": "Accept", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "[REDACTED]" + } + ], + "queryString": [], + "headersSize": -1, + "bodySize": 0 + }, + "response": { + "status": 500, + "statusText": "", + "httpVersion": "", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "application/json" + } + ], + "content": { + "size": 16, + "mimeType": "application/json", + "text": "{\"error\":\"boom\"}" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 16 + }, + "cache": {}, + "timings": { + "blocked": -1, + "dns": -1, + "connect": -1, + "ssl": -1, + "send": 0, + "wait": 41, + "receive": 0 + }, + "_resourceType": "fetch", + "_lustro": { + "id": "tx_77e2c014", + "isMocked": true, + "categories": [ + "api" + ], + "requestBodyTruncated": false, + "responseBodyTruncated": false, + "responseComplete": true, + "error": null + } + }, + { + "startedDateTime": "2026-09-28T14:22:13.140Z", + "time": 35, + "request": { + "method": "GET", + "url": "https://cdn.example.com/pixel.gif", + "httpVersion": "h2", + "cookies": [], + "headers": [ + { + "name": "Accept", + "value": "image/*" + } + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1 + }, + "response": { + "status": 200, + "statusText": "", + "httpVersion": "h2", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "image/gif" + }, + { + "name": "Content-Length", + "value": "43" + } + ], + "content": { + "size": 43, + "mimeType": "image/gif", + "text": "R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==", + "encoding": "base64" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 43 + }, + "cache": {}, + "timings": { + "blocked": -1, + "dns": -1, + "connect": -1, + "ssl": -1, + "send": 0, + "wait": 35, + "receive": 0 + }, + "_resourceType": "image", + "_lustro": { + "id": "tx_e6b0d413", + "isMocked": false, + "categories": [ + "media" + ], + "requestBodyTruncated": false, + "responseBodyTruncated": false, + "responseComplete": true, + "error": null + } + }, + { + "startedDateTime": "2026-09-28T14:22:14.022Z", + "time": 1210, + "request": { + "method": "GET", + "url": "https://api.example.com/v1/catalog?page=2&tag=new%20in", + "httpVersion": "h2", + "cookies": [], + "headers": [ + { + "name": "Accept", + "value": "application/json" + } + ], + "queryString": [ + { + "name": "page", + "value": "2" + }, + { + "name": "tag", + "value": "new in" + } + ], + "headersSize": -1, + "bodySize": -1 + }, + "response": { + "status": 200, + "statusText": "", + "httpVersion": "h2", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "application/json; charset=utf-8" + } + ], + "content": { + "size": 524288, + "mimeType": "application/json; charset=utf-8", + "text": "[{\"sku\":\"A-1\",\"name\":\"Desk lamp\"},{\"sku\":\"A-2\",\"na" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 524288 + }, + "cache": {}, + "timings": { + "blocked": -1, + "dns": -1, + "connect": -1, + "ssl": -1, + "send": 0, + "wait": 1210, + "receive": 0 + }, + "_resourceType": "fetch", + "_lustro": { + "id": "tx_5e8a2f61", + "isMocked": false, + "categories": [ + "api" + ], + "requestBodyTruncated": false, + "responseBodyTruncated": true, + "responseComplete": true, + "error": null + } + }, + { + "startedDateTime": "2026-09-28T14:22:15.917Z", + "time": 8, + "request": { + "method": "GET", + "url": "https://telemetry.example.com/v1/events", + "httpVersion": "", + "cookies": [], + "headers": [ + { + "name": "Accept", + "value": "application/json" + } + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1 + }, + "response": { + "status": 0, + "statusText": "", + "httpVersion": "", + "cookies": [], + "headers": [], + "content": { + "size": 0, + "mimeType": "x-unknown" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": -1, + "_error": "java.net.UnknownHostException: Unable to resolve host \"telemetry.example.com\"" + }, + "cache": {}, + "timings": { + "blocked": -1, + "dns": -1, + "connect": -1, + "ssl": -1, + "send": 0, + "wait": 8, + "receive": 0 + }, + "_resourceType": "fetch", + "_lustro": { + "id": "tx_d03a9b44", + "isMocked": false, + "categories": [], + "requestBodyTruncated": false, + "responseBodyTruncated": false, + "responseComplete": true, + "error": "java.net.UnknownHostException: Unable to resolve host \"telemetry.example.com\"" + } + } + ] + } +} diff --git a/lustro-cli/src/lustro_cli/wire/network.openapi.json b/lustro-cli/src/lustro_cli/wire/network.openapi.json index d35c3fe..9bc205b 100644 --- a/lustro-cli/src/lustro_cli/wire/network.openapi.json +++ b/lustro-cli/src/lustro_cli/wire/network.openapi.json @@ -73,6 +73,25 @@ } } }, + "/api/v1/network/transactions/_/export": { + "get": { + "operationId": "exportTransactions", + "summary": "Captured transactions as a HAR 1.2 document.", + "description": "Returns the transactions that ids names, or every one when ids is left out, as a HAR 1.2 document that browser devtools and HTTP tools import, oldest first. It is built from the store, so headers, the URL, and text bodies are redacted as in the detail. An id the store no longer has is left out, and an ids parameter that names none returns no entries. Each entry spends its whole durationMs in timings.wait, with send and receive at 0 and the other phases at -1. A body kept as bytes goes out in base64: content.encoding is base64 for a response, and postData._encoding for a request, which HAR gives no encoding field. _lustro on each entry carries the transaction id and what HAR has no field for: isMocked, categories, the truncation flags, responseComplete, and error. A request still in flight has status 0 and time 0. The request line and headers must stay under 8 KB, so a client that exports many ids sends them in batches.", + "parameters": [ + { "name": "format", "in": "query", "required": false, "schema": { "type": "string", "enum": ["har"], "default": "har" }, "description": "The export format. 400 for any other." }, + { "name": "ids", "in": "query", "required": false, "schema": { "type": "string" }, "description": "Transaction ids separated by commas; the parameter can also repeat. Left out, the export has every transaction." } + ], + "responses": { + "200": { "description": "A HAR document.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HarDocument" } } } }, + "400": { "$ref": "#/components/responses/Error" }, + "401": { "$ref": "#/components/responses/Error" }, + "403": { "$ref": "#/components/responses/Error" }, + "413": { "$ref": "#/components/responses/Error" }, + "503": { "$ref": "#/components/responses/Error" } + } + } + }, "/api/v1/network/clear": { "post": { "operationId": "clearTransactions", @@ -281,6 +300,117 @@ "responseBodyBinary": { "type": "boolean", "description": "Detail only. true when the response body was kept as bytes, as it arrived, because the Redactor can't read it: an image. responseBody is then null; fetch the bytes from GET transactions/{id}/body/response." } } }, + "HarDocument": { + "type": "object", + "description": "A HAR 1.2 document. Only the parts the export writes are described; the HAR 1.2 specification describes the rest.", + "required": ["log"], + "properties": { + "log": { + "type": "object", + "required": ["version", "creator", "entries"], + "properties": { + "version": { "const": "1.2" }, + "creator": { + "type": "object", + "required": ["name", "version"], + "properties": { "name": { "const": "Lustro" }, "version": { "type": "string", "description": "The library version." } } + }, + "entries": { "type": "array", "items": { "$ref": "#/components/schemas/HarEntry" }, "description": "Oldest first." } + } + } + } + }, + "HarEntry": { + "type": "object", + "required": ["startedDateTime", "time", "request", "response", "cache", "timings", "_resourceType", "_lustro"], + "properties": { + "startedDateTime": { "type": "string", "description": "startedAt in ISO 8601, in UTC with milliseconds, e.g. 2026-09-28T14:22:09.003Z." }, + "time": { "type": "integer", "minimum": 0, "description": "durationMs; 0 while the request is in flight." }, + "request": { + "type": "object", + "required": ["method", "url", "httpVersion", "cookies", "headers", "queryString", "headersSize", "bodySize"], + "properties": { + "method": { "type": "string" }, + "url": { "type": "string", "description": "Redacted at capture time." }, + "httpVersion": { "type": "string", "description": "protocol, e.g. h2; empty when it is null." }, + "cookies": { "type": "array", "maxItems": 0, "description": "Always empty; the Cookie header is in headers." }, + "headers": { "type": "array", "items": { "$ref": "#/components/schemas/HarNameValue" } }, + "queryString": { "type": "array", "items": { "$ref": "#/components/schemas/HarNameValue" }, "description": "The URL's query parameters, decoded." }, + "postData": { + "type": "object", + "description": "Present when the request body was kept, as text or as bytes.", + "required": ["mimeType", "text"], + "properties": { + "mimeType": { "type": "string", "description": "requestContentType; empty when it is null." }, + "text": { "type": "string", "description": "The redacted text, or the bytes in base64 when _encoding is base64." }, + "_encoding": { "const": "base64", "description": "Present when the body was kept as bytes." } + } + }, + "headersSize": { "const": -1 }, + "bodySize": { "type": "integer", "description": "requestBodyBytes; -1 when it is null." } + } + }, + "response": { + "type": "object", + "required": ["status", "statusText", "httpVersion", "cookies", "headers", "content", "redirectURL", "headersSize", "bodySize"], + "properties": { + "status": { "type": "integer", "description": "statusCode; 0 before a response and for a failed request." }, + "statusText": { "const": "", "description": "The capture keeps no reason phrase." }, + "httpVersion": { "type": "string" }, + "cookies": { "type": "array", "maxItems": 0, "description": "Always empty; the Set-Cookie header is in headers." }, + "headers": { "type": "array", "items": { "$ref": "#/components/schemas/HarNameValue" } }, + "content": { + "type": "object", + "required": ["size", "mimeType"], + "properties": { + "size": { "type": "integer", "minimum": 0, "description": "The larger of responseBodyBytes and the size of the body kept, which is more for a body kept inflated." }, + "mimeType": { "type": "string", "description": "responseContentType, or x-unknown when it is null." }, + "text": { "type": "string", "description": "The redacted text, or the bytes in base64 when encoding is base64. Absent when no body was kept." }, + "encoding": { "const": "base64" } + } + }, + "redirectURL": { "type": "string", "description": "The Location header, or empty." }, + "headersSize": { "const": -1 }, + "bodySize": { "type": "integer", "description": "responseBodyBytes; -1 when it is null." }, + "_error": { "type": "string", "description": "error, for a failed request, where Chrome's own export puts it." } + } + }, + "cache": { "type": "object", "maxProperties": 0 }, + "_resourceType": { "type": "string", "enum": ["fetch", "image"], "description": "How Chrome DevTools files the request on import: image for an image/* response, else fetch." }, + "timings": { + "type": "object", + "required": ["send", "wait", "receive"], + "properties": { + "blocked": { "const": -1 }, + "dns": { "const": -1 }, + "connect": { "const": -1 }, + "ssl": { "const": -1 }, + "send": { "const": 0 }, + "wait": { "type": "integer", "minimum": 0, "description": "durationMs; 0 while the request is in flight." }, + "receive": { "const": 0 } + } + }, + "_lustro": { + "type": "object", + "description": "What HAR has no field for. HAR reserves names that start with an underscore for a tool's own fields.", + "required": ["id", "isMocked", "categories", "requestBodyTruncated", "responseBodyTruncated", "responseComplete", "error"], + "properties": { + "id": { "type": "string", "description": "The transaction id." }, + "isMocked": { "type": "boolean" }, + "categories": { "type": "array", "items": { "type": "string" } }, + "requestBodyTruncated": { "type": "boolean", "description": "true when postData has only the first part of the body." }, + "responseBodyTruncated": { "type": "boolean", "description": "true when content has only the first part of the body." }, + "responseComplete": { "type": "boolean", "description": "false while the request is in flight, including a streaming response." }, + "error": { "type": ["string", "null"] } + } + } + } + }, + "HarNameValue": { + "type": "object", + "required": ["name", "value"], + "properties": { "name": { "type": "string" }, "value": { "type": "string" } } + }, "MockRule": { "type": "object", "required": ["id", "urlPattern", "statusCode", "enabled"], diff --git a/lustro-cli/tests/conftest.py b/lustro-cli/tests/conftest.py index b18b774..2c2a6f5 100644 --- a/lustro-cli/tests/conftest.py +++ b/lustro-cli/tests/conftest.py @@ -28,6 +28,7 @@ "send-result.json": None, # validated against the inline sendRequest 200 schema "rules-list.json": None, # validated against the inline listMockRules 200 schema "error-envelope.json": "ErrorEnvelope", + "export-har.json": "HarDocument", } diff --git a/lustro-cli/tests/test_cli_dispatch.py b/lustro-cli/tests/test_cli_dispatch.py index aa8a90d..febec55 100644 --- a/lustro-cli/tests/test_cli_dispatch.py +++ b/lustro-cli/tests/test_cli_dispatch.py @@ -319,6 +319,7 @@ def test_all_openapi_paths_are_covered(): "/api/v1/network/transactions", "/api/v1/network/transactions/{id}", "/api/v1/network/transactions/{id}/body/{direction}", + "/api/v1/network/transactions/_/export", "/api/v1/network/clear", "/api/v1/network/rules", "/api/v1/network/rules/_/sync", diff --git a/lustro-cli/tests/test_cli_export.py b/lustro-cli/tests/test_cli_export.py new file mode 100644 index 0000000..651440a --- /dev/null +++ b/lustro-cli/tests/test_cli_export.py @@ -0,0 +1,175 @@ +"""Tests for ``lustro net export``: the HAR export route, id batches, and the file it writes.""" + +from __future__ import annotations + +import json + +import pytest + +from lustro_cli import cli, wire +from lustro_cli.client import LustroClient +from lustro_cli.discovery import Endpoint + +EXPORT = "/api/v1/network/transactions/_/export" +TRANSACTIONS = "/api/v1/network/transactions" + + +def full_id(n): + return "3f2a9c1d-5b7e-4c8a-9d6f-{:012d}".format(n) + + +def entry(tx_id, started="2026-09-28T14:22:07.512Z"): + return {"startedDateTime": started, "time": 1, "_lustro": {"id": tx_id}} + + +def started_at(n): + return "2026-09-28T14:{:02d}:{:02d}.000Z".format(22 + n // 60, n % 60) + + +def har(*entries): + return {"log": {"version": "1.2", "creator": {"name": "Lustro", "version": "0.1.0"}, "entries": list(entries)}} + + +class _ExportServer(LustroClient): + """Answers the list route with ``listed``, and the export route with an entry + for each requested id that the app still has, as the server does.""" + + def __init__(self, listed=()): + super().__init__("http://127.0.0.1:8080", "tok") + self.calls = [] + self.listed = list(listed) + self.known = {tx_id: entry(tx_id, started_at(i)) for i, tx_id in enumerate(listed)} + self.everything = har(*self.known.values()) + + def request(self, method, path, *, params=None, json_body=None): + self.calls.append((method, path, dict(params or {}))) + if path == TRANSACTIONS: + return {"cursor": "c:1", "status": "reset", "items": [{"id": tx_id} for tx_id in self.listed]} + assert path == EXPORT + if "ids" not in (params or {}): + return json.loads(json.dumps(self.everything)) + ids = params["ids"].split(",") + found = sorted((self.known[i] for i in ids if i in self.known), key=lambda e: e["startedDateTime"]) + return har(*found) + + +@pytest.fixture +def server(monkeypatch): + holder = {} + + def build(args): + return holder["client"] + + def install(listed=()): + holder["client"] = _ExportServer(listed) + return holder["client"] + + monkeypatch.setattr(cli, "_build_client", build) + monkeypatch.setattr(cli, "_build_endpoint", lambda args: Endpoint("127.0.0.1", 8080, "tok")) + return install + + +def exports(client): + return [call for call in client.calls if call[1] == EXPORT] + + +def test_export_without_ids_saves_every_transaction(server, tmp_path, capsys): + client = server([full_id(1), full_id(2)]) + out = tmp_path / "all.har" + assert cli.main(["net", "export", "--har", str(out)]) == 0 + assert client.calls == [("GET", EXPORT, {"format": "har"})] + assert json.loads(out.read_text("utf-8")) == client.everything + assert "saved 2 transactions" in capsys.readouterr().out + + +def test_the_file_is_indented_as_devtools_write_one(server, tmp_path): + server([full_id(1)]) + out = tmp_path / "all.har" + cli.main(["net", "export", "--har", str(out)]) + text = out.read_text("utf-8") + assert text.startswith('{\n "log": {\n "version": "1.2"') + assert text.endswith("}\n") + + +def test_full_ids_need_no_list_request(server, tmp_path): + client = server([full_id(1), full_id(2), full_id(3)]) + cli.main(["net", "export", "--har", str(tmp_path / "x.har"), "--ids", full_id(3), full_id(1)]) + assert client.calls == [("GET", EXPORT, {"format": "har", "ids": full_id(3) + "," + full_id(1)})] + + +def test_ids_take_commas_and_starts_and_one_list_request_resolves_them(server, tmp_path): + client = server([full_id(1), full_id(2), "tx_other"]) + out = tmp_path / "x.har" + assert cli.main(["net", "export", "--har", str(out), "--ids", full_id(1) + ",tx_o", full_id(2)]) == 0 + assert [call[1] for call in client.calls] == [TRANSACTIONS, EXPORT] + assert exports(client)[0][2]["ids"] == ",".join([full_id(1), "tx_other", full_id(2)]) + + +def test_an_ambiguous_start_exits_2(server, tmp_path, capsys): + server([full_id(1), full_id(2)]) + assert cli.main(["net", "export", "--har", str(tmp_path / "x.har"), "--ids", "3f2a"]) == 2 + assert "2 transaction ids start with 3f2a" in capsys.readouterr().err + + +def test_many_ids_go_in_batches_and_join_oldest_first(server, tmp_path): + ids = [full_id(n) for n in range(120)] + client = server(ids) + out = tmp_path / "many.har" + # Newest first, as a list prints them. + assert cli.main(["net", "export", "--har", str(out), "--ids", ",".join(reversed(ids))]) == 0 + batches = [call[2]["ids"].split(",") for call in exports(client)] + assert [len(batch) for batch in batches] == [50, 50, 20] + entries = json.loads(out.read_text("utf-8"))["log"]["entries"] + assert [e["_lustro"]["id"] for e in entries] == ids + + +def test_ids_the_app_no_longer_has_are_left_out_with_a_warning(server, tmp_path, capsys): + server([full_id(1)]) + out = tmp_path / "x.har" + assert cli.main(["net", "export", "--har", str(out), "--ids", full_id(1), full_id(9)]) == 0 + err = capsys.readouterr().err + assert "no longer has 1 of the transactions" in err + assert full_id(9) in err + assert len(json.loads(out.read_text("utf-8"))["log"]["entries"]) == 1 + + +def test_a_dash_writes_the_har_to_stdout_and_nothing_else(server, capsysbinary): + client = server([full_id(1)]) + assert cli.main(["net", "export", "--har", "-"]) == 0 + assert json.loads(capsysbinary.readouterr().out.decode("utf-8")) == client.everything + + +def test_json_prints_a_summary(server, tmp_path, capsys): + server([full_id(1), full_id(2)]) + out = tmp_path / "x.har" + cli.main(["--json", "net", "export", "--har", str(out), "--ids", full_id(2), full_id(7)]) + summary = json.loads(capsys.readouterr().out) + assert summary == {"path": str(out), "entries": 1, "bytes": out.stat().st_size, "missing": [full_id(7)]} + + +def test_an_unwritable_file_exits_2(server, tmp_path, capsys): + server([full_id(1)]) + out = tmp_path / "missing" / "x.har" + assert cli.main(["net", "export", "--har", str(out)]) == 2 + assert "could not write HAR file" in capsys.readouterr().err + + +def test_a_response_that_is_not_har_exits_1(server, tmp_path, capsys): + client = server() + client.everything = {"status": "ok"} + assert cli.main(["net", "export", "--har", str(tmp_path / "x.har")]) == 1 + assert "not a HAR document" in capsys.readouterr().err + + +def test_har_is_required(server, capsys): + server() + with pytest.raises(SystemExit): + cli.main(["net", "export"]) + + +def test_the_golden_export_round_trips_unchanged(server, tmp_path): + client = server() + client.everything = wire.load_golden("export-har.json") + out = tmp_path / "golden.har" + cli.main(["net", "export", "--har", str(out)]) + assert json.loads(out.read_text("utf-8")) == wire.load_golden("export-har.json") diff --git a/lustro-cli/tests/test_golden_schemas.py b/lustro-cli/tests/test_golden_schemas.py index 795214b..6cd9f65 100644 --- a/lustro-cli/tests/test_golden_schemas.py +++ b/lustro-cli/tests/test_golden_schemas.py @@ -57,6 +57,7 @@ def _component_validator(openapi, component_name): ("transaction.json", "Transaction"), ("transaction-image.json", "Transaction"), ("error-envelope.json", "ErrorEnvelope"), + ("export-har.json", "HarDocument"), ] @@ -169,3 +170,29 @@ def test_the_body_route_is_declared_with_both_directions(): direction = next(p for p in op["parameters"] if p["name"] == "direction") assert direction["schema"]["enum"] == ["request", "response"] assert "404" in op["responses"] + + +def test_the_export_route_is_declared_with_its_format_and_ids(): + op = wire.load_openapi()["paths"]["/api/v1/network/transactions/_/export"]["get"] + params = {p["name"]: p for p in op["parameters"]} + assert params["format"]["schema"]["enum"] == ["har"] + assert params["ids"]["required"] is False + assert "400" in op["responses"] + + +def test_the_golden_har_marks_what_har_has_no_field_for(): + entries = wire.load_golden("export-har.json")["log"]["entries"] + by_id = {entry["_lustro"]["id"]: entry for entry in entries} + # Oldest first, as HAR readers prefer. + started = [entry["startedDateTime"] for entry in entries] + assert started == sorted(started) + assert by_id["tx_77e2c014"]["_lustro"]["isMocked"] is True + assert by_id["tx_5e8a2f61"]["_lustro"]["responseBodyTruncated"] is True + assert by_id["tx_d03a9b44"]["_lustro"]["error"] == by_id["tx_d03a9b44"]["response"]["_error"] + # A body kept as bytes is base64. + assert by_id["tx_e6b0d413"]["response"]["content"]["encoding"] == "base64" + # Each entry spends its duration waiting, so time adds up as HAR requires. + for entry in entries: + timings = entry["timings"] + assert entry["time"] == sum(value for value in timings.values() if value != -1) + diff --git a/lustro/src/main/assets/lustro/network.css b/lustro/src/main/assets/lustro/network.css index 6bbe076..c50ca19 100644 --- a/lustro/src/main/assets/lustro/network.css +++ b/lustro/src/main/assets/lustro/network.css @@ -181,6 +181,31 @@ .net-filter-pill.method-patch { --c: var(--method-patch); } .net-filter-pill.method-other { --c: var(--method-head); } +/* ── Select mode: a checkbox column and a bar with what to do with the rows ── */ +.net-select-bar { + display: flex; + align-items: center; + gap: 8px; + padding: 10px 22px; + border-bottom: 1px solid var(--border-soft); +} +.net-select-bar[hidden] { display: none; } +.net-select-count { + margin-right: auto; + color: var(--accent-2); + font-size: 12px; +} +/* The whole cell toggles the row, so the target is wider than the box. */ +.net-select-col, .net-cell-select { width: 36px; padding-right: 0; text-align: center; cursor: pointer; } +.net-check { + width: 14px; + height: 14px; + margin: 0; + vertical-align: middle; + accent-color: var(--accent); + cursor: pointer; +} + /* ── Traffic grid ── */ #tx-list tr { cursor: pointer; } .net-cell-method { font-weight: 600; } diff --git a/lustro/src/main/assets/lustro/network.js b/lustro/src/main/assets/lustro/network.js index bf06fca..6b15ccb 100644 --- a/lustro/src/main/assets/lustro/network.js +++ b/lustro/src/main/assets/lustro/network.js @@ -28,6 +28,8 @@ var allTransactions = []; var selectedTxId = null; + var selectMode = false; + var selectedIds = {}; // transaction id -> true; only ids the filters show var lastCursor = null; // opaque cursor token; omitted on the first poll var searchText = ''; var searchTimer = null; @@ -128,6 +130,29 @@ }); } + // Copies text that is still being fetched. Safari allows a copy only in the + // click itself, so the clipboard item is made there and the text follows. + function copyNetworkTextLater(textPromise, ev) { + var write = null; + try { + if (window.isSecureContext && navigator.clipboard && navigator.clipboard.write && typeof ClipboardItem === 'function') { + write = navigator.clipboard.write([new ClipboardItem({ + 'text/plain': textPromise.then(function(text) { return new Blob([text], { type: 'text/plain' }); }), + })]); + } + } catch(e) { + write = null; + } + var copied = write + ? write.catch(function() { return textPromise.then(window.debugWriteToClipboard); }) + : textPromise.then(window.debugWriteToClipboard); + copied.then(function() { + showCopyPopup(ev); + }).catch(function(e) { + debugToast('Failed to copy' + (e && e.message ? ': ' + e.message : ''), 'error'); + }); + } + function showCopyPopup(ev) { var popup = document.createElement('div'); popup.className = 'net-copy-popup'; @@ -340,6 +365,12 @@ var tbody = document.getElementById('tx-list'); var countEl = document.getElementById('tx-count'); if (!tbody) return; + // The selection follows the filters: what they hide is no longer selected, + // so an export has exactly the selected rows on screen. + var kept = {}; + filtered.forEach(function(tx) { if (selectedIds[tx.id]) kept[tx.id] = true; }); + selectedIds = kept; + updateSelectionControls(filtered); var visible = filtered.slice(0, displayLimit); var label = filtered.length + (filtered.length !== allTransactions.length ? '/' + allTransactions.length : '') + ' requests'; @@ -358,7 +389,13 @@ var sel = tx.id === selectedTxId ? ' dc-row--selected' : ''; var mockedBadge = tx.isMocked ? ' Mocked' : ''; var streamingBadge = streaming ? ' Streaming' : ''; + var check = selectMode + ? '' + + '' + : ''; return '' + + check + '' + debugEscapeHtml(tx.method || '') + '' + '' + debugEscapeHtml(shortUrl) + '' + '' + statusText + mockedBadge + streamingBadge + '' @@ -394,6 +431,143 @@ displayLimit = DISPLAY_LIMIT_INITIAL; } + function updateSelectionControls(filtered) { + var count = Object.keys(selectedIds).length; + var countEl = document.getElementById('select-count'); + if (countEl) countEl.textContent = count + ' of ' + filtered.length + ' selected'; + ['copy-selection-md-btn', 'export-har-btn'].forEach(function(id) { + var btn = document.getElementById(id); + if (btn) btn.disabled = count === 0; + }); + var all = document.getElementById('select-all'); + if (all) { + all.checked = count > 0 && count === filtered.length; + all.indeterminate = count > 0 && count < filtered.length; + } + } + + window.toggleSelectMode = function() { + selectMode = !selectMode; + selectedIds = {}; + var btn = document.getElementById('select-btn'); + if (btn) btn.classList.toggle('dc-btn--active', selectMode); + var bar = document.getElementById('select-bar'); + if (bar) bar.hidden = !selectMode; + var col = document.getElementById('select-col'); + if (col) col.style.display = selectMode ? '' : 'none'; + renderList(); + }; + + window.toggleTxSelection = function(id) { + if (selectedIds[id]) delete selectedIds[id]; + else selectedIds[id] = true; + renderList(); + }; + + // Every row the filters show, past the ones rendered so far too; or none + // when every one is selected already. + window.toggleSelectAll = function() { + var filtered = filterTransactions(); + var all = filtered.length > 0 && filtered.every(function(tx) { return selectedIds[tx.id]; }); + selectedIds = {}; + if (!all) filtered.forEach(function(tx) { selectedIds[tx.id] = true; }); + renderList(); + }; + + // An export reads oldest first. The list is newest first by when each capture + // was stored, which is not always when its request started, so sort by that. + function selectedOldestFirst() { + return filterTransactions().filter(function(tx) { return selectedIds[tx.id]; }).reverse() + .sort(function(a, b) { return (a.startedAt || 0) - (b.startedAt || 0); }); + } + + // A request line and its headers must stay under 8 KB, and an id is 36 characters. + var EXPORT_BATCH = 50; + + window.exportSelectionHar = function() { + var ids = selectedOldestFirst().map(function(tx) { return tx.id; }); + if (!ids.length) return; + var batches = []; + for (var i = 0; i < ids.length; i += EXPORT_BATCH) batches.push(ids.slice(i, i + EXPORT_BATCH)); + var parts = []; + var chain = Promise.resolve(); + batches.forEach(function(batch) { + chain = chain.then(function() { + var query = 'format=har&ids=' + batch.map(encodeURIComponent).join(','); + return debugFetch(netUrl('transactions/_/export?' + query)) + .then(function(r) { return r.json(); }) + .then(function(part) { parts.push(part); }); + }); + }); + chain.then(function() { + var har = window.netMergeHar(parts); + saveText(JSON.stringify(har, null, 2) + '\n', 'lustro-' + fileTimestamp(new Date()) + '.har', 'application/json'); + var exported = har.log.entries.length; + var gone = ids.length - exported; + debugToast('Exported ' + exported + ' request' + (exported === 1 ? '' : 's') + + (gone > 0 ? '; ' + gone + ' no longer captured' : ''), gone > 0 ? 'warning' : 'success'); + }).catch(function(e) { + debugToast('Failed to export: ' + e.message, 'error'); + }); + }; + + // The export batches as one HAR document, oldest first. Each batch is in order + // already, and the times are ISO 8601 in UTC, so they sort as text; the sort + // is stable, so a tie keeps the order the batches had. + window.netMergeHar = function(parts) { + var har = parts[0] || { log: { version: '1.2', creator: { name: 'Lustro', version: '' }, entries: [] } }; + for (var i = 1; i < parts.length; i++) har.log.entries = har.log.entries.concat(parts[i].log.entries); + har.log.entries.sort(function(a, b) { + return a.startedDateTime < b.startedDateTime ? -1 : (a.startedDateTime > b.startedDateTime ? 1 : 0); + }); + return har; + }; + + function saveText(text, name, type) { + var url = URL.createObjectURL(new Blob([text], { type: type })); + var link = document.createElement('a'); + link.href = url; + link.download = name; + document.body.appendChild(link); + link.click(); + link.remove(); + // Some browsers start the download only after the click returns. + setTimeout(function() { URL.revokeObjectURL(url); }, 10000); + } + + function fileTimestamp(date) { + var pad = function(n) { return (n < 10 ? '0' : '') + n; }; + return date.getFullYear() + pad(date.getMonth() + 1) + pad(date.getDate()) + + '-' + pad(date.getHours()) + pad(date.getMinutes()) + pad(date.getSeconds()); + } + + window.copySelectionMarkdown = function(ev) { + var txs = selectedOldestFirst(); + if (!txs.length) return; + copyNetworkTextLater(fetchDetails(txs).then(function(details) { + if (!details.length) throw new Error('the requests are no longer captured'); + return window.netTransactionsMarkdown(details); + }), ev); + }; + + // The details of txs, in their order, four requests at a time. One the app no + // longer has is left out. + function fetchDetails(txs) { + var details = []; + var next = 0; + function worker() { + if (next >= txs.length) return Promise.resolve(); + var index = next++; + return debugFetch(netUrl('transactions/' + encodeURIComponent(txs[index].id))) + .then(function(r) { return r.json(); }) + .then(function(tx) { details[index] = tx; }, function(e) { if (!e || e.status !== 404) throw e; }) + .then(worker); + } + var workers = []; + for (var i = 0; i < Math.min(4, txs.length); i++) workers.push(worker()); + return Promise.all(workers).then(function() { return details.filter(Boolean); }); + } + function methodClass(tx) { return 'net-m-' + methodKeyForTx(tx).toLowerCase(); } @@ -527,10 +701,7 @@ }); if (!previous || previous.id !== tx.id) hexDumps = {}; currentDetailTx = tx; - var copyAllEl = document.getElementById('copy-all-btn'); - if (copyAllEl) copyAllEl.style.display = ''; - var copyCurlEl = document.getElementById('copy-curl-btn'); - if (copyCurlEl) copyCurlEl.style.display = ''; + showDetailActions(true); copyStore = {}; copyIdSeq = 0; var el = document.getElementById('detail-content'); @@ -932,6 +1103,115 @@ copyNetworkText(buildCurlCommand(currentDetailTx), ev); }; + window.copyMarkdown = function(ev) { + if (!currentDetailTx) return; + copyNetworkText(window.netTransactionsMarkdown([currentDetailTx]), ev); + }; + + function showDetailActions(show) { + ['copy-curl-btn', 'copy-md-btn', 'copy-all-btn'].forEach(function(id) { + var btn = document.getElementById(id); + if (btn) btn.style.display = show ? '' : 'none'; + }); + } + + // Transaction details as one Markdown document, oldest first as given: for + // each, the method and URL as a heading, then the headers in http blocks and + // the bodies in blocks tagged with a language for their content type. A + // body the capture cut short says so. + window.netTransactionsMarkdown = function(txs) { + return txs.map(transactionMarkdown).join('\n\n---\n\n') + '\n'; + }; + + function transactionMarkdown(tx) { + var out = ['## ' + markdownCode((tx.method || '') + ' ' + (tx.url || ''))]; + var meta = []; + var status = tx.error ? 'Failed' : (tx.statusCode != null ? String(tx.statusCode) : 'Pending'); + if (isStreaming(tx)) status += ', streaming'; + meta.push('**' + status + '**'); + if (tx.durationMs != null) meta.push(tx.durationMs + ' ms'); + if (tx.startedAt != null) meta.push(new Date(tx.startedAt).toISOString()); + else if (tx.timestamp) meta.push(tx.timestamp); + if (tx.protocol) meta.push(markdownCode(tx.protocol)); + if (tx.isMocked) meta.push('mocked by Lustro'); + (tx.categories || []).forEach(function(c) { meta.push(markdownCode(c)); }); + out.push(meta.join(' · ')); + if (tx.error) out.push('**Error:** ' + markdownCode(tx.error)); + ['request', 'response'].forEach(function(dir) { + var label = dir === 'request' ? 'Request' : 'Response'; + var headers = tx[dir + 'Headers'] || {}; + var names = Object.keys(headers); + if (names.length) { + out.push('### ' + label + ' headers'); + out.push(markdownFence(names.map(function(k) { return k + ': ' + headers[k]; }).join('\n'), 'http')); + } + var body = markdownBody(tx, dir); + if (body) out.push('### ' + label + ' body', body); + }); + return out.join('\n\n'); + } + + function markdownBody(tx, dir) { + var text = tx[dir + 'Body']; + var contentType = tx[dir + 'ContentType']; + if (tx[dir + 'BodyBinary']) { + var size = formatBytes(tx[dir + 'BodyBytes']); + return '_' + (mediaEssence(contentType) || 'Binary') + ' body' + (size ? ', ' + size : '') + + ': kept as bytes, so it is not included.' + + (tx[dir + 'BodyTruncated'] ? ' Lustro kept only its first part.' : '') + '_'; + } + if (text == null) { + var coding = undecodedCoding(tx[dir + 'Headers']); + return coding ? '_Not captured: Lustro does not decode the ' + coding + ' encoding._' : ''; + } + if (text === '') return ''; + var kind = debugBodyKind(contentType, text, false); + var shown = text; + if (kind === 'json') { + // Indents without changing a value; a body cut short stays as it is. + var pieces = debugScanJsonSource(text, 2); + if (pieces) shown = pieces.map(function(piece) { return piece.text; }).join(''); + } + var block = markdownFence(shown, markdownLanguage(kind, contentType)); + return tx[dir + 'BodyTruncated'] + ? '_Truncated: Lustro kept only the first part of this body._\n\n' + block + : block; + } + + // No prototype: the media type comes from the captured headers. + var MARKDOWN_LANGUAGES = Object.assign(Object.create(null), { + 'text/css': 'css', 'text/csv': 'csv', 'text/markdown': 'markdown', + 'application/javascript': 'javascript', 'text/javascript': 'javascript', + 'application/graphql': 'graphql', 'application/yaml': 'yaml', 'application/x-yaml': 'yaml', 'text/yaml': 'yaml', + }); + function markdownLanguage(kind, contentType) { + if (kind === 'json' || kind === 'xml' || kind === 'html') return kind; + return MARKDOWN_LANGUAGES[mediaEssence(contentType)] || 'text'; + } + + // A fenced code block. The fence is longer than any run of backticks in the + // text, so nothing in it can close the block. + function markdownFence(text, language) { + var ticks = backticks(Math.max(3, longestBacktickRun(text) + 1)); + return ticks + language + '\n' + text + (/\n$/.test(text) ? '' : '\n') + ticks; + } + + // An inline code span on one line, delimited as the fence is. + function markdownCode(text) { + text = String(text).replace(/[\r\n]+/g, ' '); + var ticks = backticks(longestBacktickRun(text) + 1); + var pad = /^`|`$/.test(text) || /^ .* $/.test(text) ? ' ' : ''; + return ticks + pad + text + pad + ticks; + } + + function longestBacktickRun(text) { + return (String(text).match(/`+/g) || []).reduce(function(longest, run) { return Math.max(longest, run.length); }, 0); + } + + function backticks(n) { + return new Array(n + 1).join('`'); + } + window.switchRightTab = function(tab) { document.getElementById('detail-content').classList.toggle('active', tab === 'detail'); document.getElementById('rules-content').classList.toggle('active', tab === 'rules'); @@ -939,11 +1219,7 @@ document.getElementById('tab-btn-detail').classList.toggle('dc-tab--active', tab === 'detail'); document.getElementById('tab-btn-rules').classList.toggle('dc-tab--active', tab === 'rules'); document.getElementById('tab-btn-send').classList.toggle('dc-tab--active', tab === 'send'); - var detailHasTx = !!currentDetailTx; - var copyAll = document.getElementById('copy-all-btn'); - var copyCurl = document.getElementById('copy-curl-btn'); - if (copyAll) copyAll.style.display = (tab === 'detail' && detailHasTx) ? '' : 'none'; - if (copyCurl) copyCurl.style.display = (tab === 'detail' && detailHasTx) ? '' : 'none'; + showDetailActions(tab === 'detail' && !!currentDetailTx); if (tab === 'rules') loadRules(); if (tab === 'send') ensureSendForm(); }; @@ -1537,10 +1813,7 @@ var dc = document.getElementById('detail-content'); if (dc) dc.innerHTML = '
🔍

Select a request to inspect

'; - var copyAll = document.getElementById('copy-all-btn'); - var copyCurl = document.getElementById('copy-curl-btn'); - if (copyAll) copyAll.style.display = 'none'; - if (copyCurl) copyCurl.style.display = 'none'; + showDetailActions(false); } return; } @@ -1587,6 +1860,12 @@ showCaptureFilter: function() { window.showCaptureFilter(); }, closeCaptureFilter: function() { window.closeCaptureFilter(); }, copyCurl: function(el, ev) { window.copyCurl(ev); }, + copyMarkdown: function(el, ev) { window.copyMarkdown(ev); }, + toggleSelectMode: function() { window.toggleSelectMode(); }, + toggleTxSelection: function(el) { window.toggleTxSelection(el.dataset.txId); }, + toggleSelectAll: function() { window.toggleSelectAll(); }, + copySelectionMarkdown: function(el, ev) { window.copySelectionMarkdown(ev); }, + exportSelectionHar: function() { window.exportSelectionHar(); }, copyAllDetail: function(el, ev) { window.copyAllDetail(ev); }, }; diff --git a/lustro/src/main/assets/lustro/network.openapi.json b/lustro/src/main/assets/lustro/network.openapi.json index d35c3fe..9bc205b 100644 --- a/lustro/src/main/assets/lustro/network.openapi.json +++ b/lustro/src/main/assets/lustro/network.openapi.json @@ -73,6 +73,25 @@ } } }, + "/api/v1/network/transactions/_/export": { + "get": { + "operationId": "exportTransactions", + "summary": "Captured transactions as a HAR 1.2 document.", + "description": "Returns the transactions that ids names, or every one when ids is left out, as a HAR 1.2 document that browser devtools and HTTP tools import, oldest first. It is built from the store, so headers, the URL, and text bodies are redacted as in the detail. An id the store no longer has is left out, and an ids parameter that names none returns no entries. Each entry spends its whole durationMs in timings.wait, with send and receive at 0 and the other phases at -1. A body kept as bytes goes out in base64: content.encoding is base64 for a response, and postData._encoding for a request, which HAR gives no encoding field. _lustro on each entry carries the transaction id and what HAR has no field for: isMocked, categories, the truncation flags, responseComplete, and error. A request still in flight has status 0 and time 0. The request line and headers must stay under 8 KB, so a client that exports many ids sends them in batches.", + "parameters": [ + { "name": "format", "in": "query", "required": false, "schema": { "type": "string", "enum": ["har"], "default": "har" }, "description": "The export format. 400 for any other." }, + { "name": "ids", "in": "query", "required": false, "schema": { "type": "string" }, "description": "Transaction ids separated by commas; the parameter can also repeat. Left out, the export has every transaction." } + ], + "responses": { + "200": { "description": "A HAR document.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HarDocument" } } } }, + "400": { "$ref": "#/components/responses/Error" }, + "401": { "$ref": "#/components/responses/Error" }, + "403": { "$ref": "#/components/responses/Error" }, + "413": { "$ref": "#/components/responses/Error" }, + "503": { "$ref": "#/components/responses/Error" } + } + } + }, "/api/v1/network/clear": { "post": { "operationId": "clearTransactions", @@ -281,6 +300,117 @@ "responseBodyBinary": { "type": "boolean", "description": "Detail only. true when the response body was kept as bytes, as it arrived, because the Redactor can't read it: an image. responseBody is then null; fetch the bytes from GET transactions/{id}/body/response." } } }, + "HarDocument": { + "type": "object", + "description": "A HAR 1.2 document. Only the parts the export writes are described; the HAR 1.2 specification describes the rest.", + "required": ["log"], + "properties": { + "log": { + "type": "object", + "required": ["version", "creator", "entries"], + "properties": { + "version": { "const": "1.2" }, + "creator": { + "type": "object", + "required": ["name", "version"], + "properties": { "name": { "const": "Lustro" }, "version": { "type": "string", "description": "The library version." } } + }, + "entries": { "type": "array", "items": { "$ref": "#/components/schemas/HarEntry" }, "description": "Oldest first." } + } + } + } + }, + "HarEntry": { + "type": "object", + "required": ["startedDateTime", "time", "request", "response", "cache", "timings", "_resourceType", "_lustro"], + "properties": { + "startedDateTime": { "type": "string", "description": "startedAt in ISO 8601, in UTC with milliseconds, e.g. 2026-09-28T14:22:09.003Z." }, + "time": { "type": "integer", "minimum": 0, "description": "durationMs; 0 while the request is in flight." }, + "request": { + "type": "object", + "required": ["method", "url", "httpVersion", "cookies", "headers", "queryString", "headersSize", "bodySize"], + "properties": { + "method": { "type": "string" }, + "url": { "type": "string", "description": "Redacted at capture time." }, + "httpVersion": { "type": "string", "description": "protocol, e.g. h2; empty when it is null." }, + "cookies": { "type": "array", "maxItems": 0, "description": "Always empty; the Cookie header is in headers." }, + "headers": { "type": "array", "items": { "$ref": "#/components/schemas/HarNameValue" } }, + "queryString": { "type": "array", "items": { "$ref": "#/components/schemas/HarNameValue" }, "description": "The URL's query parameters, decoded." }, + "postData": { + "type": "object", + "description": "Present when the request body was kept, as text or as bytes.", + "required": ["mimeType", "text"], + "properties": { + "mimeType": { "type": "string", "description": "requestContentType; empty when it is null." }, + "text": { "type": "string", "description": "The redacted text, or the bytes in base64 when _encoding is base64." }, + "_encoding": { "const": "base64", "description": "Present when the body was kept as bytes." } + } + }, + "headersSize": { "const": -1 }, + "bodySize": { "type": "integer", "description": "requestBodyBytes; -1 when it is null." } + } + }, + "response": { + "type": "object", + "required": ["status", "statusText", "httpVersion", "cookies", "headers", "content", "redirectURL", "headersSize", "bodySize"], + "properties": { + "status": { "type": "integer", "description": "statusCode; 0 before a response and for a failed request." }, + "statusText": { "const": "", "description": "The capture keeps no reason phrase." }, + "httpVersion": { "type": "string" }, + "cookies": { "type": "array", "maxItems": 0, "description": "Always empty; the Set-Cookie header is in headers." }, + "headers": { "type": "array", "items": { "$ref": "#/components/schemas/HarNameValue" } }, + "content": { + "type": "object", + "required": ["size", "mimeType"], + "properties": { + "size": { "type": "integer", "minimum": 0, "description": "The larger of responseBodyBytes and the size of the body kept, which is more for a body kept inflated." }, + "mimeType": { "type": "string", "description": "responseContentType, or x-unknown when it is null." }, + "text": { "type": "string", "description": "The redacted text, or the bytes in base64 when encoding is base64. Absent when no body was kept." }, + "encoding": { "const": "base64" } + } + }, + "redirectURL": { "type": "string", "description": "The Location header, or empty." }, + "headersSize": { "const": -1 }, + "bodySize": { "type": "integer", "description": "responseBodyBytes; -1 when it is null." }, + "_error": { "type": "string", "description": "error, for a failed request, where Chrome's own export puts it." } + } + }, + "cache": { "type": "object", "maxProperties": 0 }, + "_resourceType": { "type": "string", "enum": ["fetch", "image"], "description": "How Chrome DevTools files the request on import: image for an image/* response, else fetch." }, + "timings": { + "type": "object", + "required": ["send", "wait", "receive"], + "properties": { + "blocked": { "const": -1 }, + "dns": { "const": -1 }, + "connect": { "const": -1 }, + "ssl": { "const": -1 }, + "send": { "const": 0 }, + "wait": { "type": "integer", "minimum": 0, "description": "durationMs; 0 while the request is in flight." }, + "receive": { "const": 0 } + } + }, + "_lustro": { + "type": "object", + "description": "What HAR has no field for. HAR reserves names that start with an underscore for a tool's own fields.", + "required": ["id", "isMocked", "categories", "requestBodyTruncated", "responseBodyTruncated", "responseComplete", "error"], + "properties": { + "id": { "type": "string", "description": "The transaction id." }, + "isMocked": { "type": "boolean" }, + "categories": { "type": "array", "items": { "type": "string" } }, + "requestBodyTruncated": { "type": "boolean", "description": "true when postData has only the first part of the body." }, + "responseBodyTruncated": { "type": "boolean", "description": "true when content has only the first part of the body." }, + "responseComplete": { "type": "boolean", "description": "false while the request is in flight, including a streaming response." }, + "error": { "type": ["string", "null"] } + } + } + } + }, + "HarNameValue": { + "type": "object", + "required": ["name", "value"], + "properties": { "name": { "type": "string" }, "value": { "type": "string" } } + }, "MockRule": { "type": "object", "required": ["id", "urlPattern", "statusCode", "enabled"], diff --git a/lustro/src/main/kotlin/io/github/twinsen81/lustro/internal/network/HarExport.kt b/lustro/src/main/kotlin/io/github/twinsen81/lustro/internal/network/HarExport.kt new file mode 100644 index 0000000..41e0394 --- /dev/null +++ b/lustro/src/main/kotlin/io/github/twinsen81/lustro/internal/network/HarExport.kt @@ -0,0 +1,159 @@ +package io.github.twinsen81.lustro.internal.network + +import io.github.twinsen81.lustro.escapeForJson +import java.time.Instant +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import java.util.Base64 +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okio.utf8Size + +/** + * Writes captured transactions as a HAR 1.2 document, the archive format that + * browser devtools and most HTTP tools import. + * + * The capture has no phase timings, so each entry spends its whole duration in + * `wait`. Headers, the URL, and text bodies are the redacted values the store + * keeps, and a body kept as bytes goes out in base64. What HAR has no field + * for goes in `_lustro`, the prefix HAR reserves for a tool's own fields. + */ +internal object HarExport { + private val STARTED_DATE_TIME: DateTimeFormatter = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(ZoneOffset.UTC) + + // What browser devtools write when a response has no media type. + private const val UNKNOWN_MIME_TYPE = "x-unknown" + + fun write(transactions: List, creatorVersion: String): String = + buildString { + append("{\"log\":{\"version\":\"1.2\",") + append("\"creator\":{\"name\":\"Lustro\",\"version\":").appendString(creatorVersion).append("},") + append("\"entries\":[") + transactions.forEachIndexed { index, tx -> + if (index > 0) append(',') + appendEntry(tx) + } + append("]}}") + } + + private fun StringBuilder.appendEntry(tx: NetworkTransaction) { + // An in-flight request has no duration yet, and HAR has no null for one. + val durationMs = (tx.durationMs ?: 0L).coerceAtLeast(0L) + append('{') + append("\"startedDateTime\":\"").append(STARTED_DATE_TIME.format(Instant.ofEpochMilli(tx.startedAt))).append("\",") + append("\"time\":").append(durationMs).append(',') + appendRequest(tx) + append(',') + appendResponse(tx) + append(",\"cache\":{},") + append("\"timings\":{\"blocked\":-1,\"dns\":-1,\"connect\":-1,\"ssl\":-1,") + append("\"send\":0,\"wait\":").append(durationMs).append(",\"receive\":0},") + // Chrome's import files a request by this, and by the media type without + // it, which would list a JSON response as a script. + val image = tx.responseContentType?.startsWith("image/", ignoreCase = true) == true + append("\"_resourceType\":\"").append(if (image) "image" else "fetch").append("\",") + append("\"_lustro\":{") + append("\"id\":").appendString(tx.id).append(',') + append("\"isMocked\":").append(tx.isMocked).append(',') + append("\"categories\":[") + tx.categories.forEachIndexed { index, category -> + if (index > 0) append(',') + appendString(category) + } + append("],") + append("\"requestBodyTruncated\":").append(tx.requestBodyTruncated).append(',') + append("\"responseBodyTruncated\":").append(tx.responseBodyTruncated).append(',') + append("\"responseComplete\":").append(tx.responseComplete).append(',') + append("\"error\":") + tx.error?.let { appendString(it) } ?: append("null") + append("}}") + } + + private fun StringBuilder.appendRequest(tx: NetworkTransaction) { + append("\"request\":{") + append("\"method\":").appendString(tx.method).append(',') + append("\"url\":").appendString(tx.url).append(',') + append("\"httpVersion\":").appendString(tx.protocol.orEmpty()).append(',') + append("\"cookies\":[],") + append("\"headers\":") + appendHeaders(tx.requestHeaders) + append(",\"queryString\":") + appendQueryString(tx.url) + val text = tx.requestBody + val bytes = tx.requestBinaryBody + if (text != null || bytes != null) { + append(",\"postData\":{\"mimeType\":").appendString(tx.requestContentType.orEmpty()) + append(",\"text\":").appendString(text ?: base64(bytes)) + // HAR gives postData no encoding field, so this one is ours. + if (text == null) append(",\"_encoding\":\"base64\"") + append('}') + } + append(",\"headersSize\":-1,") + append("\"bodySize\":").append(tx.requestBodyBytes ?: -1L) + append('}') + } + + private fun StringBuilder.appendResponse(tx: NetworkTransaction) { + val headers = tx.responseHeaders.orEmpty() + append("\"response\":{") + append("\"status\":").append(tx.statusCode ?: 0).append(',') + // The capture keeps no reason phrase, and HTTP/2 has none. + append("\"statusText\":\"\",") + append("\"httpVersion\":").appendString(tx.protocol.orEmpty()).append(',') + append("\"cookies\":[],") + append("\"headers\":") + appendHeaders(headers) + append(',') + appendContent(tx) + val location = headers.entries.firstOrNull { it.key.equals("Location", ignoreCase = true) }?.value + append(",\"redirectURL\":").appendString(location.orEmpty()).append(',') + append("\"headersSize\":-1,") + append("\"bodySize\":").append(tx.responseBodyBytes ?: -1L) + // Where Chrome's own export puts why a request failed. + tx.error?.let { append(",\"_error\":").appendString(it) } + append('}') + } + + private fun StringBuilder.appendContent(tx: NetworkTransaction) { + val text = tx.responseBody + val bytes = tx.responseBinaryBody + // content.size is the size of the decoded body, and a gzip or deflate body + // is kept inflated, so what was kept can be more than its size on the wire. + val retained = text?.utf8Size() ?: bytes?.size?.toLong() ?: 0L + append("\"content\":{") + append("\"size\":").append(maxOf(tx.responseBodyBytes ?: 0L, retained)).append(',') + append("\"mimeType\":").appendString(tx.responseContentType ?: UNKNOWN_MIME_TYPE) + when { + text != null -> append(",\"text\":").appendString(text) + bytes != null -> append(",\"text\":").appendString(base64(bytes)).append(",\"encoding\":\"base64\"") + } + append('}') + } + + private fun StringBuilder.appendHeaders(headers: Map) { + append('[') + headers.entries.forEachIndexed { index, (name, value) -> + if (index > 0) append(',') + append("{\"name\":").appendString(name).append(",\"value\":").appendString(value).append('}') + } + append(']') + } + + private fun StringBuilder.appendQueryString(url: String) { + append('[') + val parsed = url.toHttpUrlOrNull() + if (parsed != null) { + for (index in 0 until parsed.querySize) { + if (index > 0) append(',') + append("{\"name\":").appendString(parsed.queryParameterName(index)) + append(",\"value\":").appendString(parsed.queryParameterValue(index).orEmpty()).append('}') + } + } + append(']') + } + + private fun base64(bytes: ByteArray?): String = Base64.getEncoder().encodeToString(bytes ?: ByteArray(0)) + + private fun StringBuilder.appendString(value: String): StringBuilder = + append('"').append(value.escapeForJson()).append('"') +} diff --git a/lustro/src/main/kotlin/io/github/twinsen81/lustro/network/NetworkDebugTab.kt b/lustro/src/main/kotlin/io/github/twinsen81/lustro/network/NetworkDebugTab.kt index a86ebbc..5b621a6 100644 --- a/lustro/src/main/kotlin/io/github/twinsen81/lustro/network/NetworkDebugTab.kt +++ b/lustro/src/main/kotlin/io/github/twinsen81/lustro/network/NetworkDebugTab.kt @@ -8,6 +8,8 @@ import io.github.twinsen81.lustro.ExperimentalPlatformCapture import io.github.twinsen81.lustro.Headers import io.github.twinsen81.lustro.MediaType import io.github.twinsen81.lustro.escapeForJson +import io.github.twinsen81.lustro.internal.Versions +import io.github.twinsen81.lustro.internal.network.HarExport import io.github.twinsen81.lustro.internal.network.HttpUrlConnectionCapture import io.github.twinsen81.lustro.internal.network.LustroNetworkInterceptor import io.github.twinsen81.lustro.internal.network.MockRuleCodec @@ -134,7 +136,8 @@ public class NetworkDebugTab private constructor(

Network Traffic

0 requests - + + Method URL Status @@ -174,6 +183,7 @@ public class NetworkDebugTab private constructor(
+
@@ -224,6 +234,7 @@ public class NetworkDebugTab private constructor( val body = request.bodyAsString() return when { path == "transactions" && method == "GET" -> handleTransactions(request) + path == "transactions/_/export" && method == "GET" -> handleExport(request) path.startsWith("transactions/") && method == "GET" -> handleTransactionPath(path.removePrefix("transactions/").split('/')) path == "clear" && method == "POST" -> handleClear() @@ -329,6 +340,25 @@ public class NetworkDebugTab private constructor( return DebugResponse.bytes(body, contentType, headers = headers) } + /** + * The transactions named by `ids`, or every one when it is left out, as a HAR + * document, oldest first. An id the store no longer has is left out, so a + * selection taken just before an eviction still exports what remains. + */ + private fun handleExport(request: DebugRequest): DebugResponse { + val format = request.queryParam("format") ?: EXPORT_FORMAT_HAR + if (!format.equals(EXPORT_FORMAT_HAR, ignoreCase = true)) { + return DebugResponse.error("Unknown export format '$format'; the only one is har", field = "format") + } + // Repeated ids parameters and comma-separated lists both work. An ids + // parameter that names none is an empty selection, not every transaction. + val ids = request.queryParams["ids"]?.flatMap { it.split(',') }?.map { it.trim() }?.filter { it.isNotEmpty() } + val transactions = + if (ids == null) store.getTransactions() else ids.distinct().mapNotNull { store.getTransaction(it) } + val oldestFirst = transactions.sortedWith(compareBy({ it.startedAt }, { it.startOrder })) + return DebugResponse.ok(HarExport.write(oldestFirst, Versions.LIBRARY_VERSION)) + } + // The filter's counts restart with the list, so they describe the requests // missing from what is on screen. private fun handleClear(): DebugResponse { @@ -713,6 +743,8 @@ public class NetworkDebugTab private constructor( private const val SEND_CANCEL_GRACE_MS = 1_000L + private const val EXPORT_FORMAT_HAR = "har" + // Types the console shows in an . Browsers render them without script. private val INLINE_BODY_TYPES = setOf("image/png", "image/jpeg", "image/gif", "image/webp") diff --git a/lustro/src/test/js/harness.js b/lustro/src/test/js/harness.js index 5999115..37b6e51 100644 --- a/lustro/src/test/js/harness.js +++ b/lustro/src/test/js/harness.js @@ -77,4 +77,14 @@ function loadSharedJs() { return sandbox; } -module.exports = { loadSharedJs }; +// shared.js, then the Network tab's network.js in the same context. The tab +// registers its DOM setup with lustroOnContentReady, which never fires here, so +// only its top-level functions run. +function loadNetworkJs() { + const sandbox = loadSharedJs(); + const source = fs.readFileSync(path.join(ASSET_DIR, 'network.js'), 'utf8'); + vm.runInContext(source, sandbox, { filename: 'network.js' }); + return sandbox; +} + +module.exports = { loadSharedJs, loadNetworkJs }; diff --git a/lustro/src/test/js/network-export.test.js b/lustro/src/test/js/network-export.test.js new file mode 100644 index 0000000..19ab3da --- /dev/null +++ b/lustro/src/test/js/network-export.test.js @@ -0,0 +1,180 @@ +// The Network tab's exports: a transaction as Markdown, and the HAR batches the +// console joins into one file. +// +// The Markdown goes into bug reports, pull requests, and chats, where a body +// that closes its own code block or a URL read as markup would break the rest +// of the document. So each test pins the text exactly. +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); +const { loadNetworkJs } = require('./harness.js'); + +const network = loadNetworkJs(); +const markdown = (...txs) => network.netTransactionsMarkdown(txs); + +const GOLDEN_DIR = path.join(__dirname, '..', '..', '..', '..', 'wire-protocol', 'v1', 'golden'); +const golden = (name) => JSON.parse(fs.readFileSync(path.join(GOLDEN_DIR, name), 'utf8')); + +function detail(fields) { + return Object.assign({ + id: 'tx_1', + timestamp: '14:22:09.003', + startedAt: 1790605329003, + method: 'GET', + url: 'https://api.example.com/v1/orders', + protocol: 'h2', + statusCode: 200, + durationMs: 12, + categories: [], + isMocked: false, + requestContentType: null, + responseContentType: 'application/json', + responseComplete: true, + error: null, + requestHeaders: {}, + requestBody: null, + requestBodyTruncated: false, + requestBodyBinary: false, + responseHeaders: {}, + responseBody: null, + responseBodyTruncated: false, + responseBodyBinary: false, + }, fields); +} + +test('a transaction renders as a heading, its meta, and fenced headers and bodies', () => { + assert.strictEqual(markdown(golden('transaction.json')), [ + '## `GET https://api.example.com/v1/orders/77e2`', + '', + '**500** · 41 ms · 2026-09-28T14:22:11.781Z · mocked by Lustro · `api`', + '', + '### Request headers', + '', + '```http', + 'Accept: application/json', + 'Authorization: ', + '```', + '', + '### Response headers', + '', + '```http', + 'Content-Type: application/json', + '```', + '', + '### Response body', + '', + '```json', + '{', + ' "error": "boom"', + '}', + '```', + '', + ].join('\n')); +}); + +test('a JSON body is indented without changing a value', () => { + const out = markdown(detail({ responseBody: '{"big":12345678901234567890,"n":1.10,"s":"\\u00e9"}' })); + assert.ok(out.includes('```json\n{\n "big": 12345678901234567890,\n "n": 1.10,\n "s": "\\u00e9"\n}\n```'), out); +}); + +test('a body cut at the capture cap stays as it is, and says it was cut', () => { + const out = markdown(detail({ responseBody: '{"items":[1,2', responseBodyTruncated: true })); + assert.ok(out.includes('### Response body\n\n_Truncated: Lustro kept only the first part of this body._\n\n```json\n{"items":[1,2\n```'), out); +}); + +test('a truncated request body is marked too', () => { + const out = markdown(detail({ method: 'POST', requestBody: 'abc', requestContentType: 'text/plain', requestBodyTruncated: true })); + assert.ok(out.includes('### Request body\n\n_Truncated: Lustro kept only the first part of this body._\n\n```text\nabc\n```'), out); +}); + +test('a body that holds backticks gets a longer fence', () => { + const body = 'see:\n```\ncode\n````\n'; + const out = markdown(detail({ responseBody: body, responseContentType: 'text/markdown' })); + assert.ok(out.includes('`````markdown\n' + body + '`````'), out); +}); + +test('a URL with backticks or markup stays one code span', () => { + const out = markdown(detail({ url: 'https://x.test/a`b?q=*x*#h' })); + assert.ok(out.startsWith('## ``GET https://x.test/a`b?q=*x*#h``\n'), out); + // A span that ends with a backtick is padded, and Markdown drops one space from each side. + const edge = markdown(detail({ method: 'GET', url: '`' })); + assert.ok(edge.startsWith('## `` GET ` ``\n'), edge); +}); + +test('each body gets a language for its content type', () => { + const cases = [ + ['application/problem+json', '{"a":1}', 'json'], + ['text/html; charset=utf-8', '

hi

', 'html'], + ['application/xml', '', 'xml'], + ['application/atom+xml', '', 'xml'], + ['text/css', 'a{}', 'css'], + ['application/javascript', 'x()', 'javascript'], + ['application/x-www-form-urlencoded', 'a=1&b=2', 'text'], + ['text/plain', 'hello', 'text'], + [null, 'hello', 'text'], + // JSON sent as text is still JSON. + ['text/plain', '[1]', 'json'], + ]; + for (const [type, body, language] of cases) { + const out = markdown(detail({ responseContentType: type, responseBody: body })); + assert.ok(out.includes('```' + language + '\n'), type + ': ' + out); + } +}); + +test('a body kept as bytes is named, not included', () => { + const out = markdown(golden('transaction-image.json')); + assert.ok(out.includes('### Response body\n\n_image/png body, 5.1 KB: kept as bytes, so it is not included._'), out); +}); + +test('a body kept as bytes and cut at the capture cap says both', () => { + const out = markdown(detail({ responseBody: null, responseBodyBinary: true, responseBodyTruncated: true, + responseContentType: 'image/jpeg', responseBodyBytes: 300000 })); + assert.ok(out.includes('_image/jpeg body, 293.0 KB: kept as bytes, so it is not included. Lustro kept only its first part._'), out); +}); + +test('a body in an encoding Lustro does not decode says so', () => { + const out = markdown(detail({ responseHeaders: { 'Content-Encoding': 'br' }, responseBody: null })); + assert.ok(out.includes('### Response body\n\n_Not captured: Lustro does not decode the br encoding._'), out); +}); + +test('a request with no body and no headers has no sections for them', () => { + const out = markdown(detail({ responseBody: '' })); + assert.strictEqual(out, '## `GET https://api.example.com/v1/orders`\n\n**200** · 12 ms · 2026-09-28T14:22:09.003Z · `h2`\n'); +}); + +test('a request in flight, a stream, and a failure say so in the meta line', () => { + const pending = markdown(detail({ statusCode: null, durationMs: null, protocol: null, responseComplete: false, responseHeaders: null })); + assert.ok(pending.includes('\n\n**Pending** · 2026-09-28T14:22:09.003Z\n'), pending); + const streaming = markdown(detail({ responseComplete: false })); + assert.ok(streaming.includes('**200, streaming**'), streaming); + const failed = markdown(detail({ statusCode: null, error: 'java.net.SocketTimeoutException: timeout', protocol: null })); + assert.ok(failed.includes('**Failed** · 12 ms · 2026-09-28T14:22:09.003Z\n\n**Error:** `java.net.SocketTimeoutException: timeout`'), failed); +}); + +test('a selection is one document, with a rule between transactions', () => { + const out = markdown(detail({ id: 'a', url: 'https://x.test/a' }), detail({ id: 'b', url: 'https://x.test/b' })); + const parts = out.split('\n\n---\n\n'); + assert.strictEqual(parts.length, 2); + assert.ok(parts[0].startsWith('## `GET https://x.test/a`')); + assert.ok(parts[1].startsWith('## `GET https://x.test/b`')); + assert.ok(out.endsWith('\n') && !out.endsWith('\n\n')); +}); + +test('HAR batches join into one log, oldest first', () => { + const entry = (id, started) => ({ startedDateTime: started, _lustro: { id } }); + const har = (...entries) => ({ log: { version: '1.2', creator: { name: 'Lustro', version: '0.1.0' }, entries } }); + const merged = network.netMergeHar([ + har(entry('b', '2026-09-28T14:22:09.003Z'), entry('d', '2026-09-28T14:22:11.000Z')), + har(entry('a', '2026-09-28T14:22:07.512Z'), entry('c', '2026-09-28T14:22:09.003Z')), + ]); + assert.deepStrictEqual(merged.log.entries.map((e) => e._lustro.id), ['a', 'b', 'c', 'd']); + assert.strictEqual(merged.log.creator.name, 'Lustro'); +}); + +test('the golden HAR merges unchanged', () => { + const merged = network.netMergeHar([golden('export-har.json')]); + assert.deepStrictEqual(merged, golden('export-har.json')); +}); diff --git a/lustro/src/test/kotlin/io/github/twinsen81/lustro/internal/network/HarExportGoldenTest.kt b/lustro/src/test/kotlin/io/github/twinsen81/lustro/internal/network/HarExportGoldenTest.kt new file mode 100644 index 0000000..bc6eef9 --- /dev/null +++ b/lustro/src/test/kotlin/io/github/twinsen81/lustro/internal/network/HarExportGoldenTest.kt @@ -0,0 +1,131 @@ +package io.github.twinsen81.lustro.internal.network + +import java.io.File +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * Pins the golden HAR fixture to what [HarExport] writes, so the fixture that + * clients test against can't drift from the server. + */ +class HarExportGoldenTest { + private val gif = + byteArrayOf( + 71, 73, 70, 56, 57, 97, 1, 0, 1, 0, -128, 0, 0, -1, -1, -1, 0, 0, 0, 33, -7, 4, 1, 0, 0, 0, 0, + 44, 0, 0, 0, 0, 1, 0, 1, 0, 0, 2, 2, 68, 1, 0, 59, + ) + + private val transactions = + listOf( + NetworkTransaction( + id = "tx_4b1d77a0", + startedAt = 1790605329003, + completedAt = 1790605329325, + durationMs = 322, + categories = listOf("api"), + method = "POST", + url = "https://api.example.com/v1/orders", + requestHeaders = + mapOf("Content-Type" to "application/json; charset=utf-8", "Authorization" to "[REDACTED]"), + requestBody = """{"sku":"A-1","quantity":2}""", + requestContentType = "application/json; charset=utf-8", + requestBodyBytes = 26, + protocol = "h2", + statusCode = 201, + responseHeaders = mapOf("Content-Type" to "application/json; charset=utf-8", "Location" to "/v1/orders/9c2e"), + responseBody = """{"id":"9c2e","sku":"A-1","quantity":2,"status":"open"}""", + responseContentType = "application/json; charset=utf-8", + responseBodyBytes = 54, + responseComplete = true, + ), + NetworkTransaction( + id = "tx_77e2c014", + startedAt = 1790605331781, + completedAt = 1790605331822, + durationMs = 41, + categories = listOf("api"), + method = "GET", + url = "https://api.example.com/v1/orders/77e2", + requestHeaders = mapOf("Accept" to "application/json", "Authorization" to "[REDACTED]"), + requestBodyBytes = 0, + statusCode = 500, + responseHeaders = mapOf("Content-Type" to "application/json"), + responseBody = """{"error":"boom"}""", + responseContentType = "application/json", + responseBodyBytes = 16, + responseComplete = true, + isMocked = true, + ), + NetworkTransaction( + id = "tx_e6b0d413", + startedAt = 1790605333140, + completedAt = 1790605333175, + durationMs = 35, + categories = listOf("media"), + method = "GET", + url = "https://cdn.example.com/pixel.gif", + requestHeaders = mapOf("Accept" to "image/*"), + protocol = "h2", + statusCode = 200, + responseHeaders = mapOf("Content-Type" to "image/gif", "Content-Length" to "43"), + responseBinaryBody = gif, + responseContentType = "image/gif", + responseBodyBytes = 43, + responseComplete = true, + ), + NetworkTransaction( + id = "tx_5e8a2f61", + startedAt = 1790605334022, + completedAt = 1790605335232, + durationMs = 1210, + categories = listOf("api"), + method = "GET", + url = "https://api.example.com/v1/catalog?page=2&tag=new%20in", + requestHeaders = mapOf("Accept" to "application/json"), + protocol = "h2", + statusCode = 200, + responseHeaders = mapOf("Content-Type" to "application/json; charset=utf-8"), + responseBody = """[{"sku":"A-1","name":"Desk lamp"},{"sku":"A-2","na""", + responseBodyTruncated = true, + responseContentType = "application/json; charset=utf-8", + responseBodyBytes = 524288, + responseComplete = true, + ), + NetworkTransaction( + id = "tx_d03a9b44", + startedAt = 1790605335917, + completedAt = 1790605335925, + durationMs = 8, + method = "GET", + url = "https://telemetry.example.com/v1/events", + requestHeaders = mapOf("Accept" to "application/json"), + responseComplete = true, + error = "java.net.UnknownHostException: Unable to resolve host \"telemetry.example.com\"", + ), + ) + + @Test + fun `the golden HAR fixture is what the export writes`() { + val written = HarExport.write(transactions, creatorVersion = "0.1.0") + assertEquals( + "$GOLDEN differs from:\n$written\n", + canonical(JSONObject(File(GOLDEN).readText())), + canonical(JSONObject(written)), + ) + } + + // Key order is not part of JSON, so objects compare as sorted maps. + private fun canonical(value: Any?): Any? = + when (value) { + is JSONObject -> value.keys().asSequence().associateWith { canonical(value.get(it)) }.toSortedMap() + is JSONArray -> (0 until value.length()).map { canonical(value.get(it)) } + else -> value + } + + private companion object { + // Unit tests run in the module directory. + const val GOLDEN = "../wire-protocol/v1/golden/export-har.json" + } +} diff --git a/lustro/src/test/kotlin/io/github/twinsen81/lustro/network/NetworkHarExportTest.kt b/lustro/src/test/kotlin/io/github/twinsen81/lustro/network/NetworkHarExportTest.kt new file mode 100644 index 0000000..8d2fa75 --- /dev/null +++ b/lustro/src/test/kotlin/io/github/twinsen81/lustro/network/NetworkHarExportTest.kt @@ -0,0 +1,289 @@ +package io.github.twinsen81.lustro.network + +import io.github.twinsen81.lustro.DebugRequest +import io.github.twinsen81.lustro.DebugResponse +import io.github.twinsen81.lustro.Headers +import io.github.twinsen81.lustro.MediaType +import io.github.twinsen81.lustro.internal.network.NetworkTrafficStore +import java.time.Instant +import java.util.Base64 +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +/** Tests for `GET transactions/_/export`, the HAR export of captured transactions. */ +class NetworkHarExportTest { + private val tab = NetworkDebugTab.create(classifier = NetworkClassifier { url -> if ("sync" in url) listOf("sync") else emptyList() }) + private val store = tab.captureSink as NetworkTrafficStore + + private fun export(vararg query: Pair): DebugResponse = + tab.handle( + DebugRequest( + path = "transactions/_/export", + method = "GET", + queryParams = query.groupBy({ it.first }, { it.second }), + ), + )!! + + private fun entries(vararg query: Pair): JSONArray { + val res = export(*query) + assertEquals(200, res.status) + return JSONObject(res.body.toString(Charsets.UTF_8)).getJSONObject("log").getJSONArray("entries") + } + + private fun JSONArray.objects(): List = (0 until length()).map { getJSONObject(it) } + + private fun JSONArray.headers(): Map = + objects().associate { it.getString("name") to it.getString("value") } + + private fun record( + url: String = "https://example.com/a", + method: String = "GET", + requestHeaders: Headers = Headers.EMPTY, + requestBody: CapturedBody? = null, + requestType: MediaType? = null, + response: CapturedResponse? = CapturedResponse.Builder(200, 5).build(), + ): String { + val id = store.beginRequest(url, method, requestHeaders, requestBody, requestType) + response?.let { store.completeRequest(id, it) } + assertTrue(store.awaitCaptures()) + return id.value + } + + @Test + fun `the export is a HAR 1_2 log of every transaction, oldest first`() { + val first = record(url = "https://example.com/first") + val second = record(url = "https://example.com/second") + + val res = export("format" to "har") + assertEquals("application/json; charset=utf-8", res.contentType.toString()) + val log = JSONObject(res.body.toString(Charsets.UTF_8)).getJSONObject("log") + assertEquals("1.2", log.getString("version")) + assertEquals("Lustro", log.getJSONObject("creator").getString("name")) + assertTrue(log.getJSONObject("creator").getString("version").isNotEmpty()) + val ids = log.getJSONArray("entries").objects().map { it.getJSONObject("_lustro").getString("id") } + assertEquals(listOf(first, second), ids) + } + + @Test + fun `format defaults to har`() { + record() + assertEquals(1, entries().length()) + } + + @Test + fun `an unknown format is a 400 naming the field`() { + val res = export("format" to "csv") + assertEquals(400, res.status) + val error = JSONObject(res.body.toString(Charsets.UTF_8)) + assertEquals("bad_request", error.getString("error")) + assertEquals("format", error.getString("field")) + } + + @Test + fun `ids selects transactions and leaves out the ones the store no longer has`() { + val a = record(url = "https://example.com/a") + record(url = "https://example.com/b") + val c = record(url = "https://example.com/c") + + fun exported(vararg query: Pair) = + entries(*query).objects().map { it.getJSONObject("_lustro").getString("id") } + + assertEquals(listOf(a, c), exported("ids" to "$c, $a,gone,$c")) + // Repeated parameters work too. + assertEquals(listOf(a, c), exported("ids" to a, "ids" to c)) + // An ids parameter that names nothing is an empty selection, not everything. + assertEquals(emptyList(), exported("ids" to "")) + } + + @Test + fun `an entry maps the request, the response, and the timings`() { + val id = + record( + url = "https://example.com/orders?q=a%20b&flag", + method = "POST", + requestHeaders = Headers.of("Accept" to "application/json"), + requestBody = CapturedBody("""{"item":1}""", byteSize = 10), + requestType = MediaType.JSON, + response = + CapturedResponse.Builder(302, 41) + .headers(Headers.of("Content-Type" to "text/plain", "location" to "/next")) + .body(CapturedBody("moved", byteSize = 5)) + .protocol("h2") + .build(), + ) + val detail = JSONObject(tab.handle(DebugRequest(path = "transactions/$id", method = "GET"))!!.body.toString(Charsets.UTF_8)) + + val entry = entries().getJSONObject(0) + val startedDateTime = entry.getString("startedDateTime") + assertTrue(startedDateTime, Regex("""\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z""").matches(startedDateTime)) + assertEquals(detail.getLong("startedAt"), Instant.parse(startedDateTime).toEpochMilli()) + assertEquals(41, entry.getLong("time")) + assertEquals(0, entry.getJSONObject("cache").length()) + assertEquals("fetch", entry.getString("_resourceType")) + val timings = entry.getJSONObject("timings") + assertEquals(0, timings.getLong("send")) + assertEquals(41, timings.getLong("wait")) + assertEquals(0, timings.getLong("receive")) + for (phase in listOf("blocked", "dns", "connect", "ssl")) assertEquals(phase, -1, timings.getLong(phase)) + + val request = entry.getJSONObject("request") + assertEquals("POST", request.getString("method")) + assertEquals("https://example.com/orders?q=a%20b&flag", request.getString("url")) + assertEquals("h2", request.getString("httpVersion")) + assertEquals(0, request.getJSONArray("cookies").length()) + assertEquals(mapOf("Accept" to "application/json"), request.getJSONArray("headers").headers()) + assertEquals(mapOf("q" to "a b", "flag" to ""), request.getJSONArray("queryString").headers()) + val postData = request.getJSONObject("postData") + assertEquals("application/json; charset=utf-8", postData.getString("mimeType")) + assertEquals("""{"item":1}""", postData.getString("text")) + assertFalse(postData.has("_encoding")) + assertEquals(-1, request.getLong("headersSize")) + assertEquals(10, request.getLong("bodySize")) + + val response = entry.getJSONObject("response") + assertEquals(302, response.getInt("status")) + assertEquals("", response.getString("statusText")) + assertEquals("h2", response.getString("httpVersion")) + assertEquals("/next", response.getString("redirectURL")) + assertEquals("text/plain", response.getJSONArray("headers").headers()["Content-Type"]) + val content = response.getJSONObject("content") + assertEquals(5, content.getLong("size")) + assertEquals("text/plain", content.getString("mimeType")) + assertEquals("moved", content.getString("text")) + assertFalse(content.has("encoding")) + assertEquals(-1, response.getLong("headersSize")) + assertEquals(5, response.getLong("bodySize")) + assertFalse(response.has("_error")) + + val lustro = entry.getJSONObject("_lustro") + assertEquals(id, lustro.getString("id")) + assertFalse(lustro.getBoolean("isMocked")) + assertEquals(0, lustro.getJSONArray("categories").length()) + assertFalse(lustro.getBoolean("requestBodyTruncated")) + assertFalse(lustro.getBoolean("responseBodyTruncated")) + assertTrue(lustro.getBoolean("responseComplete")) + assertTrue(lustro.isNull("error")) + } + + @Test + fun `the export keeps headers and bodies redacted`() { + record( + method = "POST", + requestHeaders = Headers.of("Authorization" to "Bearer secret"), + requestBody = CapturedBody("""{"password":"hunter2"}"""), + requestType = MediaType.JSON, + ) + + val text = export().body.toString(Charsets.UTF_8) + assertFalse(text, "secret" in text) + assertFalse(text, "hunter2" in text) + } + + @Test + fun `a body kept as bytes goes out in base64`() { + val png = byteArrayOf(-119, 80, 78, 71, 13, 10, 26, 10) + record( + method = "PUT", + requestBody = CapturedBody(text = null, byteSize = 8, bytes = png), + requestType = MediaType.parse("image/png"), + response = + CapturedResponse.Builder(200, 5) + .headers(Headers.of("Content-Type" to "image/png")) + .body(CapturedBody(text = null, byteSize = 8, bytes = png)) + .build(), + ) + + val entry = entries().getJSONObject(0) + assertEquals("image", entry.getString("_resourceType")) + val base64 = Base64.getEncoder().encodeToString(png) + val content = entry.getJSONObject("response").getJSONObject("content") + assertEquals(base64, content.getString("text")) + assertEquals("base64", content.getString("encoding")) + assertEquals(8, content.getLong("size")) + val postData = entry.getJSONObject("request").getJSONObject("postData") + assertEquals(base64, postData.getString("text")) + assertEquals("base64", postData.getString("_encoding")) + } + + @Test + fun `an inflated body reports its kept size as the content size`() { + val inflated = "x".repeat(100) + record( + response = + CapturedResponse.Builder(200, 5) + .headers(Headers.of("Content-Type" to "text/plain", "Content-Encoding" to "gzip")) + .body(CapturedBody(inflated, byteSize = 20)) + .build(), + ) + + val response = entries().getJSONObject(0).getJSONObject("response") + assertEquals(20, response.getLong("bodySize")) + assertEquals(100, response.getJSONObject("content").getLong("size")) + } + + @Test + fun `mocks, truncated bodies, and categories are marked in _lustro`() { + record( + url = "https://example.com/sync", + method = "POST", + requestBody = CapturedBody("abc", truncated = true, byteSize = 300_000), + requestType = MediaType.TEXT, + response = + CapturedResponse.Builder(500, 3) + .body(CapturedBody("def", truncated = true, byteSize = 400_000)) + .mocked(true) + .build(), + ) + + val lustro = entries().getJSONObject(0).getJSONObject("_lustro") + assertTrue(lustro.getBoolean("isMocked")) + assertTrue(lustro.getBoolean("requestBodyTruncated")) + assertTrue(lustro.getBoolean("responseBodyTruncated")) + assertEquals("sync", lustro.getJSONArray("categories").getString(0)) + } + + @Test + fun `a failed request carries its error`() { + val id = store.beginRequest("https://example.com/down", "GET", Headers.EMPTY, null, null) + store.failRequest(id, 12, "java.net.UnknownHostException: example.com") + assertTrue(store.awaitCaptures()) + + val entry = entries().getJSONObject(0) + val response = entry.getJSONObject("response") + assertEquals(0, response.getInt("status")) + assertEquals("java.net.UnknownHostException: example.com", response.getString("_error")) + assertEquals("java.net.UnknownHostException: example.com", entry.getJSONObject("_lustro").getString("error")) + assertEquals(12, entry.getLong("time")) + } + + @Test + fun `a request still in flight exports with no response yet`() { + record(response = null) + + val entry = entries().getJSONObject(0) + assertEquals(0, entry.getLong("time")) + assertEquals(0, entry.getJSONObject("timings").getLong("wait")) + assertFalse(entry.getJSONObject("request").has("postData")) + val response = entry.getJSONObject("response") + assertEquals(0, response.getInt("status")) + assertEquals("", response.getString("httpVersion")) + assertEquals(0, response.getJSONArray("headers").length()) + assertEquals("", response.getString("redirectURL")) + assertEquals(-1, response.getLong("bodySize")) + val content = response.getJSONObject("content") + assertEquals(0, content.getLong("size")) + assertEquals("x-unknown", content.getString("mimeType")) + assertFalse(content.has("text")) + assertFalse(entry.getJSONObject("_lustro").getBoolean("responseComplete")) + } + + @Test + fun `the export route takes GET only`() { + assertNull(tab.handle(DebugRequest(path = "transactions/_/export", method = "POST"))) + } +} diff --git a/wire-protocol/v1/README.md b/wire-protocol/v1/README.md index e2e1d69..08142b2 100644 --- a/wire-protocol/v1/README.md +++ b/wire-protocol/v1/README.md @@ -56,7 +56,10 @@ gets a `reset`. `GET network/transactions/{id}/body/{request|response}` returns a body as it is stored. The poll `state` gains `captureFilter`: `null` when the app set no capture filter, or its `description` and how many requests it `skipped` and - `failed` on, counted since the list was last cleared. + `failed` on, counted since the list was last cleared. The new route + `GET network/transactions/_/export?format=har&ids=...` returns the + transactions as a HAR 1.2 document, every one when `ids` is left out, with + the transaction id and the fields HAR has no place for in `_lustro`. ## Status diff --git a/wire-protocol/v1/golden/README.md b/wire-protocol/v1/golden/README.md index 87806cb..74e84af 100644 --- a/wire-protocol/v1/golden/README.md +++ b/wire-protocol/v1/golden/README.md @@ -22,6 +22,7 @@ with no source checkout required. | `transaction-image.json` | `GET network/transactions/{id}` detail of an image response | OpenAPI `Transaction` | | `rules-list.json` | `GET network/rules` | OpenAPI `listMockRules` 200 (`items: [MockRule]`) | | `send-result.json` | `POST network/send` synchronous result | OpenAPI `sendRequest` 200 | +| `export-har.json` | `GET network/transactions/_/export` HAR document | OpenAPI `HarDocument` | ## Notes on the shapes @@ -46,5 +47,13 @@ with no source checkout required. `GET network/transactions/{id}/body/response`, which is not JSON and so has no fixture. +- **HAR export.** `export-har.json` is the export of five transactions: a POST + with a JSON body, the mocked `tx_77e2c014`, a GIF kept as bytes (base64 with + `encoding`), a response cut at the capture cap + (`_lustro.responseBodyTruncated`), and a failed request (status `0`, with + the error in `response._error` and `_lustro.error`). A unit test in + `:lustro` checks that the export writes exactly this document for those + transactions. + When the server's emitted shapes change, update these fixtures in the same change that bumps the protocol version, and keep the schema validation green. diff --git a/wire-protocol/v1/golden/export-har.json b/wire-protocol/v1/golden/export-har.json new file mode 100644 index 0000000..c0a7ff0 --- /dev/null +++ b/wire-protocol/v1/golden/export-har.json @@ -0,0 +1,339 @@ +{ + "log": { + "version": "1.2", + "creator": { + "name": "Lustro", + "version": "0.1.0" + }, + "entries": [ + { + "startedDateTime": "2026-09-28T14:22:09.003Z", + "time": 322, + "request": { + "method": "POST", + "url": "https://api.example.com/v1/orders", + "httpVersion": "h2", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "name": "Authorization", + "value": "[REDACTED]" + } + ], + "queryString": [], + "postData": { + "mimeType": "application/json; charset=utf-8", + "text": "{\"sku\":\"A-1\",\"quantity\":2}" + }, + "headersSize": -1, + "bodySize": 26 + }, + "response": { + "status": 201, + "statusText": "", + "httpVersion": "h2", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "name": "Location", + "value": "/v1/orders/9c2e" + } + ], + "content": { + "size": 54, + "mimeType": "application/json; charset=utf-8", + "text": "{\"id\":\"9c2e\",\"sku\":\"A-1\",\"quantity\":2,\"status\":\"open\"}" + }, + "redirectURL": "/v1/orders/9c2e", + "headersSize": -1, + "bodySize": 54 + }, + "cache": {}, + "timings": { + "blocked": -1, + "dns": -1, + "connect": -1, + "ssl": -1, + "send": 0, + "wait": 322, + "receive": 0 + }, + "_resourceType": "fetch", + "_lustro": { + "id": "tx_4b1d77a0", + "isMocked": false, + "categories": [ + "api" + ], + "requestBodyTruncated": false, + "responseBodyTruncated": false, + "responseComplete": true, + "error": null + } + }, + { + "startedDateTime": "2026-09-28T14:22:11.781Z", + "time": 41, + "request": { + "method": "GET", + "url": "https://api.example.com/v1/orders/77e2", + "httpVersion": "", + "cookies": [], + "headers": [ + { + "name": "Accept", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "[REDACTED]" + } + ], + "queryString": [], + "headersSize": -1, + "bodySize": 0 + }, + "response": { + "status": 500, + "statusText": "", + "httpVersion": "", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "application/json" + } + ], + "content": { + "size": 16, + "mimeType": "application/json", + "text": "{\"error\":\"boom\"}" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 16 + }, + "cache": {}, + "timings": { + "blocked": -1, + "dns": -1, + "connect": -1, + "ssl": -1, + "send": 0, + "wait": 41, + "receive": 0 + }, + "_resourceType": "fetch", + "_lustro": { + "id": "tx_77e2c014", + "isMocked": true, + "categories": [ + "api" + ], + "requestBodyTruncated": false, + "responseBodyTruncated": false, + "responseComplete": true, + "error": null + } + }, + { + "startedDateTime": "2026-09-28T14:22:13.140Z", + "time": 35, + "request": { + "method": "GET", + "url": "https://cdn.example.com/pixel.gif", + "httpVersion": "h2", + "cookies": [], + "headers": [ + { + "name": "Accept", + "value": "image/*" + } + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1 + }, + "response": { + "status": 200, + "statusText": "", + "httpVersion": "h2", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "image/gif" + }, + { + "name": "Content-Length", + "value": "43" + } + ], + "content": { + "size": 43, + "mimeType": "image/gif", + "text": "R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==", + "encoding": "base64" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 43 + }, + "cache": {}, + "timings": { + "blocked": -1, + "dns": -1, + "connect": -1, + "ssl": -1, + "send": 0, + "wait": 35, + "receive": 0 + }, + "_resourceType": "image", + "_lustro": { + "id": "tx_e6b0d413", + "isMocked": false, + "categories": [ + "media" + ], + "requestBodyTruncated": false, + "responseBodyTruncated": false, + "responseComplete": true, + "error": null + } + }, + { + "startedDateTime": "2026-09-28T14:22:14.022Z", + "time": 1210, + "request": { + "method": "GET", + "url": "https://api.example.com/v1/catalog?page=2&tag=new%20in", + "httpVersion": "h2", + "cookies": [], + "headers": [ + { + "name": "Accept", + "value": "application/json" + } + ], + "queryString": [ + { + "name": "page", + "value": "2" + }, + { + "name": "tag", + "value": "new in" + } + ], + "headersSize": -1, + "bodySize": -1 + }, + "response": { + "status": 200, + "statusText": "", + "httpVersion": "h2", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "application/json; charset=utf-8" + } + ], + "content": { + "size": 524288, + "mimeType": "application/json; charset=utf-8", + "text": "[{\"sku\":\"A-1\",\"name\":\"Desk lamp\"},{\"sku\":\"A-2\",\"na" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 524288 + }, + "cache": {}, + "timings": { + "blocked": -1, + "dns": -1, + "connect": -1, + "ssl": -1, + "send": 0, + "wait": 1210, + "receive": 0 + }, + "_resourceType": "fetch", + "_lustro": { + "id": "tx_5e8a2f61", + "isMocked": false, + "categories": [ + "api" + ], + "requestBodyTruncated": false, + "responseBodyTruncated": true, + "responseComplete": true, + "error": null + } + }, + { + "startedDateTime": "2026-09-28T14:22:15.917Z", + "time": 8, + "request": { + "method": "GET", + "url": "https://telemetry.example.com/v1/events", + "httpVersion": "", + "cookies": [], + "headers": [ + { + "name": "Accept", + "value": "application/json" + } + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1 + }, + "response": { + "status": 0, + "statusText": "", + "httpVersion": "", + "cookies": [], + "headers": [], + "content": { + "size": 0, + "mimeType": "x-unknown" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": -1, + "_error": "java.net.UnknownHostException: Unable to resolve host \"telemetry.example.com\"" + }, + "cache": {}, + "timings": { + "blocked": -1, + "dns": -1, + "connect": -1, + "ssl": -1, + "send": 0, + "wait": 8, + "receive": 0 + }, + "_resourceType": "fetch", + "_lustro": { + "id": "tx_d03a9b44", + "isMocked": false, + "categories": [], + "requestBodyTruncated": false, + "responseBodyTruncated": false, + "responseComplete": true, + "error": "java.net.UnknownHostException: Unable to resolve host \"telemetry.example.com\"" + } + } + ] + } +}