Export captured traffic as HAR and Markdown

.github/scripts/cli-e2e.sh

@@ -153,4 +153,16 @@ esac
 lustro_to net-get-short.json --json net get "$short"
 check transaction "$RULES" < "$OUT/net-get-short.json"
 
+# net export takes the short id too, and writes the same entry that a full export has.
+step "lustro net export --har: the mocked request, then every transaction"
+lustro net export --har "$OUT/export-one.har" --ids "$short"
+check har "$RULES" < "$OUT/export-one.har"
+entries="$("$OUT/venv/bin/python" -c 'import json,sys; print(len(json.load(sys.stdin)["log"]["entries"]))' < "$OUT/export-one.har")"
+if [ "$entries" != "1" ]; then
+  echo "error: --ids $short exported $entries entries" >&2
+  exit 1
+fi
+lustro net export --har "$OUT/export-all.har"
+check har "$RULES" < "$OUT/export-all.har"
+
 step "The CLI end to end passed"

.github/scripts/cli_e2e_check.py

@@ -129,6 +129,19 @@ def check_transaction(text: str, rules: List[dict]) -> None:
     expect(tx["responseBody"] == rule["responseBody"], "body {!r}, the rule sets {!r}".format(tx["responseBody"], rule["responseBody"]))
 
 
+def check_har(text: str, rules: List[dict]) -> None:
+    """``net export --har`` writes a HAR document; the request the first rule served is in it."""
+    har = json.loads(text)
+    component("HarDocument").validate(har)
+    rule = rules[0]
+    entries = [entry for entry in har["log"]["entries"] if rule["urlPattern"] in entry["request"]["url"]]
+    expect(bool(entries), "no entry matches {}".format(rule["urlPattern"]))
+    for entry in entries:
+        expect(entry["_lustro"]["isMocked"] is True, "the rule did not serve {}".format(entry["_lustro"]["id"]))
+        expect(entry["response"]["status"] == rule["statusCode"], "status {}".format(entry["response"]["status"]))
+        expect(entry["response"]["content"].get("text") == rule["responseBody"], "body {!r}".format(entry["response"]["content"].get("text")))
+
+
 CHECKS: Dict[str, Callable[[str, List[dict]], None]] = {
     "open": check_open,
     "meta": check_meta,
@@ -139,6 +152,7 @@ def check_transaction(text: str, rules: List[dict]) -> None:
     "state": check_state,
     "find": find,
     "transaction": check_transaction,
+    "har": check_har,
 }
 
 

CHANGELOG.md

@@ -197,6 +197,28 @@ see [DECISIONS.md](DECISIONS.md).
   `docs/STYLEGUIDE.md`. The JavaScript tests cover the escaping of each viewer.
   The sample has a request for each viewer, a PNG and a BMP that it sends as
   request bodies, and a request that sends a 250 KB JSON body.
+- **Export captured traffic as HAR and Markdown.** A transaction left Lustro
+  only as JSON from the API, as a cURL command, or as plain text, one at a
+  time. `GET network/transactions/_/export?format=har&ids=...` returns the
+  transactions as a HAR 1.2 document, oldest first, and every transaction when
+  `ids` is left out (wire protocol 1.2). It is built from the store, so it has
+  the redacted values. Each entry spends its `durationMs` in `timings.wait`,
+  `_resourceType` lets Chrome DevTools list it under Fetch/XHR or Img, a body
+  kept as bytes is base64, and `_lustro` carries the transaction id,
+  `isMocked`, `categories`, the truncation flags, `responseComplete`, and
+  `error`. `lustro net export --har FILE [--ids ID ...]` saves it, and sends
+  many ids in batches, because a request line must stay under 8 KB. In the
+  Network tab, **Select** adds a checkbox to each row, a checkbox in the header
+  that selects every row the filters show, and a bar with the count, **Export
+  HAR**, which saves a file, and **Copy Markdown**. A filter change keeps only
+  the selected rows it still shows. The detail's new **Markdown** button copies
+  one transaction: the method and URL as a heading, the headers in `http`
+  blocks, and each body in a block tagged with a language for its content type,
+  JSON indented without changing a value. Both formats mark a body that the
+  capture cut short. The OpenAPI document describes the route and the HAR
+  shape, the golden fixture `export-har.json` shows it, a unit test checks that
+  the export writes exactly that fixture, and the CLI end-to-end run exports
+  the sample's mocked request.
 
 ### Fixed
 

README.md

@@ -248,6 +248,16 @@ The panel reports only the status and outcome, so the sender reads at most
 download or an endless stream cannot exhaust the app's heap. A send still running when the
 per-request timeout answers `504` has its call cancelled right after.
 
+## Export
+
+The Network tab's **Select** button adds a checkbox to each row. **Export HAR** saves the selected
+requests as a HAR 1.2 file, which browser devtools and most HTTP tools import, and
+**Copy Markdown** copies them as one Markdown document for a bug report, a pull request, or a
+chat. The detail's **Markdown** button copies one request. A filter change keeps only the selected
+requests that it still shows, so an export has the rows on screen. Both formats have the redacted
+values that the tab shows, and say when the capture cut a body short. Agents and scripts get the
+HAR from `GET /api/v1/network/transactions/_/export` or `lustro net export --har FILE`.
+
 ## Mock rules
 
 The Network tab's **Mock Rules** panel short-circuits matching requests with a synthetic

docs/AGENTS.md

@@ -38,10 +38,11 @@ lustro open --print-only          # forward the port; exits 1 when adb forward f
 lustro net list                   # the newest 50 transactions, newest first
 lustro net get <id> --no-body     # one transaction, without its bodies
 lustro net wait --url /v1/orders -- adb shell input tap 540 1200
+lustro net export --har out.har   # every transaction as a HAR file
 ```
 
 Each row of `net list` starts with a short transaction id: the first 8 characters of the id, or more
-when two listed ids start with the same 8. `net get` and `net body` take the short id, the full id,
+when two listed ids start with the same 8. `net get`, `net body`, and `net export --ids` take the short id, the full id,
 or any start of an id that only one transaction has. When more than one id starts with it, they
 print the matches and exit 2; when no id does, they exit 1. JSON output has the full id, which the
 wire routes in the curl examples need. The filters `--url`, `--method`, `--status` (a code such as
@@ -179,6 +180,7 @@ below summarizes it. All routes are token-authenticated and use the shared error
 | Poll transactions | `GET transactions?cursor=&search=` | Cursor envelope. First poll (no/invalid cursor) → `reset` with the full list; cursor unchanged → `unchanged` (items omitted); after a change → `delta`. `search` filters case-insensitively over URL, method, and bodies. Carries a top-level `state` `{ paused, overwriteMode, throttleDelayMs, captureFilter }`. |
 | Transaction detail | `GET transactions/{id}` | Full object (headers + bodies, with truncation flags). Enveloped `404` if missing. |
 | Transaction body | `GET transactions/{id}/body/{request\|response}` | One body as it is stored: the bytes of an image, or the redacted text as UTF-8. Not JSON. Enveloped `404` when no body was kept. See below. |
+| Export as HAR | `GET transactions/_/export?format=har&ids=` | A HAR 1.2 document of the transactions in `ids` (comma-separated), or of every one. See below. |
 | Clear | `POST clear` | Clears the captured list and starts the capture filter's counts again; mock rules and settings are preserved. |
 | List rules | `GET rules` | `{ items: [MockRule...] }`. |
 | Add / upsert rule | `POST rules` | Body `MockRuleInput` (`urlPattern` required). Supplying a stable `id` makes the write **idempotent** (upsert by id); omitting it generates one. Returns `{ status: "ok", id }`. |
@@ -242,6 +244,18 @@ body, or a binary type other than an image. The redactor never sees an image, so
 unredacted. `lustro net body <id> [request|response] -o FILE` wraps the route. These fields and
 the route are part of protocol 1.2.
 
+**Export as HAR.** `GET transactions/_/export?format=har&ids=<id>,<id>` returns the transactions as a
+HAR 1.2 document, oldest first, which browser devtools, proxies, and HAR viewers import. It is
+built from the store, so it has the same redacted values as the detail. Without `ids` it has every
+transaction; an id the app no longer has is left out. Each entry's `_lustro` has the transaction
+`id`, `isMocked`, `categories`, `requestBodyTruncated`, `responseBodyTruncated`,
+`responseComplete`, and `error`. The capture has no phase timings, so each entry spends its whole
+`durationMs` in `timings.wait`, and `_resourceType` (`fetch`, or `image` for an image) tells
+Chrome DevTools how to file it. A body kept as bytes is base64: `content.encoding` says so for a
+response, and `postData._encoding` for a request. The request line and its headers must stay
+under 8 KB, so send many ids in batches; `lustro net export --har FILE [--ids ID ...]` does that,
+and `--har -` writes the document to stdout. This route is part of protocol 1.2.
+
 **Compressed bodies.** A body sent or received with `Content-Encoding: gzip`, `x-gzip`, or
 `deflate` is stored inflated: `requestBody` and `responseBody` hold the text, and
 `...BodyTruncated` says whether the inflated body passed the capture cap. The headers still name

llms.txt

@@ -27,7 +27,7 @@ Facts to get right when you add Lustro to an app:
 ## Tools
 
 - [lustro CLI](https://raw.githubusercontent.com/Twinsen81/Lustro/main/lustro-cli/README.md): a Python command-line client for the HTTP API. `lustro open` forwards the port and opens the console with the token
-- Agents: use the `lustro` CLI rather than curl, and keep its output small with `lustro net list --errors --last 20`, `lustro net get <id> --no-body`, and `lustro net body <id> | head -c 4000`. `lustro net wait --url <text> -- <command>` runs the command and waits for the request it causes
+- Agents: use the `lustro` CLI rather than curl, and keep its output small with `lustro net list --errors --last 20`, `lustro net get <id> --no-body`, and `lustro net body <id> | head -c 4000`. `lustro net wait --url <text> -- <command>` runs the command and waits for the request it causes, and `lustro net export --har out.har [--ids <id> ...]` saves transactions as a HAR file
 
 ## Optional
 

lustro-cli/README.md

@@ -65,6 +65,7 @@ discovery), `--json` (JSON output). They go before or after the command:
 | `lustro net wait [-- COMMAND]` | cursor loop; runs `COMMAND`, prints the first matching request that finishes, and exits |
 | `lustro net get <id>` | `GET network/transactions/<id>`, with each body cut at 2 KB |
 | `lustro net body <id> [request\|response] [-o FILE]` | `GET network/transactions/<id>/body/<direction>`: saves a body as it is stored, the response by default; stdout without `-o` |
+| `lustro net export --har FILE [--ids ID ...]` | `GET network/transactions/_/export`: saves the transactions, or only those in `--ids`, as a HAR file; stdout for `--har -` |
 | `lustro net clear` | `POST network/clear` |
 | `lustro net pause` | `POST network/pause` |
 | `lustro net overwrite` | `POST network/overwrite-mode` |
@@ -111,8 +112,8 @@ The output is small by default, because an agent reads all of it.
   `ERR` for a failed request), the duration, and the URL. The short id is the
   first 8 characters of the id, or more characters when two listed ids start
   with the same 8. JSON output has the full id.
-- **Ids.** `net get` and `net body` take a full id, or the start of one, such as
-  the short id of a row. They find the id that starts with it in the transaction
+- **Ids.** `net get`, `net body`, and `net export --ids` take a full id, or the
+  start of one, such as the short id of a row. They find the id that starts with it in the transaction
   list, which is one more request. When more than one id starts with it, they
   print the matches and exit 2. When no id starts with it, they exit 1. The wire
   routes take only the full id.

lustro-cli/src/lustro_cli/cli.py

@@ -37,6 +37,7 @@
 
 NETWORK = "/api/v1/network"
 TRANSACTIONS = NETWORK + "/transactions"
+EXPORT = TRANSACTIONS + "/_/export"
 
 # The flags that every command takes, before or after its name, and their values
 # when they are not given. See _global_options for why main() sets these.
@@ -55,6 +56,9 @@
 # How many of the matches an ambiguous id prints.
 AMBIGUOUS_ROWS = 10
 ID_HELP = "a transaction id, or its start, such as the short id of a row"
+# How many ids one export request names. The server takes a request line and
+# headers of up to 8 KB, and a full id is 36 characters.
+EXPORT_BATCH = 50
 
 # The Lustro runtime makes each transaction id with UUID.randomUUID(). An id of
 # this shape goes to the server as it is, without the list request that a
@@ -233,11 +237,27 @@ def _resolve_id(client: LustroClient, given: str) -> str:
     """The full id of the transaction that ``given`` names: the id itself, or
     the start of only one id, such as the short id of a row. A start costs one
     list request, because the wire routes take only a full id."""
-    if _FULL_ID.fullmatch(given):
-        return given
     if not given:
         raise UsageError("expected a transaction id, or the start of one")
-    items = _list_items(client.get(TRANSACTIONS))
+    return _resolve_ids(client, [given])[0]
+
+
+def _resolve_ids(client: LustroClient, given: List[str]) -> List[str]:
+    """The full id of each transaction in ``given``, as _resolve_id finds it.
+    One list request covers every start."""
+    items: Optional[List[dict]] = None
+    resolved = []
+    for start in given:
+        if _FULL_ID.fullmatch(start):
+            resolved.append(start)
+            continue
+        if items is None:
+            items = _list_items(client.get(TRANSACTIONS))
+        resolved.append(_match_id(items, start))
+    return resolved
+
+
+def _match_id(items: List[dict], given: str) -> str:
     matches = [tx for tx in items if isinstance(tx.get("id"), str) and tx["id"].startswith(given)]
     # An id of another shape can also be the start of a longer one.
     if any(tx["id"] == given for tx in matches):
@@ -274,6 +294,13 @@ def _status_range(value: str) -> Tuple[int, int]:
     raise argparse.ArgumentTypeError("expected a status code such as 404, or a class such as 4xx")
 
 
+def _id_list(value: str) -> List[str]:
+    ids = [part.strip() for part in value.split(",") if part.strip()]
+    if not ids:
+        raise argparse.ArgumentTypeError("expected transaction ids, or their starts, separated by commas")
+    return ids
+
+
 def _field_names(value: str) -> List[str]:
     names = [name.strip() for name in value.split(",") if name.strip()]
     if not names:
@@ -727,6 +754,71 @@ def cmd_net_body(args: argparse.Namespace) -> int:
     return 0
 
 
+def _export_entries(har: Any) -> List[Any]:
+    entries = har.get("log", {}).get("entries") if isinstance(har, dict) else None
+    if not isinstance(entries, list):
+        raise LustroError("invalid_response", "the export is not a HAR document: it has no log.entries")
+    return entries
+
+
+def _export_har(client: LustroClient, ids: Optional[List[str]]) -> Any:
+    """The HAR document of the transactions that ``ids`` names, or of every one.
+    Many ids go in batches, and the entries of each batch join the first one's."""
+    if ids is None:
+        har = client.get(EXPORT, params={"format": "har"})
+        _export_entries(har)
+        return har
+    har = None
+    for start in range(0, len(ids), EXPORT_BATCH):
+        batch = client.get(EXPORT, params={"format": "har", "ids": ",".join(ids[start : start + EXPORT_BATCH])})
+        entries = _export_entries(batch)
+        if har is None:
+            har = batch
+        else:
+            _export_entries(har).extend(entries)
+    # Each batch is oldest first, and the times are ISO 8601 in UTC, so they sort as text.
+    _export_entries(har).sort(key=lambda entry: str(entry.get("startedDateTime", "")))
+    return har
+
+
+def cmd_net_export(args: argparse.Namespace) -> int:
+    """Save captured transactions as a HAR file, or write it to stdout."""
+    client = _build_client(args)
+    ids = None
+    if args.ids is not None:
+        # In the order given, each id once.
+        ids = list(dict.fromkeys(_resolve_ids(client, [given for group in args.ids for given in group])))
+    har = _export_har(client, ids)
+    entries = _export_entries(har)
+    # Indented, as browser devtools write a HAR file, and as the console saves one.
+    data = (json.dumps(har, indent=2, ensure_ascii=False) + "\n").encode("utf-8")
+    if args.har == "-":
+        sys.stdout.buffer.write(data)
+        sys.stdout.flush()
+    else:
+        try:
+            with open(args.har, "wb") as fh:
+                fh.write(data)
+        except OSError as exc:
+            print("could not write HAR file {}: {}".format(args.har, exc), file=sys.stderr)
+            return 2
+    exported = {entry.get("_lustro", {}).get("id") for entry in entries if isinstance(entry, dict)}
+    missing = [tx_id for tx_id in ids or [] if tx_id not in exported]
+    if missing:
+        print(
+            "warning: the app no longer has {} of the transactions, so the file leaves them out: {}".format(
+                len(missing), ", ".join(missing)
+            ),
+            file=sys.stderr,
+        )
+    if args.har != "-":
+        if args.json:
+            _emit({"path": args.har, "entries": len(entries), "bytes": len(data), "missing": missing}, raw_json=True)
+        else:
+            print("saved {} transactions ({} bytes) to {}".format(len(entries), len(data), args.har))
+    return 0
+
+
 def cmd_net_clear(args: argparse.Namespace) -> int:
     client = _build_client(args)
     _emit(client.post(NETWORK + "/clear"), raw_json=args.json)
@@ -1016,6 +1108,27 @@ def build_parser() -> argparse.ArgumentParser:
     n_body.add_argument("-o", "--output", default=None, metavar="FILE", help="write the body to FILE (default: stdout)")
     n_body.set_defaults(func=cmd_net_body)
 
+    n_export = net_sub.add_parser(
+        "export",
+        parents=[common],
+        help="GET transactions/_/export: save transactions as a HAR file",
+        description="Save the captured transactions as a HAR 1.2 file, which browser devtools and HTTP tools "
+        "import. Headers and bodies are redacted as in the app. Each entry has the transaction id, and what HAR "
+        "has no field for, in _lustro.",
+    )
+    n_export.add_argument(
+        "--har", required=True, metavar="FILE", help="write the HAR document to FILE, or to stdout for -"
+    )
+    n_export.add_argument(
+        "--ids",
+        nargs="+",
+        type=_id_list,
+        default=None,
+        metavar="ID",
+        help="only these transactions, separated by spaces or commas; each is {} (default: every one)".format(ID_HELP),
+    )
+    n_export.set_defaults(func=cmd_net_export)
+
     n_clear = net_sub.add_parser("clear", parents=[common], help="POST clear")
     n_clear.set_defaults(func=cmd_net_clear)