From 675f2488c01b90c8338566bf493d7bead00b0f65 Mon Sep 17 00:00:00 2001 From: Tim <51825276+tjdonley@users.noreply.github.com> Date: Sun, 26 Apr 2026 18:32:16 -0400 Subject: [PATCH] Harden privacy boundaries for trace identifiers --- README.md | 11 +-- agentprof/cli.py | 44 ++++++++++- agentprof/config.py | 15 ++++ agentprof/ingest/langfuse_export.py | 40 +++++++++- agentprof/normalize/langfuse.py | 4 +- agentprof/normalize/runner.py | 12 ++- agentprof/normalize/schema.py | 2 + agentprof/privacy/hashing.py | 18 ++++- tests/test_cli.py | 73 +++++++++++++++++- tests/test_langfuse_export_import.py | 106 ++++++++++++++++++++++++++- tests/test_multi_agent_waste.py | 2 +- tests/test_normalize.py | 98 +++++++++++++++++++++++-- tests/test_privacy_hashing.py | 12 ++- tests/test_retry_loop.py | 2 +- 14 files changed, 410 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index bdd1148..7255697 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,9 @@ Run the built-in Langfuse fixture through the current end-to-end workflow: ```bash uv sync uv run agentprof init +export AGENTPROF_HASH_SALT='dev-salt-value-at-least-16-bytes' uv run agentprof doctor -AGENTPROF_HASH_SALT=dev-salt uv run agentprof import langfuse-export \ +uv run agentprof import langfuse-export \ --observations tests/fixtures/langfuse_observations.json uv run agentprof normalize uv run agentprof analyze retry-loops @@ -85,7 +86,7 @@ The workflow creates local files only. After `agentprof report generate`, the de uv run agentprof init ``` -This creates `agentprof.yml`, local workspace directories under `.agentprof/`, and the DuckDB store at `.agentprof/data/agentprof.duckdb`. +This creates `agentprof.yml`, local workspace directories under `.agentprof/`, `.agentprof/.gitignore`, and the DuckDB store at `.agentprof/data/agentprof.duckdb`. 2. Configure privacy. @@ -95,7 +96,7 @@ By default, `agentprof.yml` sets `privacy.hash_inputs: true`, so imports that co export AGENTPROF_HASH_SALT='replace-with-a-stable-secret-for-this-project' ``` -Use a stable per-project salt if you want hashes to remain comparable across repeated imports. Do not commit the salt. +Use a stable per-project salt of at least 16 bytes if you want hashes to remain comparable across repeated imports. Do not commit the salt. 3. Import a Langfuse observations export. @@ -181,7 +182,7 @@ The costed multi-agent fixture shows the current baseline-estimate story without ```bash uv run agentprof store reset --yes -AGENTPROF_HASH_SALT=dev-salt uv run agentprof import langfuse-export \ +AGENTPROF_HASH_SALT=dev-salt-value-at-least-16-bytes uv run agentprof import langfuse-export \ --observations tests/fixtures/langfuse_multi_agent_observations.json uv run agentprof normalize uv run agentprof analyze multi-agent-waste --baseline-ratio 0.50 @@ -370,7 +371,7 @@ Useful local smoke workflow: ```bash uv run agentprof store reset --yes -AGENTPROF_HASH_SALT=dev-salt uv run agentprof import langfuse-export \ +AGENTPROF_HASH_SALT=dev-salt-value-at-least-16-bytes uv run agentprof import langfuse-export \ --observations tests/fixtures/langfuse_observations.json uv run agentprof normalize uv run agentprof analyze retry-loops diff --git a/agentprof/cli.py b/agentprof/cli.py index 6ddb885..eb0eb84 100644 --- a/agentprof/cli.py +++ b/agentprof/cli.py @@ -23,6 +23,7 @@ AgentProfConfig, ensure_workspace_dirs, load_config, + write_workspace_gitignore, write_default_config, ) from agentprof.cost.runner import build_cost_ledger @@ -32,7 +33,7 @@ import_langfuse_export, ) from agentprof.normalize.runner import normalize_store -from agentprof.privacy.hashing import MissingSaltError +from agentprof.privacy.hashing import MissingSaltError, salt_from_env from agentprof.report.runner import DEFAULT_REPORT_DIR, generate_report, validate_report_id from agentprof.store.duckdb_store import DuckDBStore @@ -48,6 +49,7 @@ class MultiAgentBaselineMode(StrEnum): console = Console() +error_console = Console(stderr=True) REPORT_SHOW_MAX_BYTES = 1_048_576 REPORT_SHOW_CHUNK_SIZE = 64 * 1024 app = typer.Typer( @@ -75,7 +77,7 @@ def _version_callback(value: bool) -> None: def _load_config_or_exit() -> AgentProfConfig: try: - return load_config() + config = load_config() except FileNotFoundError as exc: console.print("[red]agentprof.yml was not found.[/red]") console.print("Run `agentprof init` to create the local workspace.") @@ -84,6 +86,8 @@ def _load_config_or_exit() -> AgentProfConfig: console.print("[red]agentprof.yml is invalid.[/red]") console.print(str(exc)) raise typer.Exit(code=2) from exc + _warn_if_raw_io_enabled(config) + return config @app.callback() @@ -118,6 +122,8 @@ def init( for path in ensure_workspace_dirs(): created.append(str(path)) + if write_workspace_gitignore(force=force): + created.append(str(APP_DIR / ".gitignore")) config = _load_config_or_exit() store = DuckDBStore(config.store.path) @@ -161,6 +167,13 @@ def doctor() -> None: console.print(f"Store/config check failed: {exc}") raise typer.Exit(code=2) from exc + privacy_errors = _privacy_doctor_errors(config) + if privacy_errors: + console.print("[red]AgentProf workspace has privacy configuration issues.[/red]") + for message in privacy_errors: + console.print(f" {message}") + raise typer.Exit(code=2) + console.print("[green]AgentProf workspace looks ready.[/green]") @@ -568,7 +581,10 @@ def import_langfuse_export_command( file_format=file_format, ) except MissingSaltError as exc: - console.print("[red]Cannot hash Langfuse I/O without a configured salt.[/red]") + console.print( + "[red]Cannot hash Langfuse identifiers or I/O without a configured salt.[/red]" + ) + console.print(str(exc)) console.print( f"Set `{config.privacy.hmac_salt_env}` or disable `privacy.hash_inputs`." ) @@ -608,6 +624,28 @@ def _parse_decimal_option(value: str, name: str) -> Decimal: raise ValueError(f"{name} must be a decimal value") from exc +def _warn_if_raw_io_enabled(config: AgentProfConfig) -> None: + if config.privacy.store_raw_io: + error_console.print( + "[yellow]WARNING: privacy.store_raw_io is true; raw Langfuse inputs " + "and outputs may be written to disk.[/yellow]" + ) + + +def _privacy_doctor_errors(config: AgentProfConfig) -> list[str]: + errors: list[str] = [] + if config.privacy.store_raw_io: + errors.append( + "privacy.store_raw_io is true; raw inputs and outputs may be persisted." + ) + if config.privacy.hash_inputs: + try: + salt_from_env(config.privacy.hmac_salt_env) + except MissingSaltError as exc: + errors.append(str(exc)) + return errors + + def _validate_report_output_dir(output_dir: Path) -> None: reports_root = _resolve_path(DEFAULT_REPORT_DIR) output_path = _resolve_path(output_dir) diff --git a/agentprof/config.py b/agentprof/config.py index 1c977af..da54493 100644 --- a/agentprof/config.py +++ b/agentprof/config.py @@ -11,6 +11,8 @@ CONFIG_FILE = Path("agentprof.yml") APP_SUBDIRS = ("data", "baselines", "reports", "cache") DEFAULT_STORE_PATH = APP_DIR / "data" / "agentprof.duckdb" +WORKSPACE_GITIGNORE = APP_DIR / ".gitignore" +WORKSPACE_GITIGNORE_CONTENT = "*\n!.gitignore\n" DEFAULT_CONFIG = """project: name: tracer @@ -122,6 +124,19 @@ def write_default_config(path: Path = CONFIG_FILE, *, force: bool = False) -> bo return True +def write_workspace_gitignore( + path: Path = WORKSPACE_GITIGNORE, + *, + force: bool = False, +) -> bool: + if path.exists() and not force: + return False + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(WORKSPACE_GITIGNORE_CONTENT, encoding="utf-8") + return True + + def load_config(path: Path = CONFIG_FILE) -> AgentProfConfig: if not path.exists(): raise FileNotFoundError(path) diff --git a/agentprof/ingest/langfuse_export.py b/agentprof/ingest/langfuse_export.py index 44500e5..1e46677 100644 --- a/agentprof/ingest/langfuse_export.py +++ b/agentprof/ingest/langfuse_export.py @@ -8,7 +8,13 @@ from typing import Any from agentprof.config import AgentProfConfig -from agentprof.privacy.hashing import content_hash, retry_fingerprint, salt_from_env +from agentprof.privacy.hashing import ( + content_hash, + retry_fingerprint, + salt_from_env, + session_id_hash, + user_id_hash, +) from agentprof.privacy.redactor import evidence_preview, redact_value, rules_from_config from agentprof.store.duckdb_store import DuckDBStore, RawSpanRecord @@ -16,6 +22,7 @@ LANGFUSE_SOURCE = "langfuse" INPUT_FIELD = "input" OUTPUT_FIELD = "output" +IDENTITY_FIELDS = ("userId", "user_id", "sessionId", "session_id") class LangfuseExportFormat(StrEnum): @@ -118,10 +125,20 @@ def sanitize_observation_payload( rules = rules_from_config(privacy.redact) raw_input = payload.get(INPUT_FIELD) raw_output = payload.get(OUTPUT_FIELD) - salt = _hash_salt(config, raw_input=raw_input, raw_output=raw_output) + raw_user_id = _string_field(payload, "userId", "user_id") + raw_session_id = _string_field(payload, "sessionId", "session_id") + salt = _hash_salt( + config, + raw_input=raw_input, + raw_output=raw_output, + raw_user_id=raw_user_id, + raw_session_id=raw_session_id, + ) payload.pop(INPUT_FIELD, None) payload.pop(OUTPUT_FIELD, None) + for key in IDENTITY_FIELDS: + payload.pop(key, None) payload = redact_value(payload, rules) if privacy.store_raw_io: if raw_input is not None: @@ -132,6 +149,8 @@ def sanitize_observation_payload( payload["_agentprof_privacy"] = _privacy_metadata( raw_input=raw_input, raw_output=raw_output, + raw_user_id=raw_user_id, + raw_session_id=raw_session_id, config=config, salt=salt, ) @@ -142,6 +161,8 @@ def _privacy_metadata( *, raw_input: Any, raw_output: Any, + raw_user_id: str | None, + raw_session_id: str | None, config: AgentProfConfig, salt: bytes | None, ) -> dict[str, Any]: @@ -152,6 +173,8 @@ def _privacy_metadata( "redacted_io_stored": privacy.store_redacted_io, "input_hash": None, "output_hash": None, + "user_hash": None, + "session_hash": None, "input_retry_fingerprint": None, "output_retry_fingerprint": None, "input_preview": None, @@ -165,6 +188,10 @@ def _privacy_metadata( if raw_output is not None: metadata["output_hash"] = content_hash(raw_output, salt) metadata["output_retry_fingerprint"] = retry_fingerprint(raw_output, salt) + if raw_user_id is not None: + metadata["user_hash"] = user_id_hash(raw_user_id, salt) + if raw_session_id is not None: + metadata["session_hash"] = session_id_hash(raw_session_id, salt) if privacy.store_redacted_io: if raw_input is not None: @@ -188,8 +215,15 @@ def _hash_salt( *, raw_input: Any, raw_output: Any, + raw_user_id: str | None, + raw_session_id: str | None, ) -> bytes | None: - if not config.privacy.hash_inputs or (raw_input is None and raw_output is None): + if not config.privacy.hash_inputs or ( + raw_input is None + and raw_output is None + and raw_user_id is None + and raw_session_id is None + ): return None return salt_from_env(config.privacy.hmac_salt_env) diff --git a/agentprof/normalize/langfuse.py b/agentprof/normalize/langfuse.py index 01c2641..847d7cf 100644 --- a/agentprof/normalize/langfuse.py +++ b/agentprof/normalize/langfuse.py @@ -49,7 +49,7 @@ "metadata", "_agentprof_privacy", } -TRACE_ATTRIBUTE_FIELDS = ("sessionId", "userId", "environment", "version") +TRACE_ATTRIBUTE_FIELDS = ("environment", "version") LANGFUSE_TYPE_TO_SPAN_TYPE: dict[str, SpanType] = { "AGENT": "agent", "EMBEDDING": "embedding", @@ -114,6 +114,8 @@ def map_langfuse_raw_span(row: RawSpanRow) -> NormalizedSpan: output_retry_fingerprint=_string_field(privacy, "output_retry_fingerprint"), input_preview=_string_field(privacy, "input_preview"), output_preview=_string_field(privacy, "output_preview"), + session_id=_string_field(privacy, "session_hash"), + user_hash=_string_field(privacy, "user_hash"), input_tokens=input_tokens, output_tokens=output_tokens, total_tokens=total_tokens, diff --git a/agentprof/normalize/runner.py b/agentprof/normalize/runner.py index dcd4742..d7aeb0b 100644 --- a/agentprof/normalize/runner.py +++ b/agentprof/normalize/runner.py @@ -54,8 +54,8 @@ def build_normalized_traces(spans: list[NormalizedSpan]) -> list[NormalizedTrace source=trace_spans[0].source, root_span_id=root.span_id if root else None, root_name=root.name if root else None, - session_id=_first_attribute(trace_spans, "sessionId"), - user_hash=_first_attribute(trace_spans, "userId"), + session_id=_first_span_value(trace_spans, "session_id"), + user_hash=_first_span_value(trace_spans, "user_hash"), environment=_first_attribute(trace_spans, "environment"), version=_first_attribute(trace_spans, "version"), start_time=start_time, @@ -170,6 +170,14 @@ def _first_attribute(spans: list[NormalizedSpan], key: str) -> str | None: return None +def _first_span_value(spans: list[NormalizedSpan], name: str) -> str | None: + for span in spans: + value = getattr(span, name) + if value not in (None, ""): + return str(value) + return None + + def _trace_duration_ms(start_time: datetime | None, end_time: datetime | None) -> float | None: if not start_time or not end_time: return None diff --git a/agentprof/normalize/schema.py b/agentprof/normalize/schema.py index 039169d..755474d 100644 --- a/agentprof/normalize/schema.py +++ b/agentprof/normalize/schema.py @@ -74,6 +74,8 @@ class NormalizedSpan(BaseModel): output_retry_fingerprint: str | None = None input_preview: str | None = None output_preview: str | None = None + session_id: str | None = None + user_hash: str | None = None input_tokens: int | None = None output_tokens: int | None = None total_tokens: int | None = None diff --git a/agentprof/privacy/hashing.py b/agentprof/privacy/hashing.py index e2d0396..1b8fadf 100644 --- a/agentprof/privacy/hashing.py +++ b/agentprof/privacy/hashing.py @@ -11,6 +11,9 @@ from typing import Any +MIN_HMAC_SALT_BYTES = 16 + + DEFAULT_VOLATILE_KEYS = frozenset( { "created_at", @@ -56,7 +59,12 @@ def salt_from_env(env_name: str) -> bytes: value = os.getenv(env_name) if not value: raise MissingSaltError(f"{env_name} is not set") - return value.encode("utf-8") + salt = value.encode("utf-8") + if len(salt) < MIN_HMAC_SALT_BYTES: + raise MissingSaltError( + f"{env_name} must be at least {MIN_HMAC_SALT_BYTES} bytes" + ) + return salt def hmac_sha256_hexdigest(content: str | bytes, salt: str | bytes) -> str: @@ -77,6 +85,14 @@ def retry_fingerprint(content: Any, salt: str | bytes) -> str: ) +def user_id_hash(user_id: str, salt: str | bytes) -> str: + return hmac_sha256_hexdigest(f"agentprof:user_id:{user_id}", salt) + + +def session_id_hash(session_id: str, salt: str | bytes) -> str: + return hmac_sha256_hexdigest(f"agentprof:session_id:{session_id}", salt) + + def normalize_volatile_text(value: str) -> str: return _normalize_text(value, strip_volatile=True) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9b489a6..36fbd63 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -26,6 +26,10 @@ def test_init_creates_workspace() -> None: assert "AgentProf initialized" in result.output assert (Path("agentprof.yml")).is_file() + assert ( + Path(".agentprof/.gitignore").read_text(encoding="utf-8") + == "*\n!.gitignore\n" + ) for subdir in APP_SUBDIRS: assert (Path(".agentprof") / subdir).is_dir() assert DEFAULT_STORE_PATH.is_file() @@ -39,7 +43,8 @@ def test_doctor_requires_init() -> None: assert "workspace is incomplete" in result.output -def test_doctor_after_init() -> None: +def test_doctor_after_init(monkeypatch) -> None: + monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt-value-123") with runner.isolated_filesystem(): init_result = runner.invoke(app, ["init"]) doctor_result = runner.invoke(app, ["doctor"]) @@ -49,6 +54,72 @@ def test_doctor_after_init() -> None: assert "workspace looks ready" in doctor_result.output +def test_doctor_allows_existing_workspace_without_gitignore(monkeypatch) -> None: + monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt-value-123") + with runner.isolated_filesystem(): + init_result = runner.invoke(app, ["init"]) + Path(".agentprof/.gitignore").unlink() + + doctor_result = runner.invoke(app, ["doctor"]) + + assert init_result.exit_code == 0 + assert doctor_result.exit_code == 0 + assert "workspace looks ready" in doctor_result.output + + +def test_doctor_fails_when_hash_salt_is_missing() -> None: + with runner.isolated_filesystem(): + init_result = runner.invoke(app, ["init"]) + doctor_result = runner.invoke(app, ["doctor"]) + + assert init_result.exit_code == 0 + assert doctor_result.exit_code == 2 + assert "privacy configuration issues" in doctor_result.output + assert "AGENTPROF_HASH_SALT is not set" in doctor_result.output + + +def test_doctor_fails_when_hash_salt_is_weak(monkeypatch) -> None: + monkeypatch.setenv("AGENTPROF_HASH_SALT", "short") + with runner.isolated_filesystem(): + init_result = runner.invoke(app, ["init"]) + doctor_result = runner.invoke(app, ["doctor"]) + + assert init_result.exit_code == 0 + assert doctor_result.exit_code == 2 + assert "at least" in doctor_result.output + + +def test_doctor_fails_when_raw_io_storage_is_enabled(monkeypatch) -> None: + monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt-value-123") + with runner.isolated_filesystem(): + init_result = runner.invoke(app, ["init"]) + Path("agentprof.yml").write_text( + "privacy:\n store_raw_io: true\n", + encoding="utf-8", + ) + + doctor_result = runner.invoke(app, ["doctor"]) + + assert init_result.exit_code == 0 + assert doctor_result.exit_code == 2 + assert "store_raw_io is true" in doctor_result.output + + +def test_cli_warns_when_raw_io_storage_is_enabled() -> None: + with runner.isolated_filesystem(): + init_result = runner.invoke(app, ["init"]) + Path("agentprof.yml").write_text( + "privacy:\n store_raw_io: true\n hash_inputs: false\n", + encoding="utf-8", + ) + + stats_result = runner.invoke(app, ["store", "stats"]) + + assert init_result.exit_code == 0 + assert stats_result.exit_code == 0 + assert "WARNING: privacy.store_raw_io is true" in stats_result.output + + def test_doctor_fails_when_store_is_missing() -> None: with runner.isolated_filesystem(): init_result = runner.invoke(app, ["init"]) diff --git a/tests/test_langfuse_export_import.py b/tests/test_langfuse_export_import.py index 0861e5a..a3fa51e 100644 --- a/tests/test_langfuse_export_import.py +++ b/tests/test_langfuse_export_import.py @@ -121,7 +121,7 @@ def test_sanitize_preserves_raw_io_only_when_raw_io_is_enabled() -> None: def test_sanitize_stores_retry_fingerprints_for_hashed_io(monkeypatch) -> None: - monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt") + monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt-value-123") config = AgentProfConfig() first = sanitize_observation_payload( @@ -147,10 +147,40 @@ def test_sanitize_stores_retry_fingerprints_for_hashed_io(monkeypatch) -> None: assert first_privacy["input_retry_fingerprint"] == second_privacy["input_retry_fingerprint"] +def test_sanitize_hashes_user_id_when_hashing_enabled(monkeypatch) -> None: + monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt-value-123") + config = AgentProfConfig() + + payload = sanitize_observation_payload( + { + "id": "obs-1", + "sessionId": "raw-session-1", + "userId": "raw-user-1", + }, + config=config, + ) + + serialized = json.dumps(payload, sort_keys=True) + privacy = payload["_agentprof_privacy"] + user_hash = privacy["user_hash"] + session_hash = privacy["session_hash"] + + assert user_hash is not None + assert user_hash != "raw-user-1" + assert len(user_hash) == 64 + assert session_hash is not None + assert session_hash != "raw-session-1" + assert len(session_hash) == 64 + assert "raw-user-1" not in serialized + assert "raw-session-1" not in serialized + assert "userId" not in payload + assert "sessionId" not in payload + + def test_import_langfuse_export_stores_sanitized_raw_spans( monkeypatch, ) -> None: - monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt") + monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt-value-123") with runner.isolated_filesystem(): init_result = runner.invoke(app, ["init"]) import_result = runner.invoke( @@ -200,6 +230,56 @@ def test_import_langfuse_export_stores_sanitized_raw_spans( assert "[SECRET]" in privacy["input_preview"] +def test_import_langfuse_export_removes_raw_identity_fields( + monkeypatch, + tmp_path: Path, +) -> None: + monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt-value-123") + observations_path = tmp_path / "observations.json" + observations_path.write_text( + json.dumps( + [ + { + "id": "obs-identity-1", + "traceId": "trace-identity-1", + "type": "SPAN", + "name": "support_agent", + "sessionId": "raw-session-1", + "userId": "raw-user-1", + } + ] + ), + encoding="utf-8", + ) + + with runner.isolated_filesystem(): + init_result = runner.invoke(app, ["init"]) + import_result = runner.invoke( + app, + [ + "import", + "langfuse-export", + "--observations", + str(observations_path), + ], + ) + + store = DuckDBStore(DEFAULT_STORE_PATH) + payload = store.fetch_raw_spans()[0].payload_json + + parsed_payload = json.loads(payload) + privacy = parsed_payload["_agentprof_privacy"] + + assert init_result.exit_code == 0 + assert import_result.exit_code == 0 + assert "raw-user-1" not in payload + assert "raw-session-1" not in payload + assert "userId" not in parsed_payload + assert "sessionId" not in parsed_payload + assert privacy["user_hash"] is not None + assert privacy["session_hash"] is not None + + def test_import_langfuse_export_requires_hash_salt_when_hashing_enabled( monkeypatch, ) -> None: @@ -218,5 +298,25 @@ def test_import_langfuse_export_requires_hash_salt_when_hashing_enabled( assert init_result.exit_code == 0 assert import_result.exit_code == 2 - assert "Cannot hash Langfuse I/O" in import_result.output + assert "Cannot hash Langfuse identifiers or I/O" in import_result.output + assert not DuckDBStore(DEFAULT_STORE_PATH).stats()["raw_spans"] + + +def test_import_langfuse_export_rejects_weak_hash_salt(monkeypatch) -> None: + monkeypatch.setenv("AGENTPROF_HASH_SALT", "short") + with runner.isolated_filesystem(): + init_result = runner.invoke(app, ["init"]) + import_result = runner.invoke( + app, + [ + "import", + "langfuse-export", + "--observations", + str(FIXTURES / "langfuse_observations.json"), + ], + ) + + assert init_result.exit_code == 0 + assert import_result.exit_code == 2 + assert "at least" in import_result.output assert not DuckDBStore(DEFAULT_STORE_PATH).stats()["raw_spans"] diff --git a/tests/test_multi_agent_waste.py b/tests/test_multi_agent_waste.py index a6688f7..f6525b6 100644 --- a/tests/test_multi_agent_waste.py +++ b/tests/test_multi_agent_waste.py @@ -453,7 +453,7 @@ def test_cli_multi_agent_waste_validates_non_finite_decimals() -> None: def test_multi_agent_fixture_smoke_path_documents_estimate(monkeypatch) -> None: - monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt") + monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt-value-123") with runner.isolated_filesystem(): init_result = runner.invoke(app, ["init"]) diff --git a/tests/test_normalize.py b/tests/test_normalize.py index 1f2b649..1cef477 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -48,7 +48,7 @@ def test_map_langfuse_raw_span_preserves_metrics_privacy_and_trace_attributes() "statusMessage": "Timeout 504 after 30000 ms", "metadata": {"error_type": "TimeoutError"}, "sessionId": "session-1", - "userId": "user-hash-1", + "userId": "raw-user-1", "environment": "prod", "version": "2026.04.26", "promptName": "refund_flow", @@ -59,6 +59,8 @@ def test_map_langfuse_raw_span_preserves_metrics_privacy_and_trace_attributes() "output_retry_fingerprint": "output-retry-fingerprint", "input_preview": "redacted input", "output_preview": "redacted output", + "session_hash": "hashed-session-1", + "user_hash": "hashed-user-1", }, } ), @@ -86,8 +88,11 @@ def test_map_langfuse_raw_span_preserves_metrics_privacy_and_trace_attributes() assert span.input_retry_fingerprint == "input-retry-fingerprint" assert span.output_retry_fingerprint == "output-retry-fingerprint" assert span.output_preview == "redacted output" + assert span.session_id == "hashed-session-1" + assert span.user_hash == "hashed-user-1" assert span.attributes["promptName"] == "refund_flow" - assert span.attributes["sessionId"] == "session-1" + assert "sessionId" not in span.attributes + assert "userId" not in span.attributes assert span.attributes["environment"] == "prod" @@ -107,6 +112,33 @@ def test_langfuse_error_signature_ignores_volatile_values() -> None: assert first.error_signature == "connection # to [hex] failed at [timestamp] for [uuid]" +def test_map_langfuse_raw_span_does_not_promote_raw_session_id() -> None: + span = map_langfuse_raw_span( + RawSpanRow( + source="langfuse", + source_id="span-identity", + trace_id="trace-identity", + span_id="span-identity", + parent_span_id=None, + payload_json=json.dumps( + { + "id": "span-identity", + "traceId": "trace-identity", + "name": "support_agent", + "sessionId": "raw-session-1", + "userId": "raw-user-1", + } + ), + raw_ref="observations.json", + ) + ) + + assert span.session_id is None + assert span.user_hash is None + assert "sessionId" not in span.attributes + assert "userId" not in span.attributes + + def test_top_level_langfuse_generation_counts_as_model_call() -> None: span = map_langfuse_raw_span( RawSpanRow( @@ -161,7 +193,7 @@ def test_build_normalized_traces_rolls_up_tree_metrics_and_outcome() -> None: assert trace.trace_id == "trace-1" assert trace.root_span_id == "root" assert trace.root_name == "support_agent" - assert trace.session_id == "session-1" + assert trace.session_id == "session-hash-1" assert trace.user_hash == "user-hash-1" assert trace.environment == "prod" assert trace.version == "2026.04.26" @@ -377,8 +409,62 @@ def test_normalize_store_persists_normalized_rows_idempotently(tmp_path: Path) - ] +def test_normalize_persists_hashed_user_without_raw_identity_attributes( + tmp_path: Path, +) -> None: + store = DuckDBStore(tmp_path / "agentprof.duckdb") + store.insert_raw_spans( + [ + RawSpanRecord( + source="langfuse", + source_id="root", + trace_id="trace-user", + span_id="root", + parent_span_id=None, + payload_json=json.dumps( + { + "id": "root", + "traceId": "trace-user", + "name": "support_agent", + "sessionId": "session-1", + "userId": "raw-user-1", + "_agentprof_privacy": { + "session_hash": "hashed-session-1", + "user_hash": "hashed-user-1", + }, + } + ), + ) + ] + ) + + normalize_store(store) + + with store.connect() as connection: + trace = connection.execute( + """ + SELECT session_id, user_hash + FROM normalized_traces + WHERE trace_id = 'trace-user' + """ + ).fetchone() + attributes_json = connection.execute( + """ + SELECT attributes_json + FROM normalized_spans + WHERE trace_id = 'trace-user' AND span_id = 'root' + """ + ).fetchone()[0] + + attributes = json.loads(attributes_json) + + assert trace == ("hashed-session-1", "hashed-user-1") + assert "sessionId" not in attributes + assert "userId" not in attributes + + def test_cli_normalize_imported_langfuse_fixture(monkeypatch) -> None: - monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt") + monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt-value-123") with runner.isolated_filesystem(): init_result = runner.invoke(app, ["init"]) import_result = runner.invoke( @@ -419,9 +505,9 @@ def _normalized_spans() -> list[NormalizedSpan]: end_time=_dt("2026-04-26T10:00:05+00:00"), status="ok", input_hash="root-input-hash", + session_id="session-hash-1", + user_hash="user-hash-1", attributes={ - "sessionId": "session-1", - "userId": "user-hash-1", "environment": "prod", "version": "2026.04.26", }, diff --git a/tests/test_privacy_hashing.py b/tests/test_privacy_hashing.py index 62e7c3f..e6a9995 100644 --- a/tests/test_privacy_hashing.py +++ b/tests/test_privacy_hashing.py @@ -5,6 +5,7 @@ import pytest from agentprof.privacy.hashing import ( + MIN_HMAC_SALT_BYTES, MissingSaltError, canonicalize_for_hash, content_hash, @@ -87,9 +88,9 @@ def test_retry_fingerprint_normalizes_long_hex_inside_text() -> None: def test_salt_from_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("AGENTPROF_TEST_SALT", "secret") + monkeypatch.setenv("AGENTPROF_TEST_SALT", "strong-test-salt") - assert salt_from_env("AGENTPROF_TEST_SALT") == b"secret" + assert salt_from_env("AGENTPROF_TEST_SALT") == b"strong-test-salt" def test_salt_from_env_requires_value(monkeypatch: pytest.MonkeyPatch) -> None: @@ -99,3 +100,10 @@ def test_salt_from_env_requires_value(monkeypatch: pytest.MonkeyPatch) -> None: salt_from_env("AGENTPROF_TEST_SALT") assert os.getenv("AGENTPROF_TEST_SALT") is None + + +def test_salt_from_env_rejects_short_value(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AGENTPROF_TEST_SALT", "x" * (MIN_HMAC_SALT_BYTES - 1)) + + with pytest.raises(MissingSaltError, match=str(MIN_HMAC_SALT_BYTES)): + salt_from_env("AGENTPROF_TEST_SALT") diff --git a/tests/test_retry_loop.py b/tests/test_retry_loop.py index 5a262c3..3d123c8 100644 --- a/tests/test_retry_loop.py +++ b/tests/test_retry_loop.py @@ -217,7 +217,7 @@ def test_analyze_retry_loops_requires_min_attempts_at_least_two( def test_cli_retry_loop_analyzer_detects_fixture(monkeypatch) -> None: - monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt") + monkeypatch.setenv("AGENTPROF_HASH_SALT", "test-salt-value-123") with runner.isolated_filesystem(): init_result = runner.invoke(app, ["init"]) import_result = runner.invoke(