From 2c9b2c9bdafb743ea6b672462f96ac2b64d64db2 Mon Sep 17 00:00:00 2001
From: Dawson
Date: Tue, 18 Aug 2026 11:52:15 +0800
Subject: [PATCH 1/4] feat(labs): add AgentStream code
---
.pre-commit-config.yaml | 3 +
CHANGELOG.md | 2 +
README.md | 2 +
labs/AgentStream/LICENSE | 201 +
labs/AgentStream/README.md | 95 +
labs/AgentStream/exgentic/.dockerignore | 19 +
labs/AgentStream/exgentic/.gitattributes | 1 +
.../exgentic/.github/workflows/pre-commit.yml | 46 +
.../.github/workflows/publish-pypi.yml | 59 +
.../exgentic/.github/workflows/tests.yml | 64 +
labs/AgentStream/exgentic/.gitignore | 121 +
.../exgentic/.pre-commit-config.yaml | 66 +
labs/AgentStream/exgentic/.whitesource | 9 +
labs/AgentStream/exgentic/CODE_OF_CONDUCT.md | 128 +
labs/AgentStream/exgentic/CONTRIBUTING.md | 78 +
labs/AgentStream/exgentic/DCO.txt | 34 +
labs/AgentStream/exgentic/DEVELOPMENT.md | 129 +
labs/AgentStream/exgentic/LICENSE | 201 +
labs/AgentStream/exgentic/README.md | 249 +
labs/AgentStream/exgentic/SECURITY.md | 152 +
labs/AgentStream/exgentic/docs/README.md | 46 +
.../exgentic/docs/adding-agents.md | 384 +
.../exgentic/docs/adding-benchmarks.md | 364 +
labs/AgentStream/exgentic/docs/batch.md | 269 +
.../exgentic/docs/cli-reference.md | 323 +
.../exgentic/docs/custom-models.md | 250 +
labs/AgentStream/exgentic/docs/huggingface.md | 35 +
.../exgentic/docs/observability/quickstart.md | 151 +
.../observability/semantic-conventions.md | 172 +
labs/AgentStream/exgentic/docs/observers.md | 221 +
.../exgentic/docs/output-format.md | 325 +
labs/AgentStream/exgentic/docs/python-api.md | 361 +
labs/AgentStream/exgentic/docs/releasing.md | 119 +
.../exgentic/docs/replay-testing.md | 229 +
labs/AgentStream/exgentic/docs/runners.md | 208 +
.../exgentic/examples/run_appworld.py | 24 +
.../exgentic/examples/run_browsecomp.py | 24 +
.../examples/run_claude_code_on_gsm8k.py | 24 +
.../examples/run_claude_code_on_tau2bench.py | 24 +
.../exgentic/examples/run_cli_agents.py | 24 +
.../exgentic/examples/run_gsm8k.py | 23 +
.../exgentic/examples/run_hotpotqa.py | 24 +
.../exgentic/examples/run_openai_mcp.py | 33 +
.../AgentStream/exgentic/examples/run_smol.py | 24 +
.../exgentic/examples/run_swebench.py | 24 +
.../exgentic/examples/run_taubench.py | 24 +
.../examples/simple_test_agent/adapter.py | 54 +
.../examples/simple_test_agent/setup.sh | 3 +
labs/AgentStream/exgentic/misc/assets/cli.png | Bin 0 -> 1046661 bytes
.../misc/assets/exgentic_banner_black.png | Bin 0 -> 38420 bytes
.../exgentic_banner_black_no_background.png | Bin 0 -> 31271 bytes
.../misc/assets/exgentic_banner_white.png | Bin 0 -> 38083 bytes
.../exgentic_banner_white_no_background.png | Bin 0 -> 33838 bytes
.../exgentic/misc/assets/exgentic_light.png | Bin 0 -> 53953 bytes
labs/AgentStream/exgentic/misc/assets/gui.png | Bin 0 -> 549586 bytes
.../AgentStream/exgentic/misc/assets/icon.png | Bin 0 -> 6373 bytes
.../exgentic/misc/assets/icon_black.png | Bin 0 -> 6897 bytes
.../misc/security/requirements/README.md | 39 +
.../requirements/appworld/requirements.txt | 195 +
.../browsecompplus/requirements.txt | 261 +
.../requirements/core/requirements.txt | 160 +
.../requirements/gsm8k/requirements.txt | 165 +
.../requirements/hotpotqa/requirements.txt | 190 +
.../requirements/swebench/requirements.txt | 194 +
.../requirements/tau2/requirements.txt | 182 +
.../misc/security/setup_environments.sh | 303 +
.../exgentic/misc/skills/add-agent/SKILL.md | 68 +
.../misc/skills/add-benchmark/SKILL.md | 62 +
.../exgentic/misc/utils/.secrets.baseline | 150 +
.../misc/utils/enforce_dependency_caps.py | 124 +
.../misc/utils/enforce_library_imports.py | 32 +
.../misc/utils/enforce_relative_imports.py | 41 +
.../misc/utils/enforce_spdx_header.py | 70 +
labs/AgentStream/exgentic/pyproject.toml | 111 +
labs/AgentStream/exgentic/renovate.json | 13 +
labs/AgentStream/exgentic/ruff.toml | 78 +
.../exgentic/scripts/a_mem/run_experiment.py | 416 +
.../exgentic/scripts/a_mem/run_experiment.sh | 72 +
.../exgentic/scripts/ace/run_experiment.py | 386 +
.../exgentic/scripts/ace/run_experiment.sh | 60 +
.../scripts/autoskill/run_experiment.py | 382 +
.../scripts/autoskill/run_experiment.sh | 61 +
.../scripts/harness/run_experiment.py | 386 +
.../scripts/harness/run_experiment.sh | 57 +
.../exgentic/scripts/litellm/run_baseline.py | 238 +
.../exgentic/scripts/litellm/run_baseline.sh | 84 +
.../scripts/reasoning_bank/run_experiment.py | 403 +
.../scripts/reasoning_bank/run_experiment.sh | 70 +
labs/AgentStream/exgentic/scripts/release.sh | 76 +
.../exgentic/scripts/utils/task_ordering.py | 210 +
.../exgentic/src/exgentic/__init__.py | 86 +
.../src/exgentic/adapters/__init__.py | 2 +
.../src/exgentic/adapters/actions/__init__.py | 4 +
.../src/exgentic/adapters/actions/chat.py | 105 +
.../exgentic/adapters/actions/functions.py | 94 +
.../src/exgentic/adapters/agents/__init__.py | 4 +
.../exgentic/adapters/agents/code_agent.py | 52 +
.../exgentic/adapters/agents/coordinator.py | 321 +
.../src/exgentic/adapters/agents/mcp_agent.py | 104 +
.../exgentic/adapters/agents/mcp_server.py | 351 +
.../exgentic/adapters/executors/__init__.py | 4 +
.../src/exgentic/adapters/executors/proxy.py | 199 +
.../src/exgentic/adapters/runners/__init__.py | 125 +
.../src/exgentic/adapters/runners/_utils.py | 138 +
.../src/exgentic/adapters/runners/direct.py | 35 +
.../src/exgentic/adapters/runners/docker.py | 227 +
.../src/exgentic/adapters/runners/process.py | 174 +
.../src/exgentic/adapters/runners/service.py | 237 +
.../src/exgentic/adapters/runners/thread.py | 92 +
.../exgentic/adapters/runners/transport.py | 188 +
.../src/exgentic/adapters/runners/venv.py | 215 +
.../src/exgentic/adapters/schemas/__init__.py | 4 +
.../exgentic/adapters/schemas/json_schema.py | 112 +
.../src/exgentic/adapters/schemas/openai.py | 67 +
.../exgentic/src/exgentic/agents/__init__.py | 4 +
.../src/exgentic/agents/a_mem/__init__.py | 6 +
.../src/exgentic/agents/a_mem/a_mem_agent.py | 71 +
.../exgentic/agents/a_mem/a_mem_instance.py | 726 +
.../src/exgentic/agents/a_mem/memory_note.py | 142 +
.../src/exgentic/agents/a_mem/memory_store.py | 417 +
.../src/exgentic/agents/a_mem/prompts.py | 405 +
.../src/exgentic/agents/a_mem/retriever.py | 54 +
.../src/exgentic/agents/ace/__init__.py | 7 +
.../src/exgentic/agents/ace/ace_agent.py | 88 +
.../src/exgentic/agents/ace/ace_instance.py | 775 +
.../agents/ace/bulletpoint_analyzer.py | 200 +
.../src/exgentic/agents/ace/playbook_store.py | 172 +
.../src/exgentic/agents/ace/playbook_utils.py | 230 +
.../exgentic/agents/ace/prompts/__init__.py | 10 +
.../exgentic/agents/ace/prompts/curator.py | 67 +
.../exgentic/agents/ace/prompts/reflector.py | 53 +
.../src/exgentic/agents/autoskill/__init__.py | 7 +
.../agents/autoskill/autoskill_agent.py | 81 +
.../agents/autoskill/autoskill_instance.py | 636 +
.../src/exgentic/agents/autoskill/prompts.py | 153 +
.../agents/autoskill/skill_extraction.py | 141 +
.../agents/autoskill/skill_maintenance.py | 207 +
.../agents/autoskill/skill_retrieval.py | 159 +
.../exgentic/agents/autoskill/skill_store.py | 208 +
.../src/exgentic/agents/cli/__init__.py | 9 +
.../exgentic/src/exgentic/agents/cli/base.py | 350 +
.../exgentic/agents/cli/claude/__init__.py | 4 +
.../src/exgentic/agents/cli/claude/agent.py | 93 +
.../src/exgentic/agents/cli/claude/cli.py | 148 +
.../src/exgentic/agents/cli/claude/setup.sh | 32 +
.../src/exgentic/agents/cli/codex/__init__.py | 4 +
.../src/exgentic/agents/cli/codex/agent.py | 71 +
.../src/exgentic/agents/cli/codex/cli.py | 70 +
.../src/exgentic/agents/cli/codex/setup.sh | 32 +
.../src/exgentic/agents/cli/command_runner.py | 471 +
.../exgentic/agents/cli/gemini/__init__.py | 4 +
.../src/exgentic/agents/cli/gemini/agent.py | 81 +
.../src/exgentic/agents/cli/gemini/cli.py | 112 +
.../src/exgentic/agents/cli/gemini/setup.sh | 32 +
.../src/exgentic/agents/cli/requirements.txt | 1 +
.../src/exgentic/agents/harness/__init__.py | 6 +
.../src/exgentic/agents/harness/evolver.py | 382 +
.../exgentic/agents/harness/harness_agent.py | 70 +
.../agents/harness/harness_instance.py | 579 +
.../exgentic/agents/harness/harness_store.py | 320 +
.../agents/harness/prompts/__init__.py | 6 +
.../agents/harness/prompts/evolver.py | 52 +
.../exgentic/agents/harness/prompts/inject.py | 30 +
.../src/exgentic/agents/harness/retriever.py | 93 +
.../agents/litellm_tool_calling/__init__.py | 2 +
.../agents/litellm_tool_calling/instance.py | 486 +
.../litellm_tool_calling_agent.py | 54 +
.../agents/litellm_tool_calling/utils.py | 135 +
.../src/exgentic/agents/openai/__init__.py | 2 +
.../src/exgentic/agents/openai/instance.py | 307 +
.../agents/openai/openai_mcp_agent.py | 68 +
.../exgentic/agents/openai/requirements.txt | 1 +
.../agents/reasoning_bank/__init__.py | 0
.../agents/reasoning_bank/evaluator.py | 72 +
.../agents/reasoning_bank/induce_memory.py | 70 +
.../reasoning_bank/memory_management.py | 112 +
.../agents/reasoning_bank/prompts/__init__.py | 0
.../reasoning_bank/prompts/eval_prompts.py | 55 +
.../prompts/memory_instruction.py | 70 +
.../agents/reasoning_bank/rb_agent.py | 77 +
.../agents/reasoning_bank/rb_instance.py | 552 +
.../agents/reasoning_bank/rb_store.py | 164 +
.../src/exgentic/agents/replay/__init__.py | 2 +
.../exgentic/agents/replay/replay_agent.py | 107 +
.../agents/replay/replay_benchmark.py | 92 +
.../exgentic/agents/replay/replay_session.py | 139 +
.../exgentic/agents/smolagents/__init__.py | 12 +
.../exgentic/agents/smolagents/base_agent.py | 36 +
.../agents/smolagents/base_instance.py | 152 +
.../exgentic/agents/smolagents/code_agent.py | 21 +
.../agents/smolagents/code_instance.py | 58 +
.../agents/smolagents/requirements.txt | 1 +
.../smolagents/structured_code_agent.yaml | 257 +
.../agents/smolagents/tool_calling_agent.py | 21 +
.../smolagents/tool_calling_instance.py | 37 +
.../src/exgentic/agents/tool_shortlisting.py | 101 +
.../src/exgentic/benchmarks/__init__.py | 2 +
.../exgentic/benchmarks/appworld/__init__.py | 8 +
.../benchmarks/appworld/appworld_benchmark.py | 57 +
.../benchmarks/appworld/appworld_eval.py | 673 +
.../benchmarks/appworld/requirements.txt | 1 +
.../src/exgentic/benchmarks/appworld/setup.sh | 25 +
.../src/exgentic/benchmarks/bfcl/__init__.py | 8 +
.../benchmarks/bfcl/bfcl_benchmark.py | 93 +
.../src/exgentic/benchmarks/bfcl/bfcl_eval.py | 576 +
.../src/exgentic/benchmarks/bfcl/bfcl_shim.py | 68 +
.../src/exgentic/benchmarks/bfcl/setup.sh | 30 +
.../benchmarks/browsecompplus/__init__.py | 2 +
.../browsecompplus/browsecomp_benchmark.py | 697 +
.../browsecompplus/browsecomp_eval.py | 156 +
.../browsecompplus/make_light_dataset.py | 25 +
.../browsecompplus/requirements.txt | 2 +
.../benchmarks/browsecompplus/retriever.py | 133 +
.../browsecompplus/search_service.py | 173 +
.../browsecompplus/search_tool_handler.py | 81 +
.../browsecompplus/searcher_cache.py | 69 +
.../benchmarks/browsecompplus/setup.sh | 93 +
.../src/exgentic/benchmarks/gsm8k/__init__.py | 2 +
.../benchmarks/gsm8k/gsm8k_benchmark.py | 351 +
.../benchmarks/gsm8k/requirements.txt | 1 +
.../src/exgentic/benchmarks/hle/__init__.py | 2 +
.../exgentic/benchmarks/hle/hle_benchmark.py | 390 +
.../exgentic/benchmarks/hle/requirements.txt | 3 +
.../exgentic/benchmarks/hotpotqa/__init__.py | 2 +
.../benchmarks/hotpotqa/hotpotqa_benchmark.py | 373 +
.../benchmarks/hotpotqa/requirements.txt | 3 +
.../exgentic/benchmarks/swebench/__init__.py | 2 +
.../exgentic/benchmarks/swebench/config.yaml | 106 +
.../exgentic/benchmarks/swebench/readme.md | 43 +
.../benchmarks/swebench/requirements.txt | 2 +
.../benchmarks/swebench/swebench_benchmark.py | 119 +
.../benchmarks/swebench/swebench_eval.py | 466 +
.../swebench/swebench_evaluation.py | 98 +
.../benchmarks/swebench/swebench_logs.py | 178 +
.../benchmarks/swebench/swebench_metrics.py | 31 +
.../src/exgentic/benchmarks/tau2/__init__.py | 29 +
.../exgentic/benchmarks/tau2/requirements.txt | 1 +
.../src/exgentic/benchmarks/tau2/setup.sh | 24 +
.../exgentic/benchmarks/tau2/system-deps.txt | 1 +
.../benchmarks/tau2/tau2_benchmark.py | 61 +
.../src/exgentic/benchmarks/tau2/tau2_eval.py | 679 +
.../src/exgentic/benchmarks/tau2/tau2_shim.py | 82 +
.../exgentic/src/exgentic/core/__init__.py | 45 +
.../exgentic/src/exgentic/core/actions.py | 340 +
.../exgentic/src/exgentic/core/agent.py | 89 +
.../src/exgentic/core/agent_instance.py | 84 +
.../exgentic/src/exgentic/core/benchmark.py | 94 +
.../exgentic/src/exgentic/core/context.py | 335 +
.../exgentic/src/exgentic/core/evaluator.py | 53 +
.../exgentic/core/orchestrator/__init__.py | 41 +
.../src/exgentic/core/orchestrator/cleanup.py | 23 +
.../exgentic/core/orchestrator/controller.py | 114 +
.../exgentic/core/orchestrator/execution.py | 418 +
.../exgentic/core/orchestrator/observer.py | 68 +
.../src/exgentic/core/orchestrator/run.py | 190 +
.../src/exgentic/core/orchestrator/session.py | 129 +
.../exgentic/core/orchestrator/termination.py | 71 +
.../src/exgentic/core/orchestrator/tracker.py | 176 +
.../src/exgentic/core/runner_mixin.py | 59 +
.../exgentic/src/exgentic/core/session.py | 161 +
.../src/exgentic/core/types/__init__.py | 75 +
.../src/exgentic/core/types/action.py | 108 +
.../src/exgentic/core/types/evaluation.py | 151 +
.../src/exgentic/core/types/model_settings.py | 62 +
.../src/exgentic/core/types/observation.py | 175 +
.../exgentic/src/exgentic/core/types/run.py | 304 +
.../src/exgentic/core/types/session.py | 223 +
.../src/exgentic/environment/__init__.py | 6 +
.../src/exgentic/environment/docker.py | 405 +
.../src/exgentic/environment/helpers.py | 200 +
.../src/exgentic/environment/instance.py | 19 +
.../src/exgentic/environment/local.py | 68 +
.../src/exgentic/environment/manager.py | 232 +
.../src/exgentic/environment/protocol.py | 39 +
.../exgentic/src/exgentic/environment/venv.py | 85 +
.../src/exgentic/integrations/__init__.py | 4 +
.../exgentic/integrations/litellm/__init__.py | 33 +
.../integrations/litellm/cache/__init__.py | 26 +
.../integrations/litellm/cache/core.py | 325 +
.../integrations/litellm/cache/key.py | 289 +
.../integrations/litellm/cache/log.py | 127 +
.../integrations/litellm/cache_utils.py | 6 +
.../exgentic/integrations/litellm/config.py | 114 +
.../exgentic/integrations/litellm/health.py | 51 +
.../exgentic/integrations/litellm/proxy.py | 296 +
.../integrations/litellm/trace_cost.py | 61 +
.../integrations/litellm/trace_logger.py | 670 +
.../src/exgentic/interfaces/__init__.py | 4 +
.../src/exgentic/interfaces/cli/__init__.py | 4 +
.../interfaces/cli/commands/__init__.py | 4 +
.../interfaces/cli/commands/analyze.py | 1223 ++
.../exgentic/interfaces/cli/commands/batch.py | 1031 +
.../interfaces/cli/commands/compare.py | 1169 ++
.../interfaces/cli/commands/dashboard.py | 25 +
.../interfaces/cli/commands/evaluate.py | 429 +
.../interfaces/cli/commands/listing.py | 105 +
.../interfaces/cli/commands/run_info.py | 212 +
.../exgentic/interfaces/cli/commands/serve.py | 61 +
.../exgentic/interfaces/cli/commands/setup.py | 166 +
.../src/exgentic/interfaces/cli/main.py | 126 +
.../src/exgentic/interfaces/cli/options.py | 474 +
.../src/exgentic/interfaces/cli/render.py | 263 +
.../src/exgentic/interfaces/cli/run.py | 9 +
.../exgentic/interfaces/dashboard/__init__.py | 2 +
.../src/exgentic/interfaces/dashboard/app.py | 243 +
.../interfaces/dashboard/views/__init__.py | 19 +
.../interfaces/dashboard/views/data.py | 529 +
.../interfaces/dashboard/views/formatting.py | 99 +
.../interfaces/dashboard/views/forms.py | 134 +
.../interfaces/dashboard/views/panels.py | 505 +
.../interfaces/dashboard/views/runtime.py | 895 +
.../interfaces/dashboard/views/state.py | 113 +
.../interfaces/dashboard/views/status.py | 87 +
.../src/exgentic/interfaces/lib/__init__.py | 4 +
.../src/exgentic/interfaces/lib/api.py | 613 +
.../src/exgentic/interfaces/registry.py | 352 +
.../src/exgentic/observers/__init__.py | 2 +
.../exgentic/observers/handlers/__init__.py | 4 +
.../exgentic/observers/handlers/configs.py | 48 +
.../observers/handlers/dashboard_events.py | 360 +
.../observers/handlers/file_logger.py | 337 +
.../src/exgentic/observers/handlers/logger.py | 508 +
.../src/exgentic/observers/handlers/otel.py | 451 +
.../src/exgentic/observers/handlers/recap.py | 145 +
.../exgentic/observers/handlers/results.py | 537 +
.../observers/handlers/session_ledger.py | 66 +
.../exgentic/observers/handlers/warnings.py | 27 +
.../exgentic/observers/logging/__init__.py | 358 +
.../exgentic/observers/tracing/__init__.py | 4 +
.../exgentic/src/exgentic/testing/__init__.py | 42 +
.../exgentic/src/exgentic/testing/agent.py | 145 +
.../src/exgentic/testing/benchmark.py | 180 +
.../src/exgentic/testing/calculator.py | 50 +
.../src/exgentic/testing/docker_session.py | 68 +
.../exgentic/src/exgentic/utils/__init__.py | 2 +
.../exgentic/src/exgentic/utils/cost.py | 222 +
.../exgentic/src/exgentic/utils/disk_cache.py | 168 +
.../exgentic/src/exgentic/utils/logging.py | 87 +
.../exgentic/src/exgentic/utils/otel.py | 532 +
.../exgentic/src/exgentic/utils/paths.py | 192 +
.../exgentic/src/exgentic/utils/settings.py | 142 +
.../exgentic/src/exgentic/utils/sync.py | 48 +
labs/AgentStream/exgentic/tests/__init__.py | 4 +
.../tests/adapters/runners/__init__.py | 2 +
.../tests/adapters/runners/conftest.py | 36 +
.../tests/adapters/runners/test_docker.py | 69 +
.../adapters/runners/test_e2e_session.py | 289 +
.../tests/adapters/runners/test_process.py | 43 +
.../tests/adapters/runners/test_thread.py | 41 +
.../tests/adapters/runners/test_transport.py | 101 +
.../tests/adapters/runners/test_utils.py | 80 +
.../tests/adapters/runners/test_venv.py | 69 +
.../agents/cli/test_claude_cli_config.py | 30 +
.../tests/agents/cli/test_cli_context_env.py | 54 +
.../agents/cli/test_cli_error_surfacing.py | 59 +
.../tests/agents/test_tool_calling_utils.py | 31 +
.../exgentic/tests/api/__init__.py | 4 +
.../exgentic/tests/api/conftest.py | 37 +
.../exgentic/tests/api/fixtures/__init__.py | 4 +
.../exgentic/tests/api/fixtures/test_agent.py | 27 +
.../tests/api/fixtures/test_benchmark.py | 15 +
.../tests/api/test_agent_package_integrity.py | 142 +
.../exgentic/tests/api/test_api_errors.py | 50 +
.../exgentic/tests/api/test_api_files.py | 34 +
.../exgentic/tests/api/test_api_instances.py | 66 +
.../exgentic/tests/api/test_api_limits.py | 50 +
.../tests/api/test_api_missing_results.py | 43 +
.../exgentic/tests/api/test_api_random.py | 30 +
.../exgentic/tests/api/test_api_reuse.py | 42 +
.../exgentic/tests/api/test_api_run_config.py | 87 +
.../exgentic/tests/api/test_api_runners.py | 165 +
.../tests/api/test_api_session_config.py | 41 +
.../exgentic/tests/api/test_cli_batch.py | 132 +
.../exgentic/tests/api/test_cli_commands.py | 102 +
.../exgentic/tests/api/test_cli_compare.py | 1638 ++
.../exgentic/tests/api/test_cli_version.py | 24 +
.../tests/api/test_package_exports.py | 30 +
.../exgentic/tests/benchmarks/__init__.py | 2 +
.../recordings/appworld/recording.json | 6 +
.../recordings/appworld/results.json | 79 +
.../recordings/appworld/session.json | 17009 ++++++++++++++++
.../recordings/appworld/trajectory.jsonl | 63 +
.../recordings/browsecompplus/recording.json | 5 +
.../recordings/browsecompplus/results.json | 206 +
.../recordings/browsecompplus/session.json | 79 +
.../browsecompplus/trajectory.jsonl | 55 +
.../recordings/swebench/recording.json | 5 +
.../recordings/swebench/results.json | 119 +
.../recordings/swebench/session.json | 48 +
.../recordings/swebench/trajectory.jsonl | 65 +
.../benchmarks/recordings/tau2/recording.json | 5 +
.../benchmarks/recordings/tau2/results.json | 77 +
.../benchmarks/recordings/tau2/session.json | 642 +
.../recordings/tau2/trajectory.jsonl | 17 +
.../tests/benchmarks/test_benchmark_replay.py | 153 +
.../tests/benchmarks/test_tau2_data_dir.py | 67 +
.../exgentic/tests/core/test_actions.py | 117 +
.../exgentic/tests/core/test_context_env.py | 45 +
.../tests/core/test_run_results_version.py | 20 +
.../exgentic/tests/environment/__init__.py | 2 +
.../tests/environment/test_integration.py | 159 +
.../tests/environment/test_manager.py | 1967 ++
.../litellm/cache/test_async_key.py | 49 +
.../cache/test_cache_logger_context.py | 24 +
.../integrations/litellm/cache/test_key.py | 91 +
.../litellm/cache/test_settings.py | 155 +
.../litellm/cache/test_sync_key.py | 76 +
.../integrations/litellm/proxy/conftest.py | 77 +
.../proxy/test_proxy_cache_execution.py | 51 +
.../proxy/test_proxy_callback_config.py | 54 +
.../proxy/test_proxy_callback_execution.py | 157 +
.../litellm/proxy/test_proxy_env.py | 39 +
.../test_proxy_subprocess_integration.py | 110 +
.../litellm/proxy/test_trace_logger_env.py | 162 +
.../tests/integrations/litellm/test_health.py | 85 +
.../integrations/litellm/test_trace_cost.py | 58 +
.../litellm/test_trace_logger_context.py | 33 +
.../tests/integrations/test_mcp_agent.py | 214 +
.../tests/integrations/test_mcp_server.py | 342 +
.../exgentic/tests/setup/__init__.py | 4 +
.../exgentic/tests/setup/test_tool_install.py | 103 +
.../exgentic/tests/test_coordinator.py | 298 +
.../exgentic/tests/test_env_loading.py | 33 +
.../tests/test_integrations_functions.py | 91 +
.../exgentic/tests/utils/test_cost.py | 87 +
.../utils/test_litellm_cache_settings.py | 51 +
labs/AgentStream/exgentic/uv.lock | 4101 ++++
labs/AgentStream/exgentic/whitesource.config | 2 +
labs/AgentStream/figs/evaluation_compare.png | Bin 0 -> 311885 bytes
429 files changed, 82578 insertions(+)
create mode 100644 labs/AgentStream/LICENSE
create mode 100644 labs/AgentStream/exgentic/.dockerignore
create mode 100644 labs/AgentStream/exgentic/.gitattributes
create mode 100644 labs/AgentStream/exgentic/.github/workflows/pre-commit.yml
create mode 100644 labs/AgentStream/exgentic/.github/workflows/publish-pypi.yml
create mode 100644 labs/AgentStream/exgentic/.github/workflows/tests.yml
create mode 100644 labs/AgentStream/exgentic/.gitignore
create mode 100644 labs/AgentStream/exgentic/.pre-commit-config.yaml
create mode 100644 labs/AgentStream/exgentic/.whitesource
create mode 100644 labs/AgentStream/exgentic/CODE_OF_CONDUCT.md
create mode 100644 labs/AgentStream/exgentic/CONTRIBUTING.md
create mode 100644 labs/AgentStream/exgentic/DCO.txt
create mode 100644 labs/AgentStream/exgentic/DEVELOPMENT.md
create mode 100644 labs/AgentStream/exgentic/LICENSE
create mode 100644 labs/AgentStream/exgentic/README.md
create mode 100644 labs/AgentStream/exgentic/SECURITY.md
create mode 100644 labs/AgentStream/exgentic/docs/README.md
create mode 100644 labs/AgentStream/exgentic/docs/adding-agents.md
create mode 100644 labs/AgentStream/exgentic/docs/adding-benchmarks.md
create mode 100644 labs/AgentStream/exgentic/docs/batch.md
create mode 100644 labs/AgentStream/exgentic/docs/cli-reference.md
create mode 100644 labs/AgentStream/exgentic/docs/custom-models.md
create mode 100644 labs/AgentStream/exgentic/docs/huggingface.md
create mode 100644 labs/AgentStream/exgentic/docs/observability/quickstart.md
create mode 100644 labs/AgentStream/exgentic/docs/observability/semantic-conventions.md
create mode 100644 labs/AgentStream/exgentic/docs/observers.md
create mode 100644 labs/AgentStream/exgentic/docs/output-format.md
create mode 100644 labs/AgentStream/exgentic/docs/python-api.md
create mode 100644 labs/AgentStream/exgentic/docs/releasing.md
create mode 100644 labs/AgentStream/exgentic/docs/replay-testing.md
create mode 100644 labs/AgentStream/exgentic/docs/runners.md
create mode 100644 labs/AgentStream/exgentic/examples/run_appworld.py
create mode 100644 labs/AgentStream/exgentic/examples/run_browsecomp.py
create mode 100644 labs/AgentStream/exgentic/examples/run_claude_code_on_gsm8k.py
create mode 100644 labs/AgentStream/exgentic/examples/run_claude_code_on_tau2bench.py
create mode 100644 labs/AgentStream/exgentic/examples/run_cli_agents.py
create mode 100644 labs/AgentStream/exgentic/examples/run_gsm8k.py
create mode 100644 labs/AgentStream/exgentic/examples/run_hotpotqa.py
create mode 100644 labs/AgentStream/exgentic/examples/run_openai_mcp.py
create mode 100644 labs/AgentStream/exgentic/examples/run_smol.py
create mode 100644 labs/AgentStream/exgentic/examples/run_swebench.py
create mode 100644 labs/AgentStream/exgentic/examples/run_taubench.py
create mode 100644 labs/AgentStream/exgentic/examples/simple_test_agent/adapter.py
create mode 100644 labs/AgentStream/exgentic/examples/simple_test_agent/setup.sh
create mode 100644 labs/AgentStream/exgentic/misc/assets/cli.png
create mode 100644 labs/AgentStream/exgentic/misc/assets/exgentic_banner_black.png
create mode 100644 labs/AgentStream/exgentic/misc/assets/exgentic_banner_black_no_background.png
create mode 100644 labs/AgentStream/exgentic/misc/assets/exgentic_banner_white.png
create mode 100644 labs/AgentStream/exgentic/misc/assets/exgentic_banner_white_no_background.png
create mode 100644 labs/AgentStream/exgentic/misc/assets/exgentic_light.png
create mode 100644 labs/AgentStream/exgentic/misc/assets/gui.png
create mode 100644 labs/AgentStream/exgentic/misc/assets/icon.png
create mode 100644 labs/AgentStream/exgentic/misc/assets/icon_black.png
create mode 100644 labs/AgentStream/exgentic/misc/security/requirements/README.md
create mode 100644 labs/AgentStream/exgentic/misc/security/requirements/appworld/requirements.txt
create mode 100644 labs/AgentStream/exgentic/misc/security/requirements/browsecompplus/requirements.txt
create mode 100644 labs/AgentStream/exgentic/misc/security/requirements/core/requirements.txt
create mode 100644 labs/AgentStream/exgentic/misc/security/requirements/gsm8k/requirements.txt
create mode 100644 labs/AgentStream/exgentic/misc/security/requirements/hotpotqa/requirements.txt
create mode 100644 labs/AgentStream/exgentic/misc/security/requirements/swebench/requirements.txt
create mode 100644 labs/AgentStream/exgentic/misc/security/requirements/tau2/requirements.txt
create mode 100644 labs/AgentStream/exgentic/misc/security/setup_environments.sh
create mode 100644 labs/AgentStream/exgentic/misc/skills/add-agent/SKILL.md
create mode 100644 labs/AgentStream/exgentic/misc/skills/add-benchmark/SKILL.md
create mode 100644 labs/AgentStream/exgentic/misc/utils/.secrets.baseline
create mode 100644 labs/AgentStream/exgentic/misc/utils/enforce_dependency_caps.py
create mode 100644 labs/AgentStream/exgentic/misc/utils/enforce_library_imports.py
create mode 100644 labs/AgentStream/exgentic/misc/utils/enforce_relative_imports.py
create mode 100644 labs/AgentStream/exgentic/misc/utils/enforce_spdx_header.py
create mode 100644 labs/AgentStream/exgentic/pyproject.toml
create mode 100644 labs/AgentStream/exgentic/renovate.json
create mode 100644 labs/AgentStream/exgentic/ruff.toml
create mode 100644 labs/AgentStream/exgentic/scripts/a_mem/run_experiment.py
create mode 100644 labs/AgentStream/exgentic/scripts/a_mem/run_experiment.sh
create mode 100644 labs/AgentStream/exgentic/scripts/ace/run_experiment.py
create mode 100644 labs/AgentStream/exgentic/scripts/ace/run_experiment.sh
create mode 100644 labs/AgentStream/exgentic/scripts/autoskill/run_experiment.py
create mode 100644 labs/AgentStream/exgentic/scripts/autoskill/run_experiment.sh
create mode 100644 labs/AgentStream/exgentic/scripts/harness/run_experiment.py
create mode 100644 labs/AgentStream/exgentic/scripts/harness/run_experiment.sh
create mode 100644 labs/AgentStream/exgentic/scripts/litellm/run_baseline.py
create mode 100644 labs/AgentStream/exgentic/scripts/litellm/run_baseline.sh
create mode 100644 labs/AgentStream/exgentic/scripts/reasoning_bank/run_experiment.py
create mode 100644 labs/AgentStream/exgentic/scripts/reasoning_bank/run_experiment.sh
create mode 100644 labs/AgentStream/exgentic/scripts/release.sh
create mode 100644 labs/AgentStream/exgentic/scripts/utils/task_ordering.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/actions/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/actions/chat.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/actions/functions.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/agents/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/agents/code_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/agents/coordinator.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/agents/mcp_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/agents/mcp_server.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/executors/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/executors/proxy.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/runners/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/runners/_utils.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/runners/direct.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/runners/docker.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/runners/process.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/runners/service.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/runners/thread.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/runners/transport.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/runners/venv.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/schemas/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/schemas/json_schema.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/adapters/schemas/openai.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/a_mem/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/a_mem/a_mem_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/a_mem/a_mem_instance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/a_mem/memory_note.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/a_mem/memory_store.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/a_mem/prompts.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/a_mem/retriever.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/ace/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/ace/ace_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/ace/ace_instance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/ace/bulletpoint_analyzer.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/ace/playbook_store.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/ace/playbook_utils.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/ace/prompts/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/ace/prompts/curator.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/ace/prompts/reflector.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/autoskill/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/autoskill/autoskill_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/autoskill/autoskill_instance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/autoskill/prompts.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/autoskill/skill_extraction.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/autoskill/skill_maintenance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/autoskill/skill_retrieval.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/autoskill/skill_store.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/base.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/claude/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/claude/agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/claude/cli.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/claude/setup.sh
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/codex/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/codex/agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/codex/cli.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/codex/setup.sh
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/command_runner.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/gemini/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/gemini/agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/gemini/cli.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/gemini/setup.sh
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/cli/requirements.txt
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/harness/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/harness/evolver.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/harness/harness_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/harness/harness_instance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/harness/harness_store.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/harness/prompts/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/harness/prompts/evolver.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/harness/prompts/inject.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/harness/retriever.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/litellm_tool_calling/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/litellm_tool_calling/instance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/litellm_tool_calling/litellm_tool_calling_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/litellm_tool_calling/utils.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/openai/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/openai/instance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/openai/openai_mcp_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/openai/requirements.txt
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/reasoning_bank/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/reasoning_bank/evaluator.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/reasoning_bank/induce_memory.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/reasoning_bank/memory_management.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/reasoning_bank/prompts/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/reasoning_bank/prompts/eval_prompts.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/reasoning_bank/prompts/memory_instruction.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/reasoning_bank/rb_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/reasoning_bank/rb_instance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/reasoning_bank/rb_store.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/replay/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/replay/replay_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/replay/replay_benchmark.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/replay/replay_session.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/smolagents/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/smolagents/base_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/smolagents/base_instance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/smolagents/code_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/smolagents/code_instance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/smolagents/requirements.txt
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/smolagents/structured_code_agent.yaml
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/smolagents/tool_calling_agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/smolagents/tool_calling_instance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/agents/tool_shortlisting.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/appworld/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/appworld/appworld_benchmark.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/appworld/appworld_eval.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/appworld/requirements.txt
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/appworld/setup.sh
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/bfcl/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/bfcl/bfcl_benchmark.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/bfcl/bfcl_eval.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/bfcl/bfcl_shim.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/bfcl/setup.sh
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/browsecompplus/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/browsecompplus/browsecomp_benchmark.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/browsecompplus/browsecomp_eval.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/browsecompplus/make_light_dataset.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/browsecompplus/requirements.txt
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/browsecompplus/retriever.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/browsecompplus/search_service.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/browsecompplus/search_tool_handler.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/browsecompplus/searcher_cache.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/browsecompplus/setup.sh
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/gsm8k/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/gsm8k/gsm8k_benchmark.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/gsm8k/requirements.txt
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/hle/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/hle/hle_benchmark.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/hle/requirements.txt
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/hotpotqa/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/hotpotqa/hotpotqa_benchmark.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/hotpotqa/requirements.txt
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/swebench/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/swebench/config.yaml
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/swebench/readme.md
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/swebench/requirements.txt
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/swebench/swebench_benchmark.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/swebench/swebench_eval.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/swebench/swebench_evaluation.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/swebench/swebench_logs.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/swebench/swebench_metrics.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/tau2/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/tau2/requirements.txt
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/tau2/setup.sh
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/tau2/system-deps.txt
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/tau2/tau2_benchmark.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/tau2/tau2_eval.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/benchmarks/tau2/tau2_shim.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/actions.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/agent_instance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/benchmark.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/context.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/evaluator.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/orchestrator/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/orchestrator/cleanup.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/orchestrator/controller.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/orchestrator/execution.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/orchestrator/observer.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/orchestrator/run.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/orchestrator/session.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/orchestrator/termination.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/orchestrator/tracker.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/runner_mixin.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/session.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/types/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/types/action.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/types/evaluation.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/types/model_settings.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/types/observation.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/types/run.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/core/types/session.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/environment/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/environment/docker.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/environment/helpers.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/environment/instance.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/environment/local.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/environment/manager.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/environment/protocol.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/environment/venv.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/integrations/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/integrations/litellm/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/integrations/litellm/cache/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/integrations/litellm/cache/core.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/integrations/litellm/cache/key.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/integrations/litellm/cache/log.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/integrations/litellm/cache_utils.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/integrations/litellm/config.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/integrations/litellm/health.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/integrations/litellm/proxy.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/integrations/litellm/trace_cost.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/integrations/litellm/trace_logger.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/commands/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/commands/analyze.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/commands/batch.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/commands/compare.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/commands/dashboard.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/commands/evaluate.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/commands/listing.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/commands/run_info.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/commands/serve.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/commands/setup.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/main.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/options.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/render.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/cli/run.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/dashboard/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/dashboard/app.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/dashboard/views/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/dashboard/views/data.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/dashboard/views/formatting.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/dashboard/views/forms.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/dashboard/views/panels.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/dashboard/views/runtime.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/dashboard/views/state.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/dashboard/views/status.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/lib/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/lib/api.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/interfaces/registry.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/handlers/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/handlers/configs.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/handlers/dashboard_events.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/handlers/file_logger.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/handlers/logger.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/handlers/otel.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/handlers/recap.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/handlers/results.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/handlers/session_ledger.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/handlers/warnings.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/logging/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/observers/tracing/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/testing/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/testing/agent.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/testing/benchmark.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/testing/calculator.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/testing/docker_session.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/utils/__init__.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/utils/cost.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/utils/disk_cache.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/utils/logging.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/utils/otel.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/utils/paths.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/utils/settings.py
create mode 100644 labs/AgentStream/exgentic/src/exgentic/utils/sync.py
create mode 100644 labs/AgentStream/exgentic/tests/__init__.py
create mode 100644 labs/AgentStream/exgentic/tests/adapters/runners/__init__.py
create mode 100644 labs/AgentStream/exgentic/tests/adapters/runners/conftest.py
create mode 100644 labs/AgentStream/exgentic/tests/adapters/runners/test_docker.py
create mode 100644 labs/AgentStream/exgentic/tests/adapters/runners/test_e2e_session.py
create mode 100644 labs/AgentStream/exgentic/tests/adapters/runners/test_process.py
create mode 100644 labs/AgentStream/exgentic/tests/adapters/runners/test_thread.py
create mode 100644 labs/AgentStream/exgentic/tests/adapters/runners/test_transport.py
create mode 100644 labs/AgentStream/exgentic/tests/adapters/runners/test_utils.py
create mode 100644 labs/AgentStream/exgentic/tests/adapters/runners/test_venv.py
create mode 100644 labs/AgentStream/exgentic/tests/agents/cli/test_claude_cli_config.py
create mode 100644 labs/AgentStream/exgentic/tests/agents/cli/test_cli_context_env.py
create mode 100644 labs/AgentStream/exgentic/tests/agents/cli/test_cli_error_surfacing.py
create mode 100644 labs/AgentStream/exgentic/tests/agents/test_tool_calling_utils.py
create mode 100644 labs/AgentStream/exgentic/tests/api/__init__.py
create mode 100644 labs/AgentStream/exgentic/tests/api/conftest.py
create mode 100644 labs/AgentStream/exgentic/tests/api/fixtures/__init__.py
create mode 100644 labs/AgentStream/exgentic/tests/api/fixtures/test_agent.py
create mode 100644 labs/AgentStream/exgentic/tests/api/fixtures/test_benchmark.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_agent_package_integrity.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_api_errors.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_api_files.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_api_instances.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_api_limits.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_api_missing_results.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_api_random.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_api_reuse.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_api_run_config.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_api_runners.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_api_session_config.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_cli_batch.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_cli_commands.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_cli_compare.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_cli_version.py
create mode 100644 labs/AgentStream/exgentic/tests/api/test_package_exports.py
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/__init__.py
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/appworld/recording.json
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/appworld/results.json
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/appworld/session.json
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/appworld/trajectory.jsonl
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/browsecompplus/recording.json
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/browsecompplus/results.json
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/browsecompplus/session.json
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/browsecompplus/trajectory.jsonl
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/swebench/recording.json
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/swebench/results.json
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/swebench/session.json
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/swebench/trajectory.jsonl
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/tau2/recording.json
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/tau2/results.json
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/tau2/session.json
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/recordings/tau2/trajectory.jsonl
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/test_benchmark_replay.py
create mode 100644 labs/AgentStream/exgentic/tests/benchmarks/test_tau2_data_dir.py
create mode 100644 labs/AgentStream/exgentic/tests/core/test_actions.py
create mode 100644 labs/AgentStream/exgentic/tests/core/test_context_env.py
create mode 100644 labs/AgentStream/exgentic/tests/core/test_run_results_version.py
create mode 100644 labs/AgentStream/exgentic/tests/environment/__init__.py
create mode 100644 labs/AgentStream/exgentic/tests/environment/test_integration.py
create mode 100644 labs/AgentStream/exgentic/tests/environment/test_manager.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/cache/test_async_key.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/cache/test_cache_logger_context.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/cache/test_key.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/cache/test_settings.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/cache/test_sync_key.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/proxy/conftest.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/proxy/test_proxy_cache_execution.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/proxy/test_proxy_callback_config.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/proxy/test_proxy_callback_execution.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/proxy/test_proxy_env.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/proxy/test_proxy_subprocess_integration.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/proxy/test_trace_logger_env.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/test_health.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/test_trace_cost.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/litellm/test_trace_logger_context.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/test_mcp_agent.py
create mode 100644 labs/AgentStream/exgentic/tests/integrations/test_mcp_server.py
create mode 100644 labs/AgentStream/exgentic/tests/setup/__init__.py
create mode 100644 labs/AgentStream/exgentic/tests/setup/test_tool_install.py
create mode 100644 labs/AgentStream/exgentic/tests/test_coordinator.py
create mode 100644 labs/AgentStream/exgentic/tests/test_env_loading.py
create mode 100644 labs/AgentStream/exgentic/tests/test_integrations_functions.py
create mode 100644 labs/AgentStream/exgentic/tests/utils/test_cost.py
create mode 100644 labs/AgentStream/exgentic/tests/utils/test_litellm_cache_settings.py
create mode 100644 labs/AgentStream/exgentic/uv.lock
create mode 100644 labs/AgentStream/exgentic/whitesource.config
create mode 100644 labs/AgentStream/figs/evaluation_compare.png
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index fb8dcce8..62a3f09a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -59,6 +59,9 @@ repos:
- "**/build/**"
- "-ignore"
- "**/vendor/**"
+ # Separately licensed Apache-2.0 project
+ - "-ignore"
+ - "labs/AgentStream/**"
# Generated protobuf / gRPC stubs
- "-ignore"
- "**/*.pb.go"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c282d64e..545112d8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,8 @@ Guidelines for editors:
### Added
+- **AgentStream:** add a streaming evaluation framework for self-evolving LLM agents under `labs/AgentStream`.
+
### Changed
### Deprecated
diff --git a/README.md b/README.md
index 17426c92..36807707 100644
--- a/README.md
+++ b/README.md
@@ -252,6 +252,8 @@ Contributions of all kinds are welcome: bug reports, feature ideas, documentatio
Sico is licensed under the [MIT License](LICENSE).
+The contents of [AgentStream](labs/AgentStream) are licensed separately under the [Apache License 2.0](labs/AgentStream/LICENSE).
+
## Trademarks
This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft trademarks or logos is subject to and must follow [Microsoft’s Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks). Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. Any use of third-party trademarks or logos are subject to those third-party’s policies.
diff --git a/labs/AgentStream/LICENSE b/labs/AgentStream/LICENSE
new file mode 100644
index 00000000..261eeb9e
--- /dev/null
+++ b/labs/AgentStream/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/labs/AgentStream/README.md b/labs/AgentStream/README.md
index 58fb6d24..a304f3a2 100644
--- a/labs/AgentStream/README.md
+++ b/labs/AgentStream/README.md
@@ -24,9 +24,14 @@
dapenghu@microsoft.com
+
+
+
+
## 🚀 News
+* **[2026/08]** Code is released!
* **[2026/07]** Code is under preparation. Stay tuned!
## 📖 Overview
@@ -38,3 +43,93 @@ Over these scenarios, we combinatorially evaluate five representative self-evolv
Our results show that self-evolution reliability varies across streaming scenarios, the benefit of self-evolution is gated by model capability and non-monotonic in model strength, and no single method dominates across models and scenarios.
These findings offer concrete guidance for selecting self-evolving methods across models and streaming scenarios.
Overall, we advocate that self-evolving agents should be evaluated under realistic task streams rather than isolated single-task settings.
+
+
+

+
+
+## ⚡️ Getting Started
+
+AgentStream is built on the [`Exgentic`](./exgentic) framework, which is bundled in this repository. The five self-evolving agents live under [`exgentic/src/exgentic/agents`](./exgentic/src/exgentic/agents), and the benchmarks are orchestrated through `exgentic`'s installation and runner infrastructure.
+
+### 1. Requirements
+
+- Python `>= 3.11`
+- [`uv`](https://github.com/astral-sh/uv)
+- Docker (optional)
+
+### 2. Install the local exgentic (agent side)
+
+Clone the repo and create an editable environment from the bundled `exgentic`:
+
+```bash
+git clone https://github.com/microsoft/Sico.git
+cd Sico/labs/AgentStream/exgentic
+
+# Install the local ./src/exgentic in editable mode into .venv/
+uv sync
+
+# Activate the environment
+source .venv/bin/activate
+```
+
+Verify that the self-evolving agents are visible from the local install:
+
+```bash
+uv run exgentic list agents
+```
+
+### 3. Install benchmarks (benchmark side)
+
+Each benchmark is installed into isolated **`venv`** environment:
+
+```bash
+cd Sico/labs/AgentStream/exgentic
+
+
+uv run exgentic install --benchmark tau2
+uv run exgentic install --benchmark bfcl
+uv run exgentic install --benchmark hle
+uv run exgentic install --benchmark appworld
+uv run exgentic install --benchmark swebench
+uv run exgentic install --benchmark browsecompplus
+```
+
+### 4. API credentials
+
+The runners call LLMs through [LiteLLM](https://docs.litellm.ai/). Set the credentials for your provider in the [`exgentic/scripts//run_experiment.sh`](./exgentic/scripts/ace/run_experiment.sh):
+
+```bash
+export OPENAI_API_KEY="..."
+export OPENAI_API_BASE="..."
+```
+
+### 5. Run the streaming experiments
+
+Each method has its own runner under [`exgentic/scripts/`](./exgentic/scripts). The shell script selects the streaming scenario via `MODE` (`isolated` | `sequential` | `interleaved`), the model, the seed, and the benchmark stream:
+
+```bash
+cd Sico/labs/AgentStream/exgentic/scripts/ace
+
+bash run_experiment.sh
+```
+
+
+## 🙏 Acknowledgement
+This work is based on [Exgentic](https://github.com/Exgentic/exgentic). We sincerely thank the authors and contributors of these excellent open-source projects.
+
+## 📚 Citation
+If you find our work helpful, please consider citing:
+
+```bibtex
+@article{yan2026agentstream,
+ title={AgentStream: How Well Do Self-Evolving LLM Agents Perform Under Streaming Tasks?},
+ author={Yan, Dong and Liang, Jian and Hu, Dapeng and He, Ran and Yuan, Nicholas Jing and Zhang, Qi and Tan, Tieniu},
+ journal={arXiv preprint arXiv:2608.00155},
+ year={2026}
+}
+```
+
+## 📄 License
+
+The contents of this AgentStream directory are licensed separately under the [Apache License 2.0](./LICENSE).
diff --git a/labs/AgentStream/exgentic/.dockerignore b/labs/AgentStream/exgentic/.dockerignore
new file mode 100644
index 00000000..79975c4c
--- /dev/null
+++ b/labs/AgentStream/exgentic/.dockerignore
@@ -0,0 +1,19 @@
+.venv/
+.git/
+outputs/
+__pycache__/
+*.pyc
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+.exgentic/
+tests/
+
+# Large benchmark data/assets — not needed in the base Docker image.
+# Benchmarks that need these should use setup_script or volumes instead.
+src/exgentic/benchmarks/browsecompplus/assets/
+# Tau2: exclude large pre-computed results and figures but keep domain
+# data files (tasks.json, policy.md, db.json) which the session needs.
+src/exgentic/benchmarks/tau2/installation/tau2-bench/data/tau2/results/
+src/exgentic/benchmarks/tau2/installation/tau2-bench/figs/
+src/exgentic/benchmarks/tau2/installation/tau2-bench/tests/
diff --git a/labs/AgentStream/exgentic/.gitattributes b/labs/AgentStream/exgentic/.gitattributes
new file mode 100644
index 00000000..6313b56c
--- /dev/null
+++ b/labs/AgentStream/exgentic/.gitattributes
@@ -0,0 +1 @@
+* text=auto eol=lf
diff --git a/labs/AgentStream/exgentic/.github/workflows/pre-commit.yml b/labs/AgentStream/exgentic/.github/workflows/pre-commit.yml
new file mode 100644
index 00000000..b76a8097
--- /dev/null
+++ b/labs/AgentStream/exgentic/.github/workflows/pre-commit.yml
@@ -0,0 +1,46 @@
+name: Pre-commit Checks
+
+on:
+ push:
+ branches: [ main, master, develop ]
+ pull_request:
+ branches: [ main, master, develop ]
+
+jobs:
+ pre-commit:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
+ - name: Cache pre-commit hooks
+ uses: actions/cache@v4
+ with:
+ path: ~/.cache/pre-commit
+ key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
+ restore-keys: |
+ pre-commit-
+
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install pre-commit
+
+ - name: Run pre-commit hooks
+ run: pre-commit run --all-files --show-diff-on-failure
+
+ - name: Upload pre-commit results
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: pre-commit-results
+ path: |
+ **/*.log
+ .pre-commit-config.yaml
+
+# Made with Bob
diff --git a/labs/AgentStream/exgentic/.github/workflows/publish-pypi.yml b/labs/AgentStream/exgentic/.github/workflows/publish-pypi.yml
new file mode 100644
index 00000000..9d5d203e
--- /dev/null
+++ b/labs/AgentStream/exgentic/.github/workflows/publish-pypi.yml
@@ -0,0 +1,59 @@
+name: Publish to PyPI
+
+on:
+ push:
+ tags:
+ - "v*"
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Ensure tag commit is on main
+ run: |
+ git fetch origin main
+ tag_commit="$(git rev-list -n 1 "$GITHUB_REF_NAME")"
+ git merge-base --is-ancestor "$tag_commit" origin/main
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+
+ - name: Build distributions
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install build twine
+ python -m build
+ python -m twine check dist/*
+
+ - name: Upload distributions
+ uses: actions/upload-artifact@v4
+ with:
+ name: python-package-distributions
+ path: dist/
+
+ publish:
+ needs: build
+ runs-on: ubuntu-latest
+ permissions:
+ id-token: write
+ environment:
+ name: pypi
+ url: https://pypi.org/project/exgentic/
+ steps:
+ - name: Download distributions
+ uses: actions/download-artifact@v4
+ with:
+ name: python-package-distributions
+ path: dist/
+
+ - name: Publish to PyPI
+ uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/labs/AgentStream/exgentic/.github/workflows/tests.yml b/labs/AgentStream/exgentic/.github/workflows/tests.yml
new file mode 100644
index 00000000..9316f916
--- /dev/null
+++ b/labs/AgentStream/exgentic/.github/workflows/tests.yml
@@ -0,0 +1,64 @@
+name: Tests
+
+on:
+ push:
+ branches: [main, master, develop]
+ pull_request:
+ branches: [main, master, develop]
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.11", "3.12"]
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@v6
+ with:
+ enable-cache: true
+ cache-dependency-glob: "uv.lock"
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install dependencies
+ run: uv sync --frozen --extra dev --extra analysis
+
+ - name: Run core tests
+ run: uv run --frozen pytest tests -v --ignore=tests/integrations --ignore=tests/adapters/runners --tb=short
+
+ - name: Run runner tests
+ run: uv run --frozen pytest tests/adapters/runners -v --tb=short -p no:faulthandler
+
+ - name: Upload test results
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: test-results-${{ matrix.python-version }}
+ path: "**/*.log"
+
+ docker-integration:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@v6
+ with:
+ enable-cache: true
+ cache-dependency-glob: "uv.lock"
+ python-version: "3.12"
+
+ - name: Install dependencies
+ run: uv sync --frozen --extra dev --extra analysis
+
+ - name: Run Docker integration tests
+ run: uv run --frozen pytest tests/environment/test_manager.py -v -k "Integration" --tb=short
diff --git a/labs/AgentStream/exgentic/.gitignore b/labs/AgentStream/exgentic/.gitignore
new file mode 100644
index 00000000..2609c5fc
--- /dev/null
+++ b/labs/AgentStream/exgentic/.gitignore
@@ -0,0 +1,121 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+!src/exgentic/interfaces/lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+src/exgentic/_version.py
+
+# Exgentic local data (venvs, caches, installations)
+.exgentic/
+.exgentic_installations/
+
+# Virtual environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# AI/CLI agent metadata
+.bob/
+.claude/
+.cursor/
+.aider*
+.copilot/
+.continue/
+
+# VS Code settings
+.vscode/
+.history/
+*.code-workspace
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+.jupyter/
+profile_default/
+
+# PyCharm
+.idea/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+outputs/
+# OS files
+.DS_Store
+Thumbs.db
+data/
+src/exgentic/benchmarks/appworld/data/
+src/exgentic/benchmarks/appworld/experiments/
+src/exgentic/benchmarks/tau2/installation/
+
+# Benchmark caches
+tau2_disk_cache/
+
+old_outputs2/
+outputs
+*outputs*
+.litellm_cache
+trace.jsonl
+
+t.*
+node_modules
+package-lock.json
+package.json
+src/exgentic/benchmarks/browsecompplus/assets
+exgentic_session_cache/
+scripts/litellm/docker_vm_data/Ubuntu.qcow2.zip
diff --git a/labs/AgentStream/exgentic/.pre-commit-config.yaml b/labs/AgentStream/exgentic/.pre-commit-config.yaml
new file mode 100644
index 00000000..b7d44f52
--- /dev/null
+++ b/labs/AgentStream/exgentic/.pre-commit-config.yaml
@@ -0,0 +1,66 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ # Ruff version.
+ rev: v0.1.6
+ hooks:
+ # Run the linter on all files except the specific one
+ - id: ruff
+ args: [--fix]
+ - id: ruff-format
+
+ - repo: https://github.com/Yelp/detect-secrets
+ rev: v1.5.0
+ hooks:
+ - id: detect-secrets
+ args: [--baseline, misc/utils/.secrets.baseline]
+ exclude: misc/utils/.secrets.baseline
+
+ - repo: https://github.com/astral-sh/uv-pre-commit
+ rev: 0.7.12
+ hooks:
+ - id: uv-lock
+ args: [--locked]
+
+ - repo: https://github.com/codespell-project/codespell
+ rev: v2.2.6
+ hooks:
+ - id: codespell
+ additional_dependencies:
+ - tomli
+
+ - repo: local
+ hooks:
+ - id: enforce-spdx-header
+ name: Enforce SPDX Header
+ entry: python3 misc/utils/enforce_spdx_header.py
+ language: system
+ files: \.py$
+ exclude: ^\.venv/
+ types: [python]
+ - id: enforce-relative-imports
+ name: Enforce Relative Imports
+ entry: python3 misc/utils/enforce_relative_imports.py
+ language: system
+ # Adjust the files pattern to match your needs
+ files: ^src/.*\.py$
+ # Optional: Specify types or exclude files
+ types: [python]
+ - id: enforce-dependency-caps
+ name: Enforce Dependency Version Caps
+ entry: python3 misc/utils/enforce_dependency_caps.py
+ language: system
+ files: ^pyproject\.toml$
+ pass_filenames: false
+
+ - repo: local
+ hooks:
+ - id: enforce-library-imports
+ name: Enforce Library Imports
+ entry: python3 misc/utils/enforce_library_imports.py
+ language: system
+ # Adjust the files pattern to match your needs
+ exclude: (^src/.*\.py$)|misc/utils/enforce_library_imports.py|misc/utils/enforce_relative_imports.py
+ # Optional: Specify types or exclude files
+ types: [python]
diff --git a/labs/AgentStream/exgentic/.whitesource b/labs/AgentStream/exgentic/.whitesource
new file mode 100644
index 00000000..5e1a3914
--- /dev/null
+++ b/labs/AgentStream/exgentic/.whitesource
@@ -0,0 +1,9 @@
+{
+ "settingsInheritedFrom": "whitesource-config/whitesource-config@master",
+ "scanSettingsSAST": {
+ "enableScan": true
+ },
+ "scanSettings": {
+ "configMode": "LOCAL"
+ }
+}
diff --git a/labs/AgentStream/exgentic/CODE_OF_CONDUCT.md b/labs/AgentStream/exgentic/CODE_OF_CONDUCT.md
new file mode 100644
index 00000000..c8e52a2e
--- /dev/null
+++ b/labs/AgentStream/exgentic/CODE_OF_CONDUCT.md
@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socioeconomic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+- Demonstrating empathy and kindness toward other people
+- Being respectful of differing opinions, viewpoints, and experiences
+- Giving and gracefully accepting constructive feedback
+- Accepting responsibility and apologizing to those affected by our mistakes,
+ and learning from the experience
+- Focusing on what is best not just for us as individuals, but for the
+ overall community
+
+Examples of unacceptable behavior include:
+
+- The use of sexualized language or imagery, and sexual attention or
+ advances of any kind
+- Trolling, insulting or derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or email
+ address, without their explicit permission
+- Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible.
+
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.
diff --git a/labs/AgentStream/exgentic/CONTRIBUTING.md b/labs/AgentStream/exgentic/CONTRIBUTING.md
new file mode 100644
index 00000000..2a03a7c2
--- /dev/null
+++ b/labs/AgentStream/exgentic/CONTRIBUTING.md
@@ -0,0 +1,78 @@
+# How to contribute to Exgentic
+
+Thank you for your interest in contributing!
+
+## Development Setup
+
+```bash
+# Install dependencies using the pinned lock file — never plain `uv sync`
+uv sync --frozen --extra dev --extra analysis
+
+# To intentionally upgrade a specific package:
+uv lock --upgrade-package
+# Review the uv.lock diff carefully before committing
+```
+
+> **Security note:** Always use `uv sync --frozen` locally. Running plain `uv sync` may silently
+> upgrade packages and introduce untested or malicious versions. Dependency upgrades should be
+> explicit, reviewed, and go through a PR.
+
+## How to Contribute
+
+1. Fork the [repository](https://github.com/exgentic/exgentic).
+2. Create a new branch for your changes.
+3. Sign your commits using the `-s` flag (see [Legal](#legal))
+4. Submit a pull request to the `main` branch with a clear title and description.
+Reference any issues fixed, for example `Fixes #1234`.
+Ensure your PR title follows [semantic commit conventions](https://www.conventionalcommits.org/).
+5. A maintainer will review your PR and may request changes.
+
+## Legal
+
+### License
+
+This project is licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE).
+
+Each source code file must include the following SPDX headers at the top of the file:
+
+**For Python files:**
+```python
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2025, The Exgentic organization and its contributors.
+
+"""Module docstring here."""
+import ...
+```
+
+**For other file types:** Use the appropriate comment syntax for that language.
+
+### Developer Certificate of Origin (DCO)
+
+We require all commits to be **signed off** to indicate agreement with the [DCO](DCO.txt).
+
+By signing off a commit, you certify:
+
+> “I have the right to submit this contribution under the Apache License, Version 2.0 (or the open source license indicated in the file), and understand this project and my contribution are public.”
+
+### How to sign off your commits
+
+The easiest way is to use the `-s` flag when committing:
+
+```bash
+git commit -s -m "Fix: Correct spelling in README"
+```
+
+This uses your Git configuration. Make sure your name and email are set:
+```bash
+git config --global user.name "Your Name"
+git config --global user.email "your.email@example.com"
+```
+
+Alternatively you can manually sign your commit by adding this line to the commit message:
+```
+Signed-off-by: Your Name
+```
+
+## Development Environment Setup
+
+For detailed instructions on setting up your local development environment, see [DEVELOPMENT.md](./DEVELOPMENT.md).
diff --git a/labs/AgentStream/exgentic/DCO.txt b/labs/AgentStream/exgentic/DCO.txt
new file mode 100644
index 00000000..49b8cb05
--- /dev/null
+++ b/labs/AgentStream/exgentic/DCO.txt
@@ -0,0 +1,34 @@
+Developer Certificate of Origin
+Version 1.1
+
+Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+
+Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+ have the right to submit it under the open source license
+ indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+ of my knowledge, is covered under an appropriate open source
+ license and I have the right under that license to submit that
+ work with modifications, whether created in whole or in part
+ by me, under the same open source license (unless I am
+ permitted to submit under a different license), as indicated
+ in the file; or
+
+(c) The contribution was provided directly to me by some other
+ person who certified (a), (b) or (c) and I have not modified
+ it.
+
+(d) I understand and agree that this project and the contribution
+ are public and that a record of the contribution (including all
+ personal information I submit with it, including my sign-off) is
+ maintained indefinitely and may be redistributed consistent with
+ this project or the open source license(s) involved.
diff --git a/labs/AgentStream/exgentic/DEVELOPMENT.md b/labs/AgentStream/exgentic/DEVELOPMENT.md
new file mode 100644
index 00000000..814ea061
--- /dev/null
+++ b/labs/AgentStream/exgentic/DEVELOPMENT.md
@@ -0,0 +1,129 @@
+# Development Guide
+
+This guide covers setting up exgentic for local development, editing, and debugging.
+
+## Setup
+
+```bash
+git clone https://github.com/Exgentic/exgentic.git
+cd exgentic
+uv sync
+```
+
+## Setup Benchmarks & Agents
+
+Benchmarks and agents declare their dependencies through two mechanisms:
+
+- **`requirements.txt`** — pip packages installed automatically via `uv pip install`
+- **`setup.sh`** — shell script for non-pip setup (apt packages, git clones, data downloads)
+
+Both are auto-discovered next to the benchmark/agent module directory. The `exgentic install` command runs both:
+
+```bash
+# Benchmarks
+uv run exgentic install --benchmark tau2
+uv run exgentic install --benchmark appworld
+uv run exgentic install --benchmark gsm8k
+uv run exgentic install --benchmark hotpotqa
+uv run exgentic install --benchmark swebench
+uv run exgentic install --benchmark browsecompplus
+
+# Agents
+uv run exgentic install --agent litellm_tool_calling
+uv run exgentic install --agent smolagents
+uv run exgentic install --agent openai
+uv run exgentic install --agent claude
+uv run exgentic install --agent codex
+uv run exgentic install --agent gemini
+```
+
+> **Note:** `exgentic setup` still works but is deprecated. Use `install`/`uninstall` instead.
+
+### Isolated Runners (venv / docker)
+
+By default, benchmarks run with the `venv` runner, which creates an isolated `uv` virtual environment per benchmark under `.exgentic//venv/`. This means **no local setup is needed** — dependencies are installed automatically in the venv on first run.
+
+You can also use the `docker` runner for full container isolation:
+
+```bash
+uv run exgentic evaluate --benchmark tau2 --agent tool_calling --subset retail --num-tasks 2 \
+ --model gpt-4o \
+ --set benchmark.runner=docker \
+ --set benchmark.user_simulator_model="gpt-4o"
+```
+
+Both isolated runners follow the same pattern:
+
+1. Install `requirements.txt` and run `setup.sh` in the isolated environment
+2. Start `exgentic serve --cls --kwargs ` inside the venv/container
+3. Communicate over HTTP via the runner transport layer
+
+Setup scripts can check the `EXGENTIC_DOCKER_BUILD` environment variable to distinguish a Docker build from a local setup (e.g., to skip interactive prompts or large downloads that are handled differently in containers).
+
+## API Credentials
+
+```bash
+export OPENAI_API_KEY=...
+# or
+export ANTHROPIC_API_KEY=...
+```
+
+Or create a `.env` file in the project root — Exgentic loads it automatically.
+
+## Running Evaluations
+
+```bash
+uv run exgentic list benchmarks
+uv run exgentic list agents
+
+uv run exgentic evaluate --benchmark tau2 --agent tool_calling --subset retail --num-tasks 2 \
+ --model gpt-4o \
+ --set benchmark.user_simulator_model="gpt-4o"
+```
+
+## Tests
+
+```bash
+# Core tests (no Docker or external services required)
+uv run pytest tests/ --ignore=tests/integrations --ignore=tests/adapters/runners
+
+# Runner/transport tests (includes Docker tests on matching Python version)
+uv run pytest tests/adapters/runners -v -p no:faulthandler
+
+# API-level tests only
+uv run pytest tests/api
+
+# Skip tests requiring external services
+uv run pytest tests/ -k "not litellm and not mcp"
+```
+
+The test suite includes **replay tests** that re-run recorded benchmark sessions without any external dependencies. Recordings are stored under `tests/benchmarks/recordings/` and use `ReplayBenchmark` + `ReplayAgent` to verify the execution loop end-to-end.
+
+## Linting
+
+```bash
+pip install pre-commit
+pre-commit install
+pre-commit run --all-files
+```
+
+## OpenTelemetry Tracing
+
+```bash
+uv sync --extra otel
+
+export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
+export EXGENTIC_OTEL_ENABLED=true
+```
+
+See [`OTEL_SEMANTIC_CONVENTIONS.md`](./OTEL_SEMANTIC_CONVENTIONS.md) for details.
+
+## Releases
+
+- Release process guide: `docs/releasing.md`
+- Benchmark adapter design guide: `docs/adding-benchmarks.md`
+- Create and push a release tag: `scripts/release.sh 0.2.0 --push`
+- After PyPI publish succeeds, create the GitHub Release manually: `gh release create v0.2.0 --generate-notes --title "v0.2.0"`
+- Release versions come from Git tags via `hatch-vcs`
+- PyPI publishing uses GitHub Actions Trusted Publishing
diff --git a/labs/AgentStream/exgentic/LICENSE b/labs/AgentStream/exgentic/LICENSE
new file mode 100644
index 00000000..261eeb9e
--- /dev/null
+++ b/labs/AgentStream/exgentic/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/labs/AgentStream/exgentic/README.md b/labs/AgentStream/exgentic/README.md
new file mode 100644
index 00000000..11de4d8b
--- /dev/null
+++ b/labs/AgentStream/exgentic/README.md
@@ -0,0 +1,249 @@
+
+
+
+ Evaluate any agent on any benchmark in the simplest way possible
+
+
+---
+
+## What is Exgentic?
+
+Exgentic is a universal evaluation framework that enables standardized testing of AI agents across diverse benchmarks and domains. It provides a consistent interface for evaluating any agent on any benchmark, making it easy to compare performance, reproduce results, and ensure your agent works reliably across different tasks and environments.
+
+## Who is it for?
+
+1. **General Audience** - Visit [www.exgentic.ai](https://www.exgentic.ai) to explore the first general agent leaderboard comparing leading agents and frontier models across varied tasks.
+2. **Agent Builders** - Evaluate your agents comprehensively across multiple domains and benchmarks.
+3. **Researchers & Component Developers** - Test agentic components (memory, context compression, planning) across different agents and domains.
+4. **Benchmark Builders** - Evaluate your benchmark across multiple agents to ensure meaningful differentiation.
+
+---
+
+## Quick Start
+
+### Installation
+
+```bash
+uv tool install exgentic
+```
+
+### API Credentials
+
+```bash
+export OPENAI_API_KEY=...
+# or
+export ANTHROPIC_API_KEY=...
+```
+
+### Run an Evaluation
+
+```bash
+# List available benchmarks and agents
+exgentic list benchmarks
+exgentic list agents
+
+# Evaluate an agent on a benchmark
+exgentic evaluate --benchmark tau2 --agent tool_calling --subset retail --num-tasks 2 \
+ --model gpt-4o \
+ --set benchmark.user_simulator_model="gpt-4o"
+```
+
+Benchmarks are automatically installed on first run — no manual installation needed. You can also install them explicitly:
+
+```bash
+exgentic install --benchmark tau2 # install deps + data (default)
+exgentic install --agent tool_calling
+exgentic install --benchmark tau2 --docker # build Docker image
+exgentic install --benchmark tau2 --local # install into local environment
+exgentic uninstall --benchmark tau2 # remove installed environment
+```
+
+> **Note:** `exgentic setup` still works but is deprecated in favor of `install`/`uninstall`.
+
+For full container isolation, use the Docker runner (`--set benchmark.runner=docker`). You only need Docker installed and running:
+
+```bash
+exgentic evaluate --benchmark tau2 --agent tool_calling --subset retail --num-tasks 2 \
+ --model gpt-4o \
+ --set benchmark.runner=docker \
+ --set benchmark.user_simulator_model="gpt-4o"
+```
+
+### Python API
+
+To use exgentic as a library, install it first:
+
+```bash
+uv add exgentic # or: pip install exgentic
+```
+
+```python
+from exgentic import evaluate
+
+results = evaluate(
+ benchmark="tau2",
+ agent="tool_calling",
+ subset="retail",
+ num_tasks=2,
+ model="gpt-4o",
+ benchmark_kwargs={"user_simulator_model": "gpt-4o"},
+)
+```
+
+For more examples, see the [`examples/`](./examples/) directory.
+
+---
+
+## Available Benchmarks
+
+```bash
+exgentic list benchmarks
+```
+
+| Benchmark | Description |
+|-----------|-------------|
+| **tau2** | Simulated customer support tasks across multiple domains (mock, retail, airline, telecom) |
+| **appworld** | Multi-app API environment testing agents' ability to interact with application interfaces |
+| **browsecompplus** | Web search and browsing benchmark for information retrieval and navigation |
+| **swebench** | Software engineering benchmark for resolving real-world GitHub issues |
+| **hotpotqa** | Multi-hop question answering over Wikipedia |
+| **gsm8k** | Grade school math word problems with optional calculator tool |
+| **bfcl** | Berkeley Function Calling Leaderboard for evaluating tool-use capabilities |
+
+## Available Agents
+
+| Agent | Description |
+|-------|-------------|
+| **LiteLLM Tool Calling** | Generic tool-calling agent via LiteLLM |
+| **SmolAgents** | HuggingFace SmolAgents framework |
+| **OpenAI MCP** | OpenAI Responses API with MCP tools |
+| **Claude Code** | Anthropic Claude Code agent |
+| **Codex CLI** | OpenAI Codex CLI agent |
+| **Gemini CLI** | Google Gemini CLI agent |
+
+---
+
+## Dashboard
+
+
+
+```bash
+exgentic dashboard
+```
+
+---
+
+## Output Structure
+
+Each run creates its own directory under `outputs//`:
+
+```text
+outputs//
+├── results.json # Overall scores, costs, per-session statistics
+├── benchmark_results.json # Benchmark-specific aggregated results
+├── run/
+│ ├── config.json # Snapshot of benchmark and agent configuration
+│ ├── run.log # Main execution log
+│ └── warnings.log # Warnings during execution
+└── sessions//
+ ├── config.json # Session configuration
+ ├── results.json # Session results
+ ├── trajectory.jsonl # One JSON line per step (action + observation)
+ ├── agent/
+ │ └── agent.log # Agent execution log
+ └── benchmark/
+ ├── results.json # Benchmark-specific results
+ └── session.log # Benchmark session log
+```
+
+---
+
+## CLI Reference
+
+
+
+```bash
+# Discover
+exgentic list benchmarks
+exgentic list subsets --benchmark tau2
+exgentic list tasks --benchmark tau2 --subset retail --limit 5
+exgentic list agents
+exgentic install --benchmark tau2
+exgentic install --benchmark tau2 --docker
+exgentic install --benchmark tau2 --local
+exgentic uninstall --benchmark tau2
+
+# Run
+exgentic evaluate --benchmark tau2 --agent tool_calling --subset airline --num-tasks 10
+exgentic batch run --benchmark tau2 --agent tool_calling --subset airline --num-tasks 10
+
+# Inspect
+exgentic status --benchmark tau2 --agent tool_calling --subset airline --num-tasks 10
+exgentic preview --benchmark tau2 --agent tool_calling --subset airline --num-tasks 10
+exgentic results --benchmark tau2 --agent tool_calling --subset airline --num-tasks 10
+
+# Analyze
+exgentic compare --agents tool_calling openai --benchmark tau2
+
+# Explore
+exgentic dashboard
+```
+
+---
+
+## Advanced
+
+### Model Configuration
+
+```bash
+exgentic evaluate --benchmark tau2 --agent tool_calling --subset retail --num-tasks 2 \
+ --set agent.model.temperature=0.2
+```
+
+Supported fields: `temperature`, `top_p`, `max_tokens`, `reasoning_effort`, `num_retries`, `retry_after`, `retry_strategy`
+
+### Run Limits
+
+```bash
+exgentic evaluate --benchmark tau2 --agent tool_calling --subset retail --num-tasks 2 \
+ --max-steps 100 --max-actions 100
+```
+
+Sessions stop at either limit and record `limit_reached` status. Default: 100 for both.
+
+### HuggingFace
+
+Use HuggingFace models or run evaluations on HuggingFace Jobs. See [docs/huggingface.md](./docs/huggingface.md).
+
+---
+
+## How It Works
+
+To learn more about Exgentic's architecture and design, see our [arXiv paper](https://arxiv.org/abs/2602.22953).
+
+## Development
+
+For local development, editing, and contributing, see [DEVELOPMENT.md](./DEVELOPMENT.md).
+
+## Contributing
+
+We welcome issues and pull requests! See [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines.
+
+## Citing Exgentic
+
+```bibtex
+@misc{bandel2026generalagentevaluation,
+ title={General Agent Evaluation},
+ author={Elron Bandel and Asaf Yehudai and Lilach Eden and Yehoshua Sagron and Yotam Perlitz and Elad Venezian and Natalia Razinkov and Natan Ergas and Shlomit Shachor Ifergan and Segev Shlomov and Michal Jacovi and Leshem Choshen and Liat Ein-Dor and Yoav Katz and Michal Shmueli-Scheuer},
+ year={2026},
+ url={https://arxiv.org/abs/2602.22953},
+}
+```
+
+## License
+
+Apache License 2.0 — see [LICENSE](LICENSE).
+
+## Support
+
+For questions and support, [open an issue](https://github.com/Exgentic/exgentic/issues) on GitHub.
diff --git a/labs/AgentStream/exgentic/SECURITY.md b/labs/AgentStream/exgentic/SECURITY.md
new file mode 100644
index 00000000..db6c7c92
--- /dev/null
+++ b/labs/AgentStream/exgentic/SECURITY.md
@@ -0,0 +1,152 @@
+# Exgentic Security Policy & Responsible Disclosure
+
+## Security Policy
+
+This security policy applies to all public projects under the Exgentic organization on GitHub. We prioritize security and continuously work to safeguard our systems. However, vulnerabilities can still exist. If you identify a security issue, please report it to us so we can address it promptly.
+
+### Security/Bugfix Versions
+
+- Fixes are released either as part of the next minor version (e.g., 1.3.0 → 1.4.0) or as an on-demand patch version (e.g., 1.3.0 → 1.3.1)
+- Security fixes are given priority and might be enough to cause a new version to be released
+
+## Reporting a Vulnerability
+
+We encourage responsible disclosure of security vulnerabilities. If you find something suspicious, we encourage and appreciate your report!
+
+### How to Report
+
+Use the "Report a vulnerability" button under the "Security" tab of the [repository](https://github.com/exgentic/exgentic/security). This creates a private communication channel between you and the maintainers.
+
+### Reporting Guidelines
+
+- Provide clear details to help us reproduce and fix the issue quickly
+- Include steps to reproduce, potential impact, and any suggested fixes
+- Your report will be kept confidential, and your details will not be shared without your consent
+
+### Response Timeline
+
+- We will acknowledge your report within 5 business days
+- We will provide an estimated resolution timeline
+- We will keep you updated on our progress
+
+### Disclosure Guidelines
+
+- Do not publicly disclose vulnerabilities until we have assessed, resolved, and notified affected users
+- If you plan to present your research (e.g., at a conference or in a blog), share a draft with us at least 30 days in advance for review
+- Avoid including:
+ - Data from any customer projects
+ - User/customer information
+ - Details about employees, contractors, or partners
+
+We appreciate your efforts in helping us maintain a secure platform and look forward to working together to resolve any issues responsibly.
+
+## Dependency Management & Supply Chain Security
+
+### Version Capping Policy
+
+All direct dependencies in `pyproject.toml` are capped at the next major version (e.g., `litellm>=1.65.0,<2`). This policy limits the blast radius of supply chain attacks by preventing automatic upgrades to arbitrary future versions.
+
+**Why we cap dependencies:**
+- **Supply chain attack mitigation**: Malicious packages can be uploaded to PyPI at any time. By capping at major versions, we limit exposure to known version ranges.
+- **Controlled upgrades**: Major version bumps require explicit review and testing before adoption.
+- **Stability**: Prevents breaking changes from being automatically pulled in.
+
+**Enforcement:**
+- A pre-commit hook (`enforce-dependency-caps`) validates that all dependencies have upper bounds.
+- CI will fail if any direct dependency lacks an upper bound.
+- The hook runs automatically on every commit and in CI.
+
+### Automated Dependency Updates via Renovate
+
+We use Renovate to keep dependencies up to date while maintaining security:
+
+**14-Day Release Age Gate:**
+- Renovate is configured with `minimumReleaseAge: 14 days` for all Python dependencies.
+- New package versions are not proposed until 2 weeks after their PyPI release.
+- This reduces exposure to day-zero malicious uploads and gives the community time to identify compromised packages.
+
+**Major Version Bumps:**
+- Renovate uses `rangeStrategy: "bump"` to update both `uv.lock` and the upper bounds in `pyproject.toml` when a new major version is stable.
+- Major version PRs require careful review of breaking changes and thorough testing.
+
+### Reviewing Renovate PRs
+
+When reviewing Renovate PRs that update `uv.lock`:
+
+1. **Check the PR description** for the list of updated packages and their version changes.
+2. **Review the lockfile diff** to understand what's changing:
+ ```bash
+ gh pr diff -- uv.lock
+ ```
+3. **Verify the release age**: Ensure the new version has been available for at least 14 days.
+4. **Check for security advisories**: Look for any CVEs or security issues in the changelog.
+5. **Review changelogs**: For major updates, read the package's changelog for breaking changes.
+6. **Test thoroughly**: Run the full test suite and any relevant integration tests.
+
+### Lockfile Integrity
+
+A `uv-lock --locked` pre-commit hook (added in PR #65) ensures `uv.lock` stays in sync with `pyproject.toml`:
+- The hook rejects commits where the lockfile is out of sync.
+- This prevents accidental lockfile drift and ensures reproducible builds.
+- If the hook fails, run `uv lock` to regenerate the lockfile, review the changes, and commit.
+
+### Incident Response: Malicious Package Detected
+
+If a malicious package version is discovered in our dependencies:
+
+1. **Immediate containment:**
+ ```bash
+ # Pin the malicious version as excluded in pyproject.toml
+ # Example: "litellm>=1.65.0,!=1.82.7,!=1.82.8,<2"
+ ```
+
+2. **Rotate credentials:**
+ - Assume any secrets or credentials accessible to the compromised environment may be compromised.
+ - Rotate API keys, tokens, and passwords that were accessible during the infection window.
+
+3. **Clean infected environments:**
+ ```bash
+ # Remove all virtual environments
+ rm -rf .venv venv .exgentic/
+
+ # Reinstall with the patched dependency specification
+ uv sync
+ ```
+
+4. **Audit for data exfiltration:**
+ - Review logs and network traffic for suspicious outbound connections.
+ - Check for unauthorized access to systems or data.
+
+5. **Update lockfile:**
+ ```bash
+ uv lock
+ git add uv.lock pyproject.toml
+ git commit -m "Pin malicious package version as excluded"
+ ```
+
+6. **Notify the team** and document the incident.
+
+### CVE Scanning with uv audit
+
+The `uv audit` command scans dependencies for known CVEs:
+
+```bash
+uv audit
+```
+
+**Current status:**
+- `uv audit` is temporarily unavailable due to the litellm quarantine (versions 1.82.7 and 1.82.8 are excluded).
+- Once the quarantine is lifted and a clean version is available, re-enable regular `uv audit` checks.
+- Consider adding `uv audit` to CI once it's operational again.
+
+### Best Practices
+
+- **Never commit lockfiles without review**: Always inspect `uv.lock` diffs before committing.
+- **Keep dependencies minimal**: Only add dependencies that are truly necessary.
+- **Monitor security advisories**: Subscribe to security mailing lists for critical dependencies.
+- **Test updates thoroughly**: Don't merge Renovate PRs without running tests.
+- **Document exceptions**: If you must exclude a version (e.g., `!=1.82.8`), document why in a comment or commit message.
+
+## Known Vulnerabilities
+
+There are currently no known vulnerabilities.
diff --git a/labs/AgentStream/exgentic/docs/README.md b/labs/AgentStream/exgentic/docs/README.md
new file mode 100644
index 00000000..225c3f78
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/README.md
@@ -0,0 +1,46 @@
+# Exgentic Documentation
+
+Welcome to the Exgentic docs. Use the table below to find what you need.
+
+---
+
+## Using Exgentic
+
+| Document | Description |
+|----------|-------------|
+| [CLI Reference](./cli-reference.md) | Every command, flag, and environment variable |
+| [Python API](./python-api.md) | `evaluate()`, `execute()`, `aggregate()`, `status()`, `list_*()`, and all other library functions |
+| [Custom Models](./custom-models.md) | Use any LLM provider (OpenAI, Anthropic, Azure, Bedrock, Ollama, and more) via LiteLLM |
+| [Batch Runs](./batch.md) | Run parameter sweeps, manage large evaluations, export to CSV, publish to HuggingFace |
+| [Runners](./runners.md) | `direct`, `venv`, `docker` — isolation levels, configuration, Docker-in-Docker |
+| [Output Format](./output-format.md) | Schema for `results.json`, `trajectory.jsonl`, session results, and cost reports |
+| [Observers](./observers.md) | Hook into the evaluation lifecycle for custom logging, monitoring, and early stopping |
+
+## Extending Exgentic
+
+| Document | Description |
+|----------|-------------|
+| [Adding Agents](./adding-agents.md) | Write a new agent adapter — design principles, required methods, file layout, and validation checklist |
+| [Adding Benchmarks](./adding-benchmarks.md) | Write a new benchmark adapter — design principles, contract rules, and validation checklist |
+| [Replay Testing](./replay-testing.md) | Test benchmark and agent adapters end-to-end without API calls, using recorded sessions |
+
+## Observability
+
+| Document | Description |
+|----------|-------------|
+| [Quick Start](./observability/quickstart.md) | Set up OpenTelemetry tracing with Jaeger in five minutes |
+| [Semantic Conventions](./observability/semantic-conventions.md) | Full reference of every span and attribute Exgentic emits |
+
+## Maintainers
+
+| Document | Description |
+|----------|-------------|
+| [Releasing](./releasing.md) | Cut a release, publish to PyPI, and create a GitHub Release |
+
+---
+
+## Other resources
+
+- [README.md](../README.md) — project overview, quick start, CLI reference, and available benchmarks/agents
+- [DEVELOPMENT.md](../DEVELOPMENT.md) — local setup, running tests, linting, and the release process
+- [CONTRIBUTING.md](../CONTRIBUTING.md) — contribution workflow, legal requirements, and PR guidelines
diff --git a/labs/AgentStream/exgentic/docs/adding-agents.md b/labs/AgentStream/exgentic/docs/adding-agents.md
new file mode 100644
index 00000000..2f6b3f2f
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/adding-agents.md
@@ -0,0 +1,384 @@
+# Adding Agents
+
+This document defines the agent design principles for Exgentic.
+
+It is intentionally opinionated. An agent adapter should not just "work"; it should cleanly separate configuration from execution, isolate heavy dependencies, and adapt to any benchmark contract without requiring the benchmark to change.
+
+Use these existing adapters as reference points:
+- `src/exgentic/agents/litellm_tool_calling/litellm_tool_calling_agent.py` + `instance.py` (split pattern)
+- `src/exgentic/agents/cli/claude/agent.py` (light pattern, single file)
+
+**Related docs:**
+[docs/](./README.md) · [Adding Benchmarks](./adding-benchmarks.md) · [Custom Models](./custom-models.md) · [Runners](./runners.md) · [Replay Testing](./replay-testing.md) · [CONTRIBUTING.md](../CONTRIBUTING.md)
+
+## Core Principle
+
+The agent adapts to the benchmark contract, not the other way around.
+
+That means:
+- the benchmark decides the task, context, actions, step flow, and scoring
+- the agent receives these through `_get_instance_kwargs()` and must work within them
+- the agent should not require benchmark modifications to function
+- the agent should not impose protocol-specific assumptions on the benchmark
+
+The default goal should be the thinnest possible agent wrapper.
+
+That means:
+- translate the benchmark's actions into whatever protocol your agent uses (tool calls, code generation, CLI commands)
+- do not reshape the benchmark contract to match your model's preferred format
+- keep the configuration surface small and explicit
+
+## Architecture
+
+Exgentic agents are split into two classes with distinct roles:
+
+### Agent (config, host-side)
+
+`Agent` is a lightweight Pydantic model that holds configuration. It lives on the host and is never sent into an isolated runner. It has no heavy dependencies.
+
+Responsibilities:
+- declare `display_name` and `slug_name` as `ClassVar[str]`
+- hold user-facing configuration fields (model name, max steps, feature flags)
+- implement `_get_instance_class()` to resolve the execution class
+- implement `_get_instance_kwargs()` to translate config + benchmark contract into constructor arguments
+- optionally override `setup()` for non-pip setup (Docker builds, npm installs)
+- optionally override `model_name` / `get_models_names()` for dashboard metadata
+
+### AgentInstance (execution, venv-side)
+
+`AgentInstance` is the execution class. It runs inside the runner (venv, Docker, or local) and may import heavy third-party libraries.
+
+Responsibilities:
+- implement `react(observation) -> Action | None` as the core decision loop
+- implement `close()` for resource cleanup
+- optionally override `start()` for initialization that happens after construction
+- optionally override `get_cost()` to report monetary cost
+
+The agent instance receives a single `session_id` in its constructor, which scopes all logs and artifacts. Additional kwargs come from `_get_instance_kwargs()`.
+
+## Key Pattern: Lazy Import for Dependency Isolation
+
+The `_get_instance_class()` classmethod must use a lazy import so that heavy dependencies are only loaded inside the runner environment, not on the host.
+
+```python
+@classmethod
+def _get_instance_class(cls):
+ from .instance import MyAgentInstance
+
+ return MyAgentInstance
+```
+
+This is the same pattern that `Benchmark._get_session_class()` uses. It ensures the host process never imports libraries like `litellm`, `smolagents`, `openai`, or any other agent-specific SDK.
+
+## When to Split Files
+
+**Split into separate files** when your agent depends on heavy third-party libraries:
+
+```
+src/exgentic/agents/my_agent/
+ __init__.py
+ my_agent.py # Agent subclass (light, no heavy imports)
+ instance.py # AgentInstance subclass (imports litellm, openai, etc.)
+ requirements.txt # Agent-specific pip dependencies
+ setup.sh # Optional non-pip setup
+ utils.py # Optional helpers
+```
+
+Examples: `litellm_tool_calling`, `smolagents`, `openai`
+
+**Keep everything in one file** when dependencies are light or already available in the base environment:
+
+```
+src/exgentic/agents/my_agent/
+ __init__.py
+ agent.py # Both Agent and AgentInstance in one file
+```
+
+Example: `cli/claude` (the instance class is in the same file because it only depends on stdlib and core Exgentic types)
+
+The rule is simple: if importing the instance class would pull in packages that are not in the base `exgentic` install, split the files.
+
+## Required Methods
+
+### On the Agent class
+
+#### `_get_instance_class()` (classmethod, abstract)
+
+Returns the `AgentInstance` subclass. Must use a lazy import.
+
+```python
+@classmethod
+def _get_instance_class(cls):
+ from .instance import MyAgentInstance
+
+ return MyAgentInstance
+```
+
+#### `_get_instance_kwargs()` (abstract)
+
+Translates the agent's configuration into constructor kwargs for the instance class. Task, context, and actions are passed separately via `start()`, not through the constructor.
+
+```python
+def _get_instance_kwargs(self, session_id: str) -> dict[str, Any]:
+ return {
+ "session_id": session_id,
+ "model": self.model,
+ "max_steps": self.max_steps,
+ }
+```
+
+The returned dict is passed directly to the instance class constructor. Every key must match a constructor parameter.
+
+### On the AgentInstance class
+
+#### `react(observation) -> Action | None` (abstract)
+
+The core decision loop. Receives an `Observation` (or `None` on the first call) and returns an `Action` to take, or `None` to signal that the agent is done.
+
+```python
+def react(self, observation: Observation | None) -> Action | None:
+ # Process observation, decide next action
+ # Return None when the agent decides it is finished
+ ...
+```
+
+#### `close()` (abstract)
+
+Cleanup resources. Called when the session ends, whether or not the agent finished normally.
+
+```python
+def close(self) -> None:
+ # Release connections, flush logs, etc.
+ pass
+```
+
+#### `start(task, context, actions)` (optional override)
+
+Called after construction but before the first `react()`. Receives the benchmark's task string, context dict, and list of action types. The base implementation stores these as `self.task`, `self.context`, and `self.actions`. Override to perform initialization that depends on these values (e.g., seeding a conversation with the task prompt).
+
+#### `get_cost()` (optional)
+
+Returns a `CostReport` with estimated monetary cost. Default returns an empty report. Override to track API costs.
+
+```python
+def get_cost(self) -> CostReport:
+ return self._cost_data
+```
+
+## Registration
+
+Every agent must be registered in `src/exgentic/interfaces/registry.py` in the `AGENTS` dict.
+
+```python
+AGENTS: dict[str, RegistryEntry] = {
+ # ...existing entries...
+ "my_agent": RegistryEntry(
+ slug_name="my_agent",
+ display_name="My Agent",
+ module="exgentic.agents.my_agent.my_agent",
+ attr="MyAgent",
+ kind="agent",
+ ),
+}
+```
+
+Requirements:
+- `slug_name` must match the `slug_name` ClassVar on the Agent class exactly
+- `display_name` must match the `display_name` ClassVar on the Agent class exactly
+- `module` is the dotted Python module path to the file containing the Agent class
+- `attr` is the class name within that module
+- `kind` must be `"agent"`
+
+The registry validates these constraints at load time. Mismatches will raise at startup.
+
+## Setup
+
+### `requirements.txt`
+
+List agent-specific pip dependencies. The runner installs these automatically into the isolated environment.
+
+```
+litellm>=1.50.0
+```
+
+Place the file in the agent's package directory. The `RunnerMixin` auto-discovers it by walking up from the module file.
+
+### `setup.sh`
+
+Optional script for non-pip setup. Runs after dependencies are installed.
+
+```bash
+#!/usr/bin/env bash
+set -euo pipefail
+# Build Docker images, install npm packages, download models, etc.
+```
+
+Place it next to the agent module. The `RunnerMixin` auto-discovers it.
+
+Both files are automatically found by the framework through `RunnerMixin.requirements_txt` and `RunnerMixin.setup_script`. No manual wiring is needed.
+
+## Recommended File Structure
+
+### Split pattern (heavy deps)
+
+```
+src/exgentic/agents/my_agent/
+ __init__.py
+ my_agent.py # Agent subclass
+ instance.py # AgentInstance subclass
+ requirements.txt # e.g., litellm>=1.50.0
+ setup.sh # optional
+ utils.py # optional helpers
+```
+
+**my_agent.py** (host-side, no heavy imports):
+
+```python
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from ...core.agent import Agent
+from ...core.types import ActionType, ModelSettings
+
+
+class MyAgent(Agent):
+ display_name: ClassVar[str] = "My Agent"
+ slug_name: ClassVar[str] = "my_agent"
+
+ model: str = "gpt-4o"
+ max_steps: int = 100
+
+ @classmethod
+ def _get_instance_class(cls):
+ from .instance import MyAgentInstance
+
+ return MyAgentInstance
+
+ @property
+ def model_name(self) -> str:
+ return self.model
+
+ def _get_instance_kwargs(self, session_id: str) -> dict[str, Any]:
+ return {
+ "session_id": session_id,
+ "model": self.model,
+ "max_steps": self.max_steps,
+ }
+```
+
+**instance.py** (runner-side, may import heavy libs):
+
+```python
+from __future__ import annotations
+
+from typing import Any, Optional
+
+import some_heavy_library # only loaded inside the runner
+
+from ...core.agent_instance import AgentInstance
+from ...core.types import Action, ActionType, Observation
+
+
+class MyAgentInstance(AgentInstance):
+ def __init__(
+ self,
+ session_id: str,
+ task: str,
+ context: dict[str, Any],
+ actions: list[ActionType],
+ model: str,
+ max_steps: int,
+ ):
+ super().__init__(session_id)
+ self.task = task
+ self.context = context
+ self.actions = actions
+ self.model = model
+ self.max_steps = max_steps
+ self._step_count = 0
+
+ def react(self, observation: Optional[Observation]) -> Optional[Action]:
+ self._step_count += 1
+ if self._step_count > self.max_steps:
+ return None
+ # Agent decision logic here
+ ...
+
+ def close(self) -> None:
+ pass
+```
+
+### Light pattern (no heavy deps)
+
+```
+src/exgentic/agents/my_agent/
+ __init__.py
+ agent.py # Both Agent and AgentInstance
+```
+
+Keep both classes in a single file when the instance has no third-party imports beyond what exgentic already provides.
+
+## Validation Checklist
+
+Before opening a PR for a new agent, validate all of the following.
+
+### Contract validation
+
+- Agent class declares `display_name` and `slug_name` as `ClassVar[str]`
+- `_get_instance_class()` uses a lazy import
+- `_get_instance_kwargs()` returns a dict whose keys match the instance constructor
+- The instance implements `react()` and `close()`
+- The instance calls `super().__init__(session_id)` in its constructor
+
+### Dependency isolation validation
+
+- The Agent file does not import heavy third-party libraries at module level
+- Heavy imports only appear inside `_get_instance_class()` or in the instance module
+- `requirements.txt` lists all agent-specific dependencies
+
+### Registry validation
+
+- `slug_name` in the registry entry matches the class `slug_name` exactly
+- `display_name` in the registry entry matches the class `display_name` exactly
+- `module` path resolves to the correct file
+- `attr` matches the Agent class name
+- `kind` is `"agent"`
+
+### Functional validation
+
+- Agent is discoverable through the registry (`load_agent("my_agent")` succeeds)
+- Agent works with at least one benchmark end to end
+- `react()` correctly returns `None` when the agent decides it is done
+- `close()` does not raise
+- `get_cost()` returns a valid `CostReport`
+
+### Quality validation
+
+- `py_compile` passes for all changed Python files
+- `pre-commit` passes for changed files
+- `git diff --check` passes
+
+## Practical Rule of Thumb
+
+When in doubt, ask:
+
+1. Does the agent adapt to the benchmark, or does it require the benchmark to change?
+2. Are heavy dependencies isolated behind a lazy import?
+3. Is the Agent file importable without installing agent-specific packages?
+4. Does `_get_instance_kwargs()` faithfully pass the benchmark contract through?
+5. Will this agent work with benchmarks that have very different action spaces?
+6. Is the configuration surface minimal and explicit?
+
+If the answer to any of those is no, the adapter is probably too coupled or too leaky.
+
+---
+
+## See also
+
+- [Adding Benchmarks](./adding-benchmarks.md) — the other side of the contract
+- [Custom Models](./custom-models.md) — configuring LLM providers and sampling parameters for the `tool_calling` agent
+- [Runners](./runners.md) — how setup.sh and requirements.txt are discovered and executed
+- [Replay Testing](./replay-testing.md) — write end-to-end tests for your agent without API calls
+- [CONTRIBUTING.md](../CONTRIBUTING.md) — PR workflow and legal requirements
+- [docs/](./README.md) — documentation index
diff --git a/labs/AgentStream/exgentic/docs/adding-benchmarks.md b/labs/AgentStream/exgentic/docs/adding-benchmarks.md
new file mode 100644
index 00000000..4fd56287
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/adding-benchmarks.md
@@ -0,0 +1,364 @@
+# Adding Benchmarks
+
+This document defines the benchmark design principles for Exgentic.
+
+It is intentionally opinionated. A benchmark adapter should not just "work"; it should preserve the benchmark's meaning while still fitting Exgentic's agent abstraction cleanly.
+
+Use these existing adapters as reference points:
+- `src/exgentic/benchmarks/tau2/tau2_benchmark.py`
+- `src/exgentic/benchmarks/bfcl/bfcl_benchmark.py`
+
+**Related docs:**
+[docs/](./README.md) · [Adding Agents](./adding-agents.md) · [Runners](./runners.md) · [Replay Testing](./replay-testing.md) · [Output Format](./output-format.md) · [CONTRIBUTING.md](../CONTRIBUTING.md)
+
+## Core Principle
+
+The benchmark owns the contract.
+
+That means the benchmark decides:
+- what the task is
+- what context the agent receives
+- what actions exist
+- how steps progress
+- when a session is finished
+- how scoring works
+
+The agent should adapt to the benchmark contract through Exgentic's normal interfaces. The benchmark should not be shaped around one specific model protocol.
+
+The default goal should be the thinnest possible benchmark wrapper.
+
+That means:
+- reuse the source benchmark wherever possible
+- add only the translation layers that are actually necessary
+- avoid reimplementing benchmark logic unless there is a clear reason
+- avoid introducing runtime behavior that exists only to satisfy one agent or one model protocol
+
+The target is simple:
+- make the benchmark accessible to any Exgentic agent
+- while adding the minimum adapter surface necessary
+- and without clashing with agent-specific assumptions
+
+## Principles
+
+### 1. Keep the agent-facing contract protocol-agnostic
+
+Do not define a benchmark in terms of OpenAI tool calls, raw assistant messages, or any other provider-specific response format.
+
+Define it in terms of:
+- task semantics
+- available actions
+- observations
+- finish conditions
+- score
+
+Protocol-specific translation belongs in adapters, not in the benchmark contract.
+
+Bad:
+- "The model must return all tool calls in one assistant message."
+
+Good:
+- "The task is complete when the required actions have been taken and the benchmark-specific finish condition is met."
+
+### 2. The task should be the real task
+
+`task` should contain the actual task the agent is meant to solve.
+
+Do not wrap the task in fake chat scaffolding unless that scaffolding is genuinely part of the benchmark.
+
+If the benchmark is not about user interaction, do not invent a chat conversation just to make it look conversational.
+
+Bad:
+- synthetic "user" messages when the benchmark is not actually testing user interaction
+- generic wrapper prompts replacing the real benchmark task
+
+Good:
+- the benchmark prompt itself is the `task`
+
+### 3. Context should contain only what the agent should know
+
+`context` is not a metadata dump.
+
+It should contain only information that is necessary for the agent to behave correctly on the task.
+
+Keep internal benchmark metadata out of `context`, including:
+- subset names
+- source dataset ids
+- registry information
+- adapter implementation details
+
+Good context:
+- policy text
+- execution constraints
+- information the agent genuinely needs to act correctly
+
+Bad context:
+- `"subset": "live_parallel_multiple"`
+- `"benchmark": "bfcl"`
+
+### 4. Actions should represent semantic operations
+
+Actions are the benchmark's action space.
+
+Name and describe them in terms of what they do, not in terms of a transport protocol.
+
+Prefer "actions" over protocol-specific terms like "tool calls" in benchmark-facing language, because not all agents consume or produce actions through the same protocol.
+
+If the source benchmark exposes functions, commands, or tools, translate those into Exgentic actions at the boundary.
+
+### 5. Use `finish` only as part of the benchmark contract
+
+`finish` is valid when the benchmark needs an explicit end-of-step or end-of-task signal.
+
+It should exist because the benchmark contract needs it, not because a specific model API needs it.
+
+Use it when:
+- the benchmark has multiple steps or turns and needs an explicit transition point
+- the benchmark needs a clear "done with this step" signal
+- the benchmark should allow completion without another normal action
+
+Do not force `finish` into a benchmark if the source benchmark's semantics are cleaner without it.
+
+### 6. Distinguish execution modes by contract, not by protocol
+
+If a benchmark has single-turn, live, or multi-turn variants, define those as execution contracts.
+
+The important differences are things like:
+- whether more steps may follow
+- whether the task ends after the current finish
+- whether action outputs affect later state
+- whether the benchmark continues after a step completes
+
+Do not define the mode in terms of how many assistant messages or tool-call payloads a model is allowed to emit.
+
+Important:
+- single-turn does not necessarily mean a single action
+- multi-action single-turn tasks are valid
+- the distinction is about step structure, not about one specific model protocol
+
+### 7. Action outputs must be honest
+
+If the benchmark can produce real execution outputs, use them.
+
+If it cannot, do not fabricate realistic outputs that imply more runtime semantics than actually exist.
+
+Be explicit in the contract when actions are only being recorded rather than executed.
+
+Good:
+- real execution results when the source benchmark exposes an official executor
+- `Action recorded.` when there is no real runtime execution for that task family
+
+Bad:
+- made-up outputs that look like real environment state changes when none were actually computed
+
+### 8. Reuse external harnesses as the source of truth where possible
+
+When adapting an external benchmark, prefer to reuse:
+- dataset loading
+- official assets
+- ground-truth files
+- official checkers or scorers
+- official execution helpers
+
+Avoid copying large chunks of benchmark logic into Exgentic if the source repository already provides them.
+
+But there is an important boundary:
+- external harnesses should be the source of truth for benchmark assets and scoring
+- they should not automatically own the Exgentic runtime contract
+
+If the external harness assumes a model-specific interaction pattern, Exgentic should usually keep its own runtime and bridge to the harness at load/score time instead.
+
+When choosing between two valid integrations, prefer the thinner one.
+
+Use the more complex approach only when the thinner one would:
+- distort benchmark meaning
+- hard-code one agent's assumptions
+- or force Exgentic to own logic that should stay with the source benchmark
+
+### 9. Be explicit about what is official and what is adapted
+
+If the adapter preserves official scoring but changes runtime behavior, document that clearly.
+
+If some subsets use official execution while others only use official scoring, document that too.
+
+Do not imply full equivalence when the integration is intentionally more abstract than the source benchmark.
+
+For each benchmark adapter, it should be easy to answer:
+- What comes directly from the source benchmark?
+- What is adapted by Exgentic?
+- What is exact?
+- What is approximate?
+
+### 10. Success, failure, and error must stay distinct
+
+Finished benchmark failures are not the same as runtime errors.
+
+The adapter should keep these states separate:
+- success: benchmark completed and passed
+- unsuccessful: benchmark completed and failed
+- unfinished: benchmark did not complete
+- error: adapter or runtime failure prevented a proper benchmark result
+
+Do not swallow real errors and report them as ordinary failures.
+
+If an exception happens, record it explicitly in session metadata.
+
+### 11. The benchmark should work for many agents, not just one
+
+A benchmark adapter should not depend on modifying one particular agent implementation.
+
+Prefer to build benchmark logic around Exgentic's shared abstractions:
+- `task`
+- `context`
+- `actions`
+- observations
+- `Session.start()`
+- `Session.step()`
+- `Session.done()`
+- `Session.score()`
+
+If the adapter only works because one agent has special behavior, the adapter is too coupled.
+
+### 12. Keep setup, runtime, and registration separate
+
+A well-structured benchmark adapter usually has three separate concerns:
+
+1. Setup
+- external checkout or installation
+- pinned dependencies
+- benchmark-specific environment preparation
+
+2. Runtime
+- session logic
+- task loading
+- action translation
+- scoring
+
+3. Registration
+- registry entry
+- subset listing
+- CLI discoverability
+
+Do not mix setup logic directly into the runtime path when it can be handled once in `setup.sh`.
+
+### 13. The main benchmark file must not import external dependencies
+
+The main benchmark file (`_benchmark.py`) defines the `Benchmark` subclass that Exgentic loads in the host process. This file **must be importable without any benchmark-specific dependencies installed**.
+
+External dependencies (benchmark harnesses, datasets, ML libraries, etc.) belong in **separate files** that are only loaded inside the runner subprocess through `_get_evaluator_class()` and `_get_session_class()`.
+
+**Rule:** The benchmark class file may only import from:
+- Python standard library
+- `pydantic`
+- `exgentic` core modules
+
+All other imports must live in evaluator/session files that are accessed through the class getters.
+
+**Why:** Exgentic loads the benchmark class in the host process to read configuration (runner type, evaluator/session class names, kwargs). The actual benchmark execution happens inside an isolated runner (venv or Docker). If the main file imports heavy dependencies, the host process fails when those deps are only installed inside the runner environment.
+
+Bad:
+```python
+# _benchmark.py
+from some_harness import HarnessRunner # ← breaks host import
+
+class MyBenchmark(Benchmark):
+ ...
+```
+
+Good:
+```python
+# _benchmark.py — no external deps
+class MyBenchmark(Benchmark):
+ def _get_evaluator_class(self):
+ from ._eval import MyEvaluator # loaded inside runner
+ return MyEvaluator
+
+# _eval.py — external deps are fine here
+from some_harness import HarnessRunner # ← only loaded in runner subprocess
+```
+
+## Required Structure
+
+For a benchmark package under `src/exgentic/benchmarks//`:
+
+- `_benchmark.py` **(required)**
+ - `Benchmark` subclass only
+ - no external dependency imports
+ - `_get_evaluator_class()` and `_get_session_class()` return classes from other files
+- `_eval.py` or `_session.py` **(required if benchmark has external deps)**
+ - evaluator, session, and runtime logic
+ - may import external dependencies at module level
+ - only loaded inside the runner subprocess
+- `setup.sh`
+ - benchmark installation/bootstrap
+- optional shim module
+ - thin import boundary around an external harness
+- optional helper modules
+ - action translation, scoring helpers, data parsing
+
+Then register it in:
+- `src/exgentic/interfaces/registry.py`
+
+## Validation Checklist
+
+Before opening a PR for a new benchmark, validate all of the following.
+
+### Contract validation
+
+- `task` is the actual task, not fake wrapper chat
+- `context` contains only agent-relevant information
+- actions are semantically named
+- `finish` exists only if the benchmark contract needs it
+- success/failure/error semantics are distinct
+
+### Import validation
+
+- the main benchmark file (`_benchmark.py`) imports **no external dependencies**
+- `_get_evaluator_class()` and `_get_session_class()` load from separate files
+- `python -c "from exgentic.benchmarks.._benchmark import "` works without deps installed
+
+### Functional validation
+
+- benchmark is discoverable through the registry
+- subsets list correctly
+- tasks list correctly
+- setup script works from a clean environment
+- at least one happy-path task works end to end
+- benchmark works with the default venv runner (not just direct)
+- at least one failure-path task is represented correctly
+- adapter errors surface as errors, not silent failures
+
+### Source-of-truth validation
+
+- official assets are reused where possible
+- official scoring is reused where possible
+- any remaining deviations from the source benchmark are documented explicitly
+
+### Quality validation
+
+- `py_compile` passes for changed Python files
+- `pre-commit` passes for changed files
+- `git diff --check` passes
+
+## Practical Rule Of Thumb
+
+When in doubt, ask:
+
+1. Is this benchmark contract describing the task, or just mirroring one model API?
+2. Is this information something the agent should truly know?
+3. Is this the thinnest adapter that still preserves the benchmark's meaning?
+4. Am I reusing the source benchmark where it helps, without letting it dictate the wrong runtime shape?
+5. Are the benchmark outputs honest about what was actually executed?
+6. Will this adapter still make sense for a very different kind of Exgentic agent?
+
+If the answer to any of those is no, the adapter is probably too coupled or too misleading.
+
+---
+
+## See also
+
+- [Adding Agents](./adding-agents.md) — the other side of the contract
+- [Runners](./runners.md) — how setup.sh and requirements.txt are discovered and executed
+- [Replay Testing](./replay-testing.md) — write end-to-end tests for your benchmark without API calls
+- [Output Format](./output-format.md) — trajectory.jsonl and results.json schemas
+- [CONTRIBUTING.md](../CONTRIBUTING.md) — PR workflow and legal requirements
+- [docs/](./README.md) — documentation index
diff --git a/labs/AgentStream/exgentic/docs/batch.md b/labs/AgentStream/exgentic/docs/batch.md
new file mode 100644
index 00000000..86e19700
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/batch.md
@@ -0,0 +1,269 @@
+# Batch Runs
+
+The `batch` commands let you manage large evaluations across multiple configurations — parameter sweeps, multi-benchmark comparisons, re-runs of failed sessions — without writing orchestration scripts.
+
+**Related docs:**
+[docs/](./README.md) · [CLI Reference](./cli-reference.md) · [Python API](./python-api.md) · [Output Format](./output-format.md)
+
+---
+
+## When to use batch vs evaluate
+
+| Scenario | Command |
+|----------|---------|
+| Single benchmark run | `exgentic evaluate` |
+| Multiple models on one benchmark | `batch evaluate` with config files |
+| Re-run only failed sessions | `batch evaluate` (skips completed by default) |
+| Sweep temperature/model grids | `batch evaluate` + `batch patch` |
+| Publish results to HuggingFace | `batch publish` |
+| Export results to CSV | `batch extract` |
+
+---
+
+## Config files
+
+Every batch command operates on **config files** — JSON files that describe a run. There are two kinds.
+
+### RunConfig
+
+Describes a full multi-task run.
+
+```json
+{
+ "benchmark": "tau2",
+ "agent": "tool_calling",
+ "subset": "retail",
+ "num_tasks": 10,
+ "model": "gpt-4o",
+ "benchmark_kwargs": {
+ "user_simulator_model": "gpt-4o"
+ },
+ "agent_kwargs": {
+ "model_settings": {
+ "temperature": 0.2
+ }
+ },
+ "output_dir": "./outputs",
+ "max_steps": 100,
+ "max_actions": 100
+}
+```
+
+### SessionConfig
+
+Describes a single task. Used when you need per-task control or when replaying individual sessions.
+
+```json
+{
+ "benchmark": "tau2",
+ "agent": "tool_calling",
+ "task_id": "retail_1",
+ "subset": "retail",
+ "model": "gpt-4o",
+ "output_dir": "./outputs"
+}
+```
+
+### Full field reference
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `benchmark` | string | required | Benchmark slug (e.g. `tau2`) |
+| `agent` | string | required | Agent slug (e.g. `tool_calling`) |
+| `subset` | string | null | Benchmark subset |
+| `task_ids` | list[string] | null | Explicit task IDs to run |
+| `num_tasks` | int | null | Number of tasks (randomly sampled if task_ids not set) |
+| `task_id` | string | required (SessionConfig) | Single task ID |
+| `model` | string | null | Model override |
+| `output_dir` | string | `./outputs` | Where to write results |
+| `cache_dir` | string | null | Cache directory |
+| `run_id` | string | auto | Deterministic ID derived from config |
+| `max_steps` | int | 100 | Steps per session |
+| `max_actions` | int | 100 | Actions per session |
+| `max_workers` | int | null | Parallel session workers |
+| `overwrite_sessions` | bool | false | Re-run already-completed sessions |
+| `benchmark_kwargs` | object | null | Extra kwargs passed to the benchmark |
+| `agent_kwargs` | object | null | Extra kwargs passed to the agent |
+
+---
+
+## Commands
+
+All batch commands accept one or more `--config` flags, each taking a file path or a glob pattern.
+
+```bash
+exgentic batch --config path/to/config.json
+exgentic batch --config "configs/*.json"
+exgentic batch --config configs/run1.json --config configs/run2.json
+```
+
+---
+
+### batch evaluate
+
+Run all configs sequentially, executing sessions and aggregating results.
+
+```bash
+exgentic batch evaluate --config "configs/*.json"
+```
+
+Already-completed sessions are skipped unless `overwrite_sessions` is true in the config. This makes it safe to re-run after partial failures — only missing or failed sessions are executed.
+
+---
+
+### batch execute
+
+Same as `batch evaluate` but skips the aggregation step. Use this when you want to run sessions and aggregate later.
+
+```bash
+exgentic batch execute --config "configs/*.json"
+exgentic batch aggregate --config "configs/*.json" # aggregate afterwards
+```
+
+---
+
+### batch aggregate
+
+Aggregate results from already-completed sessions without running anything.
+
+```bash
+exgentic batch aggregate --config "configs/*.json"
+```
+
+Useful when you have sessions from a previous run and want to recompute scores.
+
+---
+
+### batch status
+
+Print a status table showing completion state for each config.
+
+```bash
+exgentic batch status --config "configs/*.json"
+```
+
+---
+
+### batch prepare
+
+Write session config files to disk without executing. Creates the session directory structure so you can inspect or modify configs before running.
+
+```bash
+exgentic batch prepare --config run.json
+exgentic batch prepare --config run.json --overwrite # overwrite existing session configs
+```
+
+---
+
+### batch patch
+
+Modify existing run or session config files in bulk using dotted-key notation.
+
+```bash
+# Preview what would change
+exgentic batch patch --config "configs/*.json" \
+ --set model=gpt-4o \
+ --dry-run
+
+# Apply changes
+exgentic batch patch --config "configs/*.json" \
+ --set model=gpt-4o \
+ --set agent_kwargs.model_settings.temperature=0.2 \
+ --apply
+```
+
+Dotted paths are resolved into nested dicts. Values are parsed as JSON first; if that fails, treated as strings. This lets you do sweeps:
+
+```bash
+# Change model across a whole grid of configs
+exgentic batch patch --config "sweep_*.json" --set model=claude-3-5-sonnet-20241022 --apply
+```
+
+---
+
+### batch extract
+
+Export results from multiple runs into a single CSV file.
+
+```bash
+exgentic batch extract --config "configs/*.json" --output results.csv
+exgentic batch extract --config "configs/*.json" --output - # print to stdout
+```
+
+Each row is one run. Columns include all `RunResults` fields (see [Output Format](./output-format.md)).
+
+---
+
+### batch publish
+
+Push results to a [HuggingFace dataset](https://huggingface.co/docs/datasets/).
+
+```bash
+exgentic batch publish \
+ --config "configs/*.json" \
+ --repo Exgentic/open-agent-leaderboard-results \
+ --append
+```
+
+Flags:
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--repo` | required | HuggingFace dataset repo ID |
+| `--append` / `--overwrite` | `--append` | Append to or replace the existing dataset |
+| `--private` / `--public` | `--private` | Dataset visibility |
+
+Deduplication: when appending, existing rows with the same `(benchmark, agent, model)` triple are replaced. New combinations are appended.
+
+Requires the `datasets` package (`pip install datasets`) and a HuggingFace token with write access:
+
+```bash
+huggingface-cli login
+# or
+export HF_TOKEN=hf_...
+```
+
+---
+
+## Typical workflows
+
+### Parameter sweep
+
+```bash
+# Create one config per model
+for model in gpt-4o claude-3-5-sonnet-20241022 gemini-2.0-flash; do
+ cp base_config.json "configs/${model}.json"
+ exgentic batch patch --config "configs/${model}.json" --set model=${model} --apply
+done
+
+# Run all
+exgentic batch evaluate --config "configs/*.json"
+
+# Export to CSV
+exgentic batch extract --config "configs/*.json" --output sweep_results.csv
+```
+
+### Resume after partial failure
+
+```bash
+# Just re-run — completed sessions are skipped automatically
+exgentic batch evaluate --config "configs/*.json"
+```
+
+### Separate execute from aggregate
+
+```bash
+# Run sessions in parallel across machines, then aggregate centrally
+exgentic batch execute --config "configs/*.json"
+# ... copy outputs to aggregation machine ...
+exgentic batch aggregate --config "configs/*.json"
+```
+
+---
+
+## See also
+
+- [CLI Reference](./cli-reference.md) — full flag reference for all commands
+- [Output Format](./output-format.md) — RunResults schema, what batch extract produces
+- [Python API](./python-api.md) — programmatic equivalents: `evaluate()`, `execute()`, `aggregate()`
+- [docs/](./README.md) — documentation index
diff --git a/labs/AgentStream/exgentic/docs/cli-reference.md b/labs/AgentStream/exgentic/docs/cli-reference.md
new file mode 100644
index 00000000..2c9235ba
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/cli-reference.md
@@ -0,0 +1,323 @@
+# CLI Reference
+
+Complete reference for all `exgentic` CLI commands.
+
+**Related docs:**
+[docs/](./README.md) · [Python API](./python-api.md) · [Batch Runs](./batch.md) · [Custom Models](./custom-models.md) · [Output Format](./output-format.md)
+
+---
+
+## Global flags
+
+| Flag | Description |
+|------|-------------|
+| `--debug` | Enable debug logging |
+| `--help` | Show help for any command |
+
+---
+
+## Discovery
+
+### list benchmarks
+
+List all available benchmarks.
+
+```bash
+exgentic list benchmarks
+```
+
+### list agents
+
+List all available agents.
+
+```bash
+exgentic list agents
+```
+
+### list subsets
+
+List subsets for a benchmark.
+
+```bash
+exgentic list subsets --benchmark tau2
+```
+
+### list tasks
+
+List task IDs for a benchmark (or subset).
+
+```bash
+exgentic list tasks --benchmark tau2 --subset retail
+exgentic list tasks --benchmark tau2 --subset retail --limit 20
+```
+
+| Flag | Description |
+|------|-------------|
+| `--benchmark` | Benchmark slug (required) |
+| `--subset` | Subset name |
+| `--limit` | Maximum tasks to show |
+
+---
+
+## install
+
+Install a benchmark's or agent's dependencies (default: isolated venv).
+
+```bash
+exgentic install --benchmark tau2 # install deps + data (default: venv)
+exgentic install --agent tool_calling
+exgentic install --benchmark tau2 --force # reinstall even if already set up
+exgentic install --benchmark tau2 --docker # build Docker image
+exgentic install --benchmark tau2 --local # install into local environment
+```
+
+| Flag | Description |
+|------|-------------|
+| `--benchmark` | Benchmark slug |
+| `--agent` | Agent slug |
+| `--force` | Force reinstall |
+| `--docker` | Build a Docker image |
+| `--local` | Install into the local environment instead of an isolated venv |
+
+See [Runners](./runners.md) for details on runner types.
+
+---
+
+## uninstall
+
+Remove an installed benchmark's or agent's environment.
+
+```bash
+exgentic uninstall --benchmark tau2
+exgentic uninstall --agent tool_calling
+```
+
+| Flag | Description |
+|------|-------------|
+| `--benchmark` | Benchmark slug |
+| `--agent` | Agent slug |
+
+---
+
+## setup (deprecated)
+
+> **Deprecated:** `exgentic setup` is an alias for `exgentic install` and will be removed in a future release. Use `install`/`uninstall` instead.
+
+---
+
+## evaluate
+
+Run an evaluation end-to-end: execute sessions and aggregate results.
+
+```bash
+exgentic evaluate \
+ --benchmark tau2 \
+ --agent tool_calling \
+ --subset retail \
+ --num-tasks 10 \
+ --model gpt-4o \
+ --set benchmark.user_simulator_model="gpt-4o"
+```
+
+| Flag | Description |
+|------|-------------|
+| `--benchmark` | Benchmark slug (required) |
+| `--agent` | Agent slug (required) |
+| `--subset` | Benchmark subset |
+| `--task` | One or more specific task IDs (repeatable) |
+| `--num-tasks` | Number of tasks to run |
+| `--model` | Model override |
+| `--max-steps` | Steps per session (default: 100) |
+| `--max-actions` | Actions per session (default: 100) |
+| `--max-workers` | Parallel session workers |
+| `--overwrite` | Re-run already-completed sessions |
+| `--output-dir` | Results output directory (default: `./outputs`) |
+| `--run-id` | Override the auto-generated run ID |
+| `--set KEY=VALUE` | Override any config field (repeatable) |
+| `--debug` | Enable debug logging |
+
+### --set syntax
+
+`--set` accepts dotted key paths and JSON-compatible values:
+
+```bash
+# Benchmark kwargs
+--set benchmark.user_simulator_model="gpt-4o"
+--set benchmark.runner=venv
+
+# Agent kwargs
+--set agent.max_steps=200
+
+# Model settings
+--set agent.model.temperature=0.2
+--set agent.model.max_tokens=4096
+--set agent.model.top_p=0.9
+--set agent.model.reasoning_effort=high
+--set agent.model.num_retries=3
+--set agent.model.retry_after=1.0
+--set agent.model.retry_strategy=constant
+```
+
+---
+
+## status
+
+Show the execution status of a run (how many sessions are done, running, missing).
+
+```bash
+exgentic status --benchmark tau2 --agent tool_calling --subset retail --num-tasks 10
+```
+
+Accepts the same flags as `evaluate`.
+
+---
+
+## preview
+
+Show which tasks would run without executing anything.
+
+```bash
+exgentic preview --benchmark tau2 --agent tool_calling --subset retail --num-tasks 10
+```
+
+Prints a plan showing which sessions would be new, which already exist, and which are currently running.
+
+---
+
+## results
+
+Load and display results from a completed run.
+
+```bash
+exgentic results --benchmark tau2 --agent tool_calling --subset retail --num-tasks 10
+```
+
+Reads `results.json` from the run directory. Accepts the same config flags as `evaluate`.
+
+See [Output Format](./output-format.md) for the full results schema.
+
+---
+
+## compare
+
+Statistical comparison between two run configurations.
+
+```bash
+exgentic compare \
+ --agents tool_calling openai_solo \
+ --benchmark tau2 \
+ --subset retail \
+ --num-tasks 50
+```
+
+Runs a Breslow-Day homogeneity test across subsets and reports whether the difference between agents is statistically significant.
+
+Requires the `analysis` extra:
+
+```bash
+pip install "exgentic[analysis]"
+```
+
+---
+
+## analyze
+
+Generate comparison plots for multiple benchmarks or agents.
+
+```bash
+exgentic analyze \
+ --agents tool_calling openai_solo \
+ --benchmarks tau2 gsm8k \
+ --output report.png
+```
+
+Requires the `analysis` extra:
+
+```bash
+pip install "exgentic[analysis]"
+```
+
+---
+
+## dashboard
+
+Launch the interactive web dashboard.
+
+```bash
+exgentic dashboard
+```
+
+Opens a NiceGUI interface for exploring runs, browsing session trajectories, and monitoring live evaluations.
+
+---
+
+## batch
+
+All batch subcommands. See [Batch Runs](./batch.md) for full documentation.
+
+```bash
+exgentic batch evaluate --config "configs/*.json"
+exgentic batch execute --config "configs/*.json"
+exgentic batch aggregate --config "configs/*.json"
+exgentic batch status --config "configs/*.json"
+exgentic batch prepare --config run.json [--overwrite]
+exgentic batch patch --config "configs/*.json" --set key=value [--apply | --dry-run]
+exgentic batch extract --config "configs/*.json" --output results.csv
+exgentic batch publish --config "configs/*.json" --repo org/dataset [--append | --overwrite] [--private | --public]
+```
+
+---
+
+## Environment variables
+
+Exgentic reads the following environment variables.
+
+### Exgentic settings
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `EXGENTIC_LOG_LEVEL` | `INFO` | Log level: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` |
+| `EXGENTIC_CACHE_DIR` | `.exgentic` | Cache directory for venvs and setup state |
+| `EXGENTIC_DOTENV_PATH` | `.env` | Path to `.env` file loaded automatically |
+| `EXGENTIC_OTEL_ENABLED` | `false` | Enable OpenTelemetry tracing |
+| `EXGENTIC_OTEL_RECORD_CONTENT` | `false` | Include prompts/responses in traces (opt-in) |
+| `EXGENTIC_LITELLM_CACHING` | `true` | Enable LiteLLM response caching |
+| `EXGENTIC_LITELLM_CACHE_DIR` | `~/.cache/exgentic/litellm` | LiteLLM cache directory |
+| `EXGENTIC_LITELLM_LOG_LEVEL` | `WARNING` | LiteLLM internal log level |
+
+### LLM provider credentials
+
+| Variable | Provider |
+|----------|----------|
+| `OPENAI_API_KEY` | OpenAI |
+| `ANTHROPIC_API_KEY` | Anthropic |
+| `AZURE_API_KEY` | Azure OpenAI |
+| `AZURE_API_BASE` | Azure OpenAI endpoint |
+| `AZURE_API_VERSION` | Azure OpenAI API version |
+| `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | AWS Bedrock |
+| `AWS_REGION_NAME` | AWS Bedrock region |
+| `VERTEXAI_PROJECT` / `VERTEXAI_LOCATION` | Google Vertex AI |
+| `OPENAI_API_BASE` | Custom OpenAI-compatible endpoint |
+
+See [Custom Models](./custom-models.md) for full provider setup instructions.
+
+### OpenTelemetry
+
+| Variable | Description |
+|----------|-------------|
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP collector endpoint |
+| `OTEL_EXPORTER_OTLP_PROTOCOL` | `http/protobuf` or `grpc` |
+
+See [Observability Quick Start](./observability/quickstart.md) for tracing setup.
+
+---
+
+## See also
+
+- [Python API](./python-api.md) — programmatic equivalents of all CLI commands
+- [Batch Runs](./batch.md) — detailed guide for batch commands
+- [Custom Models](./custom-models.md) — LLM provider and `--set agent.model.*` reference
+- [Runners](./runners.md) — `--set benchmark.runner=*` options
+- [Output Format](./output-format.md) — what `results` and `extract` produce
+- [Observability Quick Start](./observability/quickstart.md) — tracing setup
+- [docs/](./README.md) — documentation index
diff --git a/labs/AgentStream/exgentic/docs/custom-models.md b/labs/AgentStream/exgentic/docs/custom-models.md
new file mode 100644
index 00000000..f81be751
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/custom-models.md
@@ -0,0 +1,250 @@
+# Custom Models
+
+Exgentic routes all LLM calls through [LiteLLM](https://docs.litellm.ai/), which means any provider or deployment LiteLLM supports works out of the box — no code changes required. You pick the model, supply credentials, and optionally tune sampling parameters.
+
+**Related docs:**
+[docs/](./README.md) · [CLI Reference](./cli-reference.md) · [Python API](./python-api.md) · [Adding Agents](./adding-agents.md) · [Observability Quick Start](./observability/quickstart.md)
+
+---
+
+## Model string format
+
+The `--model` flag (and the `model` parameter in the Python API) accepts any model string that LiteLLM recognises. The general pattern is:
+
+```
+/
+```
+
+For OpenAI-native models the provider prefix is optional:
+
+```bash
+# These are equivalent
+--model gpt-4o
+--model openai/gpt-4o
+```
+
+For every other provider the prefix is required. See the provider examples below.
+
+---
+
+## Supported providers
+
+### OpenAI
+
+```bash
+exgentic evaluate --benchmark gsm8k --agent tool_calling \
+ --model gpt-4o
+```
+
+Required environment variable:
+
+```bash
+export OPENAI_API_KEY=sk-...
+```
+
+### Anthropic
+
+```bash
+exgentic evaluate --benchmark gsm8k --agent tool_calling \
+ --model claude-3-5-sonnet-20241022
+```
+
+Required environment variable:
+
+```bash
+export ANTHROPIC_API_KEY=sk-ant-...
+```
+
+### Azure OpenAI
+
+```bash
+exgentic evaluate --benchmark gsm8k --agent tool_calling \
+ --model azure/
+```
+
+Required environment variables:
+
+```bash
+export AZURE_API_KEY=...
+export AZURE_API_BASE=https://.openai.azure.com
+export AZURE_API_VERSION=2024-02-01 # or whichever version your deployment uses
+```
+
+### AWS Bedrock
+
+```bash
+exgentic evaluate --benchmark gsm8k --agent tool_calling \
+ --model bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
+```
+
+Required environment variables:
+
+```bash
+export AWS_ACCESS_KEY_ID=...
+export AWS_SECRET_ACCESS_KEY=...
+export AWS_REGION_NAME=us-east-1
+```
+
+### Google Vertex AI
+
+```bash
+exgentic evaluate --benchmark gsm8k --agent tool_calling \
+ --model vertex_ai/gemini-1.5-pro
+```
+
+Required environment variables:
+
+```bash
+export VERTEXAI_PROJECT=my-gcp-project
+export VERTEXAI_LOCATION=us-central1
+```
+
+### Ollama (local)
+
+```bash
+exgentic evaluate --benchmark gsm8k --agent tool_calling \
+ --model ollama/llama3
+```
+
+Required: Ollama running locally. Set the base URL if it differs from the default:
+
+```bash
+export OLLAMA_API_BASE=http://localhost:11434 # default; only needed if different
+```
+
+### Any OpenAI-compatible endpoint
+
+```bash
+exgentic evaluate --benchmark gsm8k --agent tool_calling \
+ --model openai/
+```
+
+Override the base URL:
+
+```bash
+export OPENAI_API_BASE=http://localhost:8080/v1
+export OPENAI_API_KEY=any-non-empty-string # required by the client even if unused
+```
+
+This works with vLLM, LM Studio, LocalAI, Together AI, Fireworks, Anyscale, and any other OpenAI-compatible server.
+
+### LiteLLM proxy
+
+If you run a [LiteLLM proxy server](https://docs.litellm.ai/docs/proxy/quick_start) in front of your models:
+
+```bash
+exgentic evaluate --benchmark gsm8k --agent tool_calling \
+ --model openai/
+```
+
+```bash
+export OPENAI_API_BASE=http://localhost:4000
+export OPENAI_API_KEY=
+```
+
+---
+
+## Sampling parameters
+
+Use `--set agent.model.*` to control sampling. These map to `ModelSettings` and are forwarded to LiteLLM on every completion call.
+
+| Flag | Type | Default | Description |
+|------|------|---------|-------------|
+| `--set agent.model.temperature` | float ≥ 0 | `1.0` | Sampling temperature |
+| `--set agent.model.top_p` | float 0–1 | `null` | Nucleus sampling |
+| `--set agent.model.max_tokens` | int ≥ 0 | `null` | Maximum tokens in response |
+| `--set agent.model.reasoning_effort` | string | `null` | Reasoning effort level (o1/o3 models) |
+| `--set agent.model.num_retries` | int ≥ 0 | `5` | Retries on transient errors |
+| `--set agent.model.retry_after` | float ≥ 0 | `0.5` | Initial retry delay in seconds |
+| `--set agent.model.retry_strategy` | string | `exponential_backoff` | `exponential_backoff` or `constant` |
+
+Example — lower temperature and capped output:
+
+```bash
+exgentic evaluate --benchmark tau2 --agent tool_calling \
+ --model gpt-4o \
+ --set benchmark.user_simulator_model="gpt-4o" \
+ --set agent.model.temperature=0.2 \
+ --set agent.model.max_tokens=2048
+```
+
+---
+
+## Python API
+
+```python
+from exgentic import evaluate
+from exgentic.core.types import ModelSettings
+
+results = evaluate(
+ benchmark="tau2",
+ agent="tool_calling",
+ subset="retail",
+ num_tasks=5,
+ model="azure/my-gpt-4o-deployment",
+ agent_kwargs={
+ "model_settings": ModelSettings(
+ temperature=0.2,
+ max_tokens=2048,
+ num_retries=3,
+ )
+ },
+ benchmark_kwargs={"user_simulator_model": "gpt-4o"},
+)
+```
+
+---
+
+## Reasoning models
+
+For models that support reasoning effort (OpenAI o1, o3, etc.):
+
+```bash
+exgentic evaluate --benchmark swebench --agent tool_calling \
+ --model o3 \
+ --set agent.model.reasoning_effort=high \
+ --set agent.model.max_tokens=32768
+```
+
+Note: temperature is typically fixed at 1 for reasoning models and will be ignored if set.
+
+---
+
+## Cost tracking
+
+Exgentic records token counts and estimated cost for every LiteLLM completion automatically. Results appear in:
+
+- `outputs//results.json` — aggregate cost across all sessions
+- `outputs//sessions//results.json` — per-session cost
+
+Cost estimates are calculated using LiteLLM's built-in pricing database. For providers or custom deployments not in the database, cost will show as `0`.
+
+---
+
+## Caching
+
+LiteLLM-level response caching is enabled by default. To disable it for a run:
+
+```bash
+export EXGENTIC_LITELLM_CACHING=false
+```
+
+The cache directory defaults to `.litellm_cache` in the working directory. To move it:
+
+```bash
+export EXGENTIC_LITELLM_CACHE_DIR=/path/to/cache
+```
+
+---
+
+## Observability
+
+All LLM inference calls emit OpenTelemetry spans automatically when tracing is enabled. See [Observability Quick Start](./observability/quickstart.md) to set up tracing, and [Semantic Conventions](./observability/semantic-conventions.md) for the full attribute reference.
+
+---
+
+## Further reading
+
+- [LiteLLM providers documentation](https://docs.litellm.ai/docs/providers)
+- [Adding a new agent adapter](./adding-agents.md) — relevant when wrapping a framework that manages its own LLM calls
+- [Observability Quick Start](./observability/quickstart.md)
diff --git a/labs/AgentStream/exgentic/docs/huggingface.md b/labs/AgentStream/exgentic/docs/huggingface.md
new file mode 100644
index 00000000..05ce9104
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/huggingface.md
@@ -0,0 +1,35 @@
+# HuggingFace
+
+## Using HuggingFace Models
+
+Set your HF token and use the `huggingface///` model string format:
+
+```bash
+export HF_TOKEN=hf_...
+```
+
+```bash
+exgentic evaluate \
+ --benchmark gsm8k \
+ --agent tool_calling \
+ --model huggingface/together/meta-llama/Llama-3.1-70B-Instruct
+```
+
+LiteLLM routes the call through HuggingFace's inference providers (billed to your HF account). Supported providers include `together`, `sambanova`, and others. Tool calling support depends on the provider and model.
+
+## Running on HuggingFace Jobs
+
+HuggingFace Jobs run containerized workloads on HF infrastructure (requires Pro/Team/Enterprise).
+
+```bash
+hf jobs run astral-sh/uv:python3.12-bookworm sh -c "
+ uvx exgentic evaluate \
+ --benchmark gsm8k \
+ --agent tool_calling \
+ --model huggingface/together/meta-llama/Llama-3.1-70B-Instruct \
+ --output-dir /tmp/outputs &&
+ uvx exgentic batch publish --repo-id your-org/eval-results /tmp/outputs
+" --env HF_TOKEN=hf_...
+```
+
+Results are published to `https://huggingface.co/datasets/your-org/eval-results`.
diff --git a/labs/AgentStream/exgentic/docs/observability/quickstart.md b/labs/AgentStream/exgentic/docs/observability/quickstart.md
new file mode 100644
index 00000000..cfe52cde
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/observability/quickstart.md
@@ -0,0 +1,151 @@
+# Observability Quick Start
+
+This guide gets you from zero to traces in five minutes using Jaeger as a local trace collector.
+
+For a full reference of every attribute Exgentic emits, see [Semantic Conventions](./semantic-conventions.md).
+
+---
+
+## Prerequisites
+
+- Docker or Podman installed and running
+- `exgentic` installed with the `otel` extra (see below)
+
+---
+
+## Step 1 — Install the OTEL extra
+
+```bash
+uv sync --extra otel
+```
+
+---
+
+## Step 2 — Start Jaeger
+
+```bash
+# Using Docker (or replace 'docker' with 'podman')
+docker run -d --name jaeger \
+ -e COLLECTOR_OTLP_ENABLED=true \
+ -p 16686:16686 \
+ -p 4317:4317 \
+ -p 4318:4318 \
+ jaegertracing/all-in-one:latest
+```
+
+Default ports:
+
+| Port | Service |
+|-------|--------------|
+| 16686 | Jaeger UI |
+| 4317 | OTLP gRPC |
+| 4318 | OTLP HTTP |
+
+---
+
+## Step 3 — Configure environment variables
+
+```bash
+export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf # or 'grpc' for port 4317
+export EXGENTIC_OTEL_ENABLED=true
+```
+
+To include task prompts, tool arguments, and LLM messages in traces (opt-in — may contain sensitive data):
+
+```bash
+export EXGENTIC_OTEL_RECORD_CONTENT=true
+```
+
+---
+
+## Step 4 — Set up and run an evaluation
+
+```bash
+exgentic install --agent tool_calling
+exgentic install --benchmark tau2
+
+exgentic evaluate \
+ --benchmark tau2 \
+ --agent tool_calling \
+ --model gpt-4o \
+ --set benchmark.user_simulator_model="gpt-4o" \
+ --task 1 \
+ --max-steps 10
+```
+
+---
+
+## Step 5 — View traces
+
+Open [http://localhost:16686](http://localhost:16686), select the `exgentic` service, and click **Find Traces**.
+
+---
+
+## Exporting traces
+
+### Via the Jaeger UI
+
+1. Open a trace.
+2. Click the **JSON** button in the top-right corner.
+
+### Via the API
+
+```bash
+# All recent traces
+curl "http://localhost:16686/api/traces?service=exgentic&limit=100" | jq '.' > traces.json
+
+# A specific trace
+curl "http://localhost:16686/api/traces/" | jq '.' > trace.json
+```
+
+---
+
+## Troubleshooting
+
+### No traces appearing
+
+1. Verify environment variables are set: `env | grep OTEL`
+2. Confirm Jaeger is running: `docker ps | grep jaeger`
+3. Check Jaeger logs: `docker logs jaeger`
+4. Ensure the evaluation completed successfully before looking for traces
+
+### Traces are incomplete
+
+- Wait a few seconds after the evaluation finishes — spans are flushed asynchronously
+- Check session logs for OTEL-related errors
+- Verify network connectivity to Jaeger
+
+### Jaeger not starting
+
+```bash
+# Check for an existing container
+docker ps -a | grep jaeger
+
+# Stop and remove, then restart
+docker stop jaeger && docker rm jaeger
+docker run -d --name jaeger \
+ -e COLLECTOR_OTLP_ENABLED=true \
+ -p 16686:16686 -p 4317:4317 -p 4318:4318 \
+ jaegertracing/all-in-one:latest
+```
+
+---
+
+## Cleanup
+
+```bash
+docker stop jaeger
+docker rm jaeger
+
+# Optional: remove the image
+docker rmi jaegertracing/all-in-one:latest
+```
+
+---
+
+## Further reading
+
+- [Semantic Conventions](./semantic-conventions.md) — full attribute reference
+- [Jaeger documentation](https://www.jaegertracing.io/docs/)
+- [OpenTelemetry GenAI conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/)
diff --git a/labs/AgentStream/exgentic/docs/observability/semantic-conventions.md b/labs/AgentStream/exgentic/docs/observability/semantic-conventions.md
new file mode 100644
index 00000000..92384e25
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/observability/semantic-conventions.md
@@ -0,0 +1,172 @@
+# Semantic Conventions
+
+This document maps Exgentic's core types to [OpenTelemetry GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/). It reflects the actual implementation in `src/exgentic/observers/handlers/otel.py` and `src/exgentic/integrations/litellm/trace_logger.py`.
+
+For setup instructions, see [Quick Start](./quickstart.md).
+
+---
+
+## Span hierarchy
+
+```
+Session Span (ROOT)
+├── execute_tool initial_observation
+├── chat {model} ← LLM inference
+├── execute_tool {tool_name}
+├── chat {model} ← LLM inference
+├── execute_tool {tool_name}
+└── ... ← continues until session ends
+```
+
+---
+
+## Attribute reference
+
+The table below documents every attribute actually emitted by the implementation, organised by span type.
+
+| Span type | OTel attribute | Exgentic source | Type | Requirement | Content-filtered | Notes |
+|-----------|---------------|-----------------|------|-------------|-----------------|-------|
+| **Session (ROOT)** | `exgentic.benchmark.slug_name` | `BenchmarkEntry.slug_name` | string | Custom | No | Heritable |
+| **Session (ROOT)** | `exgentic.benchmark.subset` | `RunConfig.subset` | string | Custom | No | Heritable |
+| **Session (ROOT)** | `exgentic.benchmark.agent.name` | `AgentEntry.display_name` | string | Custom | No | Heritable |
+| **Session (ROOT)** | `exgentic.agent.slug` | `RunConfig.agent` | string | Custom | No | Heritable |
+| **Session (ROOT)** | `exgentic.run.id` | `Context.run_id` | string | Custom | No | Heritable |
+| **Session (ROOT)** | `gen_ai.request.model` | `RunConfig.model` | string | Recommended | No | Heritable; set when model is known at run start |
+| **Session (ROOT)** | `gen_ai.conversation.id` | `Session.session_id` | string | Recommended | No | Heritable; primary correlation attribute |
+| **Session (ROOT)** | `exgentic.session.id` | `Session.session_id` | string | Custom | No | Heritable; kept for backwards compatibility |
+| **Session (ROOT)** | `exgentic.session.task_id` | `Session.task_id` | string | Custom | No | |
+| **Session (ROOT)** | `exgentic.session.task` | `Session.task` | string | Opt-in | **Yes** | Task prompt; requires `EXGENTIC_OTEL_RECORD_CONTENT=true` |
+| **Session (ROOT)** | `exgentic.session.action.{name}.name` | `ActionType.name` | string | Custom | No | One entry per action in `Session.actions` |
+| **Session (ROOT)** | `exgentic.session.action.{name}.description` | `ActionType.description` | string | Custom | No | |
+| **Session (ROOT)** | `exgentic.session.action.{name}.is_message` | `ActionType.is_message` | bool | Custom | No | |
+| **Session (ROOT)** | `exgentic.session.action.{name}.is_finish` | `ActionType.is_finish` | bool | Custom | No | |
+| **Session (ROOT)** | `exgentic.context.{key}` | `Session.context[key]` | string | Custom | No | One entry per context key |
+| **Session (ROOT)** | `exgentic.session.agent.id` | `AgentInstance.agent_id` | string | Custom | No | |
+| **Session (ROOT)** | `exgentic.session.agent.path` | `AgentInstance.paths.agent_dir` | string | Custom | No | |
+| **Session (ROOT)** | `exgentic.score.success` | `SessionScore.success` | bool | Custom | No | Set on session close |
+| **Session (ROOT)** | `exgentic.score` | `SessionScore.score` | float | Custom | No | Set on session close |
+| **Session (ROOT)** | `exgentic.score.is_finished` | `SessionScore.is_finished` | bool | Custom | No | Set on session close |
+| **Session (ROOT)** | `exgentic.session.steps` | step counter | int | Custom | No | Set on session close |
+| **Session (ROOT)** | `exgentic.agent.agent_cost` | `AgentInstance.get_cost()` | string (JSON) | Custom | No | Set on session close |
+| **Session (ROOT)** | `exgentic.session.cost` | `Session.get_cost()` | string (JSON) | Custom | No | Set on session close |
+| **execute_tool** | `gen_ai.operation.name` | `"execute_tool"` | string | Required | No | Constant value |
+| **execute_tool** | `gen_ai.tool.name` | `Action.name` | string | Required | No | |
+| **execute_tool** | `gen_ai.tool.id` | `Action.id` | string | Recommended | No | |
+| **execute_tool** | `gen_ai.tool.description` | `ActionType.description` | string | Recommended | No | Looked up from `Session.actions` |
+| **execute_tool** | `gen_ai.tool.parameters` | `Action.arguments` | string (JSON) | Opt-in | **Yes** | Requires `EXGENTIC_OTEL_RECORD_CONTENT=true` |
+| **execute_tool** | `gen_ai.tool.result` | `Observation` | string | Opt-in | **Yes** | Requires `EXGENTIC_OTEL_RECORD_CONTENT=true` |
+| **execute_tool** | `gen_ai.conversation.id` | `Session.session_id` | string | Recommended | No | Inherited from session span |
+| **LLM inference** | `gen_ai.operation.name` | `"chat"` or `"text_completion"` | string | Required | No | |
+| **LLM inference** | `gen_ai.provider.name` | `litellm_params.custom_llm_provider` | string | Required | No | Mapped to standard provider names |
+| **LLM inference** | `gen_ai.request.model` | `LitellmKwargs.model` | string | Required | No | |
+| **LLM inference** | `error.type` | exception class name | string | Required | No | Set on failure |
+| **LLM inference** | `gen_ai.conversation.id` | `Context.session_id` | string | Recommended | No | |
+| **LLM inference** | `gen_ai.request.max_tokens` | `optional_params.max_tokens` | int | Recommended | No | |
+| **LLM inference** | `gen_ai.request.temperature` | `optional_params.temperature` | float | Recommended | No | |
+| **LLM inference** | `gen_ai.request.top_p` | `optional_params.top_p` | float | Recommended | No | |
+| **LLM inference** | `gen_ai.request.top_k` | `optional_params.top_k` | float | Recommended | No | |
+| **LLM inference** | `gen_ai.request.frequency_penalty` | `optional_params.frequency_penalty` | float | Recommended | No | |
+| **LLM inference** | `gen_ai.request.presence_penalty` | `optional_params.presence_penalty` | float | Recommended | No | |
+| **LLM inference** | `gen_ai.request.stop_sequences` | `optional_params.stop` | string[] | Recommended | No | |
+| **LLM inference** | `gen_ai.request.choice.count` | `optional_params.n` | int | Required | No | Only when `n != 1` |
+| **LLM inference** | `gen_ai.request.seed` | `optional_params.seed` | int | Required | No | |
+| **LLM inference** | `gen_ai.response.id` | `ResponseObject.id` | string | Recommended | No | |
+| **LLM inference** | `gen_ai.response.model` | `ResponseObject.model` | string | Recommended | No | Actual model resolved by the provider |
+| **LLM inference** | `gen_ai.usage.input_tokens` | `usage.prompt_tokens` | int | Recommended | No | |
+| **LLM inference** | `gen_ai.usage.output_tokens` | `usage.completion_tokens` | int | Recommended | No | |
+| **LLM inference** | `gen_ai.response.finish_reasons` | `choices[*].finish_reason` | string[] | Recommended | No | |
+| **LLM inference** | `gen_ai.tool.definitions` | `LitellmKwargs.tools` | string (JSON) | Opt-in | **Yes** | Requires `EXGENTIC_OTEL_RECORD_CONTENT=true` |
+| **LLM inference** | `gen_ai.input.messages` | `LitellmKwargs.messages` | string (JSON) | Opt-in | **Yes** | Requires `EXGENTIC_OTEL_RECORD_CONTENT=true` |
+| **LLM inference** | `gen_ai.output.messages` | `choices[*].message` | string (JSON) | Opt-in | **Yes** | Requires `EXGENTIC_OTEL_RECORD_CONTENT=true` |
+
+---
+
+## Span details
+
+### Session span (ROOT)
+
+- **Name**: `{benchmark_name} {subset} session`
+- **Kind**: `INTERNAL`
+- **Opened**: `OtelTracingObserver.on_session_creation`
+- **Closed**: `OtelTracingObserver.on_session_success` or `on_session_error`
+
+### execute_tool span
+
+- **Name**: `execute_tool {tool_name}` or `execute_tool initial_observation`
+- **Kind**: `CLIENT`
+- **Opened**: `OtelTracingObserver.on_session_start` (initial), `on_react_success`, or `on_react_error`
+- **Closed**: `OtelTracingObserver.on_step_success` or `on_step_error`
+- **Reference**: [OTel GenAI execute_tool span](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#execute-tool-span)
+
+### LLM inference span
+
+- **Name**: `{operation} {model}` (e.g., `chat gpt-4o`)
+- **Kind**: `CLIENT`
+- **Opened/Closed**: `TraceLogger._write_otel` (LiteLLM callback)
+- **Parent**: session span (via OTEL context propagation)
+- **Reference**: [OTel GenAI inference span](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#inference)
+
+---
+
+## Attribute inheritance
+
+The following attributes are set on the session span and automatically propagated to all child spans via `SessionSpanManager.set_heritable_attribute()`:
+
+| Attribute | Source |
+|-----------|--------|
+| `gen_ai.conversation.id` | `Session.session_id` — primary correlation key |
+| `exgentic.session.id` | `Session.session_id` — backwards compatibility |
+| `gen_ai.request.model` | `RunConfig.model` (when available) |
+| `exgentic.run.id` | `Context.run_id` |
+| `exgentic.benchmark.slug_name` | `BenchmarkEntry.slug_name` |
+| `exgentic.benchmark.subset` | `RunConfig.subset` |
+| `exgentic.benchmark.agent.name` | `AgentEntry.display_name` |
+| `exgentic.agent.slug` | `RunConfig.agent` |
+
+---
+
+## Content filtering
+
+Attributes marked **Yes** in the content-filtered column contain user data (prompts, tool arguments, model responses). They are **not recorded by default** and must be explicitly enabled:
+
+```bash
+export EXGENTIC_OTEL_RECORD_CONTENT=true
+```
+
+Attributes that are never filtered include all IDs, names, counters, scores, and static schemas — only runtime user content requires opt-in.
+
+---
+
+## Implementation notes
+
+### Model name resolution
+
+Because `AgentInstance` does not expose model settings, the model name is extracted from `RunConfig` at run start:
+
+```python
+model_name = run_config.model or (run_config.agent_kwargs or {}).get("model")
+```
+
+### Cost attributes
+
+`LiteLLMCostReport` and `UpdatableCostReport` are serialized to JSON strings for OTEL compatibility:
+
+- `exgentic.agent.agent_cost` — agent-level cost report
+- `exgentic.session.cost` — full session cost report
+
+### LLM span parent context
+
+LLM inference spans are created inside the LiteLLM callback and attached to the session span via OTEL context propagation:
+
+1. The session span manager writes the current OTEL context into the `Context` ContextVar via `update_tracing_context()`.
+2. The LiteLLM trace logger reads the OTEL context from that ContextVar.
+3. LLM spans are created with the session span as their parent using `_get_parent_context()`.
+
+---
+
+## References
+
+- [OTel GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/)
+- [execute_tool span spec](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#execute-tool-span)
+- [Inference span spec](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#inference)
+- [Quick Start](./quickstart.md)
diff --git a/labs/AgentStream/exgentic/docs/observers.md b/labs/AgentStream/exgentic/docs/observers.md
new file mode 100644
index 00000000..bd29313b
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/observers.md
@@ -0,0 +1,221 @@
+# Observers and Controllers
+
+Observers let you hook into the evaluation lifecycle to add custom logging, monitoring, alerting, or analytics — without modifying benchmarks or agents. Controllers extend this with the ability to stop a run early.
+
+**Related docs:**
+[docs/](./README.md) · [Python API](./python-api.md) · [Output Format](./output-format.md) · [Observability](./observability/quickstart.md)
+
+---
+
+## Observer interface
+
+All observers extend `exgentic.core.orchestrator.observer.Observer`. Every method has a default no-op implementation, so you only override what you need.
+
+```python
+from exgentic.core.orchestrator.observer import Observer
+
+class Observer:
+ # Run-level callbacks
+ def on_run_start(self, run_config) -> None: ...
+ def on_run_success(self, results, run_config) -> None: ...
+ def on_run_error(self, error) -> None: ...
+
+ # Session-level callbacks
+ def on_session_creation(self, session) -> None: ...
+ def on_session_start(self, session, agent, observation) -> None: ...
+ def on_session_scoring(self, session) -> None: ...
+ def on_session_success(self, session, score, agent) -> None: ...
+ def on_session_error(self, session, error) -> None: ...
+ def on_session_reuse(self, task_result) -> None: ...
+
+ # Step-level: agent.react() returned an action
+ def on_react_success(self, session, action) -> None: ...
+ def on_react_error(self, session, error) -> None: ...
+
+ # Step-level: session.step(action) returned an observation
+ def on_step_success(self, session, observation) -> None: ...
+ def on_step_error(self, session, error) -> None: ...
+```
+
+### Callback parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `run_config` | `RunConfig` | The run configuration |
+| `results` | `RunResults` | Aggregated results (available in `on_run_success`) |
+| `session` | `Session` | Session object with `session_id`, `task_id`, `paths` |
+| `agent` | `Agent` | Agent config object with `get_cost()` |
+| `observation` | `Observation \| None` | Observation returned by the benchmark (None on first step) |
+| `action` | `Action \| None` | Action returned by the agent (None if agent is done) |
+| `score` | `SessionScore` | Score with `score`, `success`, `is_finished`, `session_metrics`, `session_metadata` |
+| `task_result` | `SessionResults` | Results for a session that was skipped/reused from cache |
+| `error` | `Exception` | The exception that occurred |
+
+---
+
+## Using observers
+
+Pass observers to `evaluate()`, `execute()`, or `aggregate()`:
+
+```python
+from exgentic import evaluate
+
+results = evaluate(
+ benchmark="tau2",
+ agent="tool_calling",
+ num_tasks=10,
+ observers=[MyObserver()],
+)
+```
+
+Multiple observers are supported:
+
+```python
+observers=[LoggingObserver(), MetricsObserver(), AlertObserver()]
+```
+
+---
+
+## Examples
+
+### Print session results as they complete
+
+```python
+from exgentic.core.orchestrator.observer import Observer
+
+class PrintObserver(Observer):
+ def on_session_success(self, session, score, agent):
+ status = "PASS" if score.success else "FAIL"
+ print(f"[{status}] {session.task_id} score={score.score:.2f} cost=${agent.get_cost().total_cost:.4f}")
+
+ def on_session_error(self, session, error):
+ print(f"[ERROR] {session.task_id} {type(error).__name__}: {error}")
+```
+
+### Track costs in real time
+
+```python
+from exgentic.core.orchestrator.observer import Observer
+
+class CostTracker(Observer):
+ def __init__(self):
+ self.total_cost = 0.0
+
+ def on_session_success(self, session, score, agent):
+ self.total_cost += agent.get_cost().total_cost
+ print(f"Running total: ${self.total_cost:.4f}")
+```
+
+### Write a custom log file
+
+```python
+import json
+from pathlib import Path
+from exgentic.core.orchestrator.observer import Observer
+
+class JsonlLogger(Observer):
+ def __init__(self, path: str):
+ self.path = Path(path)
+
+ def on_session_success(self, session, score, agent):
+ entry = {
+ "session_id": session.session_id,
+ "task_id": session.task_id,
+ "success": score.success,
+ "score": score.score,
+ }
+ with self.path.open("a") as f:
+ f.write(json.dumps(entry) + "\n")
+```
+
+### Send a Slack alert on failure
+
+```python
+import requests
+from exgentic.core.orchestrator.observer import Observer
+
+class SlackAlerter(Observer):
+ def __init__(self, webhook_url: str):
+ self.webhook_url = webhook_url
+
+ def on_session_error(self, session, error):
+ requests.post(self.webhook_url, json={
+ "text": f":x: Session `{session.task_id}` failed: `{error}`"
+ })
+```
+
+---
+
+## Controllers
+
+Controllers extend observers with the ability to raise errors that stop the run. Use them to implement early stopping — e.g. abort if too many consecutive failures occur.
+
+```python
+from exgentic.core.orchestrator.observer import Observer
+
+class EarlyStopController(Observer):
+ def __init__(self, max_failures: int = 3):
+ self.max_failures = max_failures
+ self.failures = 0
+
+ def on_session_error(self, session, error):
+ self.failures += 1
+ if self.failures >= self.max_failures:
+ raise RuntimeError(f"Stopping: {self.failures} consecutive session errors")
+```
+
+Pass controllers via the `controllers` parameter:
+
+```python
+results = evaluate(
+ benchmark="tau2",
+ agent="tool_calling",
+ num_tasks=50,
+ controllers=[EarlyStopController(max_failures=5)],
+)
+```
+
+---
+
+## Built-in observers
+
+Exgentic uses these observers internally. They run automatically — you do not need to register them.
+
+| Observer | What it does |
+|----------|-------------|
+| `ResultsObserver` | Writes `trajectory.jsonl` and `results.json` for every session; computes `RunResults` |
+| `LoggerObserver` | Logs run progress to the console |
+| `FileLoggerObserver` | Writes `run.log` and per-session logs |
+| `OtelTracingObserver` | Emits OpenTelemetry spans (active when `EXGENTIC_OTEL_ENABLED=true`) |
+| `DashboardEventsObserver` | Streams events to the live dashboard |
+| `WarningsObserver` | Captures and writes `warnings.log` |
+| `RecapObserver` | Prints a summary table at the end of a run |
+
+---
+
+## Thread safety
+
+Observers may be called from multiple threads when `max_workers > 1`. If your observer maintains shared state (counters, file handles, accumulators), protect it with a lock:
+
+```python
+import threading
+from exgentic.core.orchestrator.observer import Observer
+
+class ThreadSafeCounter(Observer):
+ def __init__(self):
+ self._lock = threading.Lock()
+ self.count = 0
+
+ def on_session_success(self, session, score, agent):
+ with self._lock:
+ self.count += 1
+```
+
+---
+
+## See also
+
+- [Python API](./python-api.md) — how to pass observers to `evaluate()`
+- [Observability Quick Start](./observability/quickstart.md) — OpenTelemetry tracing (built-in observer)
+- [Output Format](./output-format.md) — the data that `ResultsObserver` writes
+- [docs/](./README.md) — documentation index
diff --git a/labs/AgentStream/exgentic/docs/output-format.md b/labs/AgentStream/exgentic/docs/output-format.md
new file mode 100644
index 00000000..8d7e9eb7
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/output-format.md
@@ -0,0 +1,325 @@
+# Output Format
+
+Every evaluation writes structured results to a directory under `outputs//`. This document covers the file layout and the schema of every results file.
+
+**Related docs:**
+[docs/](./README.md) · [Python API](./python-api.md) · [Batch Runs](./batch.md) · [CLI Reference](./cli-reference.md)
+
+---
+
+## Directory layout
+
+```
+outputs//
+├── results.json # Aggregated run-level results
+├── benchmark_results.json # Benchmark-specific aggregated results
+├── run/
+│ ├── config.json # Snapshot of RunConfig used for this run
+│ ├── run.log # Main execution log
+│ └── warnings.log # Warnings captured during execution
+└── sessions//
+ ├── config.json # SessionConfig for this task
+ ├── results.json # Session-level results
+ ├── trajectory.jsonl # One JSON line per step (action + observation)
+ ├── agent/
+ │ └── agent.log # Agent execution log
+ └── benchmark/
+ ├── results.json # Benchmark-specific session results
+ └── session.log # Benchmark session log
+```
+
+### run_id and session_id
+
+Both IDs are deterministic SHA256 hashes of the configuration:
+
+- `run_id` — 12-character prefix of the hash of `(benchmark, agent, subset, model, benchmark_kwargs, agent_kwargs)` — everything except the task list.
+- `session_id` — 8-character prefix of the hash of `(benchmark, agent, subset, task_id, model, benchmark_kwargs, agent_kwargs)` — includes the task, so each task gets a stable ID.
+
+This means re-running the same config produces the same IDs. Completed sessions are skipped unless `overwrite_sessions` is set.
+
+---
+
+## results.json
+
+The top-level aggregated results for the entire run.
+
+```json
+{
+ "benchmark_name": "Tau2Bench",
+ "benchmark_slug_name": "tau2",
+ "agent_name": "LiteLLM Tool Calling",
+ "agent_slug_name": "tool_calling",
+ "model_name": "gpt-4o",
+ "model_names": ["gpt-4o"],
+ "subset_name": "retail",
+ "total_sessions": 10,
+ "planned_sessions": 10,
+ "successful_sessions": 7,
+
+ "benchmark_score": 0.72,
+ "average_score": 0.65,
+
+ "average_agent_cost": 0.043,
+ "total_agent_cost": 0.43,
+ "average_benchmark_cost": 0.021,
+ "total_benchmark_cost": 0.21,
+ "total_run_cost": 0.64,
+
+ "average_steps": 12.4,
+ "average_action_count": 15.1,
+ "average_invalid_action_count": 0.8,
+ "average_invalid_action_percent": 5.3,
+
+ "percent_finished": 80.0,
+ "percent_successful": 70.0,
+ "percent_finished_successful": 70.0,
+ "percent_finished_unsuccessful": 10.0,
+ "percent_unfinished": 10.0,
+ "percent_error": 10.0,
+
+ "aggregation_mode": "completed_only",
+ "completed_sessions": 10,
+ "incomplete_sessions": 0,
+ "missing_sessions": 0,
+ "aggregated_session_ids": ["a1b2c3d4", "..."],
+ "skipped_session_ids": [],
+ "skipped_session_reasons": {},
+
+ "exgentic_version": "0.3.0",
+
+ "session_results": [ ... ]
+}
+```
+
+### Field reference
+
+#### Identity
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `benchmark_name` | string | Benchmark display name |
+| `benchmark_slug_name` | string | Benchmark CLI identifier |
+| `agent_name` | string | Agent display name |
+| `agent_slug_name` | string | Agent CLI identifier |
+| `model_name` | string \| null | Primary model used |
+| `model_names` | list[string] \| null | All models used (if multiple) |
+| `subset_name` | string \| null | Benchmark subset |
+| `exgentic_version` | string \| null | Exgentic version that produced these results |
+
+#### Session counts
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `total_sessions` | int | Sessions that were executed |
+| `planned_sessions` | int \| null | Sessions originally planned (from `num_tasks` or `task_ids`) |
+| `successful_sessions` | int | Sessions where `success=true` |
+
+#### Scores
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `benchmark_score` | float \| null | Primary score from `Benchmark.aggregate_sessions()` — benchmark-specific |
+| `average_score` | float \| null | Mean of per-session `score` values |
+
+#### Costs
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `average_agent_cost` | float \| null | Mean agent API cost per session (USD) |
+| `total_agent_cost` | float \| null | Total agent API cost across all sessions |
+| `average_benchmark_cost` | float \| null | Mean benchmark API cost per session (e.g. simulator LLM) |
+| `total_benchmark_cost` | float \| null | Total benchmark API cost |
+| `total_run_cost` | float \| null | Total cost (agent + benchmark) |
+
+#### Performance statistics
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `average_steps` | float \| null | Mean number of steps per session |
+| `average_action_count` | float \| null | Mean number of actions per session |
+| `average_invalid_action_count` | float \| null | Mean invalid actions per session |
+| `average_invalid_action_percent` | float \| null | Invalid actions as percentage of total |
+
+#### Outcome breakdown
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `percent_finished` | float \| null | Sessions that reached a terminal state (success or failure) |
+| `percent_successful` | float \| null | Sessions with `success=true` |
+| `percent_finished_successful` | float \| null | Sessions that finished successfully |
+| `percent_finished_unsuccessful` | float \| null | Sessions that finished unsuccessfully |
+| `percent_unfinished` | float \| null | Sessions that ran out of steps |
+| `percent_error` | float \| null | Sessions that raised an exception |
+
+#### Aggregation provenance
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `aggregation_mode` | string \| null | Always `"completed_only"` — only completed sessions are aggregated |
+| `completed_sessions` | int \| null | Sessions with a `results.json` on disk |
+| `incomplete_sessions` | int \| null | Sessions with a directory but no `results.json` |
+| `missing_sessions` | int \| null | Planned sessions with no directory at all |
+| `aggregated_session_ids` | list[string] \| null | Sessions included in score aggregation |
+| `skipped_session_ids` | list[string] \| null | Sessions excluded from aggregation |
+| `skipped_session_reasons` | dict \| null | Reason per skipped session ID |
+
+---
+
+## sessions//results.json
+
+Per-session results.
+
+```json
+{
+ "session_id": "a1b2c3d4",
+ "task_id": "retail_1",
+ "success": true,
+ "score": 1.0,
+ "is_finished": true,
+ "status": "success",
+ "steps": 14,
+ "action_count": 17,
+ "invalid_action_count": 1,
+ "agent_cost": 0.038,
+ "benchmark_cost": 0.019,
+ "execution_time": 42.3,
+ "details": { ... },
+ "cost_reports": {
+ "agent": { "model_name": "gpt-4o", "input_tokens": 9400, "output_tokens": 820, ... },
+ "benchmark": { ... }
+ }
+}
+```
+
+### Field reference
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `session_id` | string | 8-char deterministic session ID |
+| `task_id` | string \| null | Task identifier from the benchmark |
+| `success` | bool | Whether the session ended successfully |
+| `score` | float \| null | Benchmark-assigned score for this session (0–1 unless benchmark uses a different scale) |
+| `is_finished` | bool \| null | Whether the agent signalled completion (as opposed to hitting a step limit) |
+| `status` | string | Session outcome status (see below) |
+| `steps` | int | Number of (action → observation) steps executed |
+| `action_count` | int | Total individual actions taken |
+| `invalid_action_count` | int | Actions that failed schema or contract validation |
+| `agent_cost` | float | Estimated agent API cost (USD) |
+| `benchmark_cost` | float | Estimated benchmark API cost (USD) |
+| `execution_time` | float | Wall-clock time in seconds |
+| `details` | object | Full `SessionScore` dump — benchmark-specific |
+| `cost_reports` | object | Detailed cost breakdown keyed by `"agent"` and `"benchmark"` |
+
+### Session outcome status
+
+| Value | Meaning |
+|-------|---------|
+| `success` | Session finished and benchmark scored it as successful |
+| `unsuccessful` | Session finished but benchmark scored it as unsuccessful |
+| `unfinished` | Agent never returned `None` — ran out of steps |
+| `limit_reached` | Hit `max_steps` or `max_actions` |
+| `error` | An exception occurred during execution |
+| `cancelled` | Run was cancelled before this session completed |
+| `unknown` | Status could not be determined |
+
+---
+
+## sessions//trajectory.jsonl
+
+A newline-delimited JSON file with one entry per step. Use this to replay or audit what the agent did.
+
+```json
+{"event": "observation", "step": 0, "initial": true, "session_id": "a1b2c3d4", "task_id": "retail_1", "observation": {...}, "action": null}
+{"event": "action", "step": 1, "initial": false, "session_id": "a1b2c3d4", "task_id": "retail_1", "observation": null, "action": {"name": "search_products", "arguments": {...}}}
+{"event": "observation", "step": 1, "initial": false, "session_id": "a1b2c3d4", "task_id": "retail_1", "observation": {"content": [...]}, "action": null}
+...
+```
+
+---
+
+## benchmark_results.json
+
+Benchmark-specific aggregated results produced by `Benchmark.aggregate_sessions()`. Schema is benchmark-defined, but always includes at minimum:
+
+```json
+{
+ "benchmark_name": "Tau2Bench",
+ "total_tasks": 10,
+ "score": 0.72,
+ "metrics": { ... }
+}
+```
+
+---
+
+## cost_reports schema
+
+Within session `results.json`, the `cost_reports` dict contains detailed token and cost breakdowns.
+
+```json
+{
+ "agent": {
+ "model_name": "gpt-4o",
+ "input_tokens": 9400,
+ "output_tokens": 820,
+ "input_cost": 0.0235,
+ "output_cost": 0.0164,
+ "total_cost": 0.0399
+ },
+ "benchmark": {
+ "model_name": "gpt-4o",
+ "input_tokens": 4200,
+ "output_tokens": 340,
+ "input_cost": 0.0105,
+ "output_cost": 0.0068,
+ "total_cost": 0.0173
+ }
+}
+```
+
+Cost estimates come from LiteLLM's pricing database. For providers or deployments not in the database, costs show as `0`.
+
+---
+
+## Reading results programmatically
+
+```python
+import json
+from pathlib import Path
+
+run_dir = Path("outputs/abc123def456")
+
+# Load run-level results
+results = json.loads((run_dir / "results.json").read_text())
+print(f"Score: {results['benchmark_score']}")
+print(f"Sessions: {results['total_sessions']}")
+
+# Load a specific session trajectory
+session_dir = run_dir / "sessions" / "a1b2c3d4"
+trajectory = [
+ json.loads(line)
+ for line in (session_dir / "trajectory.jsonl").read_text().splitlines()
+]
+```
+
+Or use the Python API to load and validate:
+
+```python
+from exgentic import results
+from exgentic.batch import RunConfig
+
+config = RunConfig(benchmark="tau2", agent="tool_calling", subset="retail")
+run_results = results(config)
+print(run_results.benchmark_score)
+```
+
+See [Python API](./python-api.md) for the full API reference.
+
+---
+
+## See also
+
+- [Python API](./python-api.md) — `results()`, `status()`, `aggregate()` functions
+- [Batch Runs](./batch.md) — `batch extract` to export results to CSV
+- [CLI Reference](./cli-reference.md) — `exgentic results` command
+- [docs/](./README.md) — documentation index
diff --git a/labs/AgentStream/exgentic/docs/python-api.md b/labs/AgentStream/exgentic/docs/python-api.md
new file mode 100644
index 00000000..a8dc49b4
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/python-api.md
@@ -0,0 +1,361 @@
+# Python API
+
+Exgentic can be used as a library. The public API is importable directly from the `exgentic` package.
+
+**Related docs:**
+[docs/](./README.md) · [CLI Reference](./cli-reference.md) · [Output Format](./output-format.md) · [Batch Runs](./batch.md) · [Custom Models](./custom-models.md)
+
+---
+
+## Installation
+
+```bash
+uv add exgentic # or: pip install exgentic
+```
+
+---
+
+## Quick example
+
+```python
+from exgentic import evaluate
+
+results = evaluate(
+ benchmark="tau2",
+ agent="tool_calling",
+ subset="retail",
+ num_tasks=5,
+ model="gpt-4o",
+ benchmark_kwargs={"user_simulator_model": "gpt-4o"},
+)
+
+print(results.benchmark_score)
+print(results.total_agent_cost)
+```
+
+---
+
+## Core functions
+
+All functions share the same config parameters. You can pass them as keyword arguments or as a pre-built `RunConfig` object.
+
+### evaluate()
+
+Run sessions and aggregate results. The standard function for most use cases.
+
+```python
+from exgentic import evaluate
+
+results = evaluate(
+ benchmark="tau2",
+ agent="tool_calling",
+ subset="retail",
+ num_tasks=10,
+ model="gpt-4o",
+ benchmark_kwargs={"user_simulator_model": "gpt-4o"},
+ agent_kwargs={"model_settings": {"temperature": 0.2}},
+ max_steps=100,
+ max_actions=100,
+ max_workers=4,
+)
+```
+
+Returns: `RunResults` — see [Output Format](./output-format.md) for the full schema.
+
+### execute()
+
+Run sessions without aggregating results. Use this when you want to separate execution from aggregation (e.g. run on multiple machines, aggregate centrally).
+
+```python
+from exgentic import execute, aggregate
+
+execute(benchmark="tau2", agent="tool_calling", subset="retail", num_tasks=10)
+# ... copy outputs to central machine ...
+results = aggregate(benchmark="tau2", agent="tool_calling", subset="retail", num_tasks=10)
+```
+
+Returns: `RunResults` with aggregation fields empty.
+
+### aggregate()
+
+Aggregate already-completed sessions without running anything. Reads `results.json` from each session directory and computes run-level statistics.
+
+```python
+from exgentic import aggregate
+
+results = aggregate(
+ benchmark="tau2",
+ agent="tool_calling",
+ subset="retail",
+ num_tasks=10,
+)
+```
+
+Returns: `RunResults`.
+
+### status()
+
+Get the current execution status of a run without running anything.
+
+```python
+from exgentic import status
+
+run_status = status(benchmark="tau2", agent="tool_calling", subset="retail", num_tasks=10)
+print(run_status.completed) # number of completed sessions
+print(run_status.running) # number of currently-running sessions
+print(run_status.missing) # number of not-yet-started sessions
+```
+
+Returns: `RunStatus`.
+
+### preview()
+
+Get the execution plan for a run — which sessions would run, which would be reused, etc. — without executing.
+
+```python
+from exgentic import preview
+from exgentic.interfaces.lib.api import RunConfig
+
+config = RunConfig(benchmark="tau2", agent="tool_calling", subset="retail", num_tasks=10)
+plan = preview(config)
+
+print(plan.to_run) # list of session configs that would run
+print(plan.reuse) # list of already-completed sessions
+print(plan.missing) # list of sessions with no output directory
+```
+
+Returns: `RunPlan`.
+
+### results()
+
+Load aggregated results from a completed run's `results.json` on disk.
+
+```python
+from exgentic import results
+from exgentic.interfaces.lib.api import RunConfig
+
+config = RunConfig(benchmark="tau2", agent="tool_calling", subset="retail", num_tasks=10)
+run_results = results(config)
+```
+
+Returns: `RunResults`.
+
+---
+
+## Parameters
+
+All core functions accept the same parameters (as kwargs or as a `RunConfig`/`SessionConfig` object).
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `benchmark` | string \| Benchmark | required | Benchmark slug or instance |
+| `agent` | string \| Agent | required | Agent slug or instance |
+| `subset` | string | null | Benchmark subset |
+| `task_ids` | list[string] | null | Explicit task IDs |
+| `num_tasks` | int | null | Number of tasks to run |
+| `model` | string | null | Model override (forwarded to the agent) |
+| `output_dir` | string | `./outputs` | Results directory |
+| `cache_dir` | string | null | Cache directory |
+| `run_id` | string | auto | Deterministic run ID derived from config |
+| `max_steps` | int | 100 | Maximum steps per session |
+| `max_actions` | int | 100 | Maximum actions per session |
+| `max_workers` | int | null | Parallel session workers |
+| `overwrite_sessions` | bool | false | Re-run completed sessions |
+| `benchmark_kwargs` | dict | null | Extra kwargs for the benchmark constructor |
+| `agent_kwargs` | dict | null | Extra kwargs for the agent constructor |
+| `observers` | list | null | Custom observers (see [Observers](./observers.md)) |
+| `controllers` | list | null | Custom controllers (see [Observers](./observers.md)) |
+
+---
+
+## Discovery functions
+
+### list_benchmarks()
+
+```python
+from exgentic import list_benchmarks
+
+for b in list_benchmarks():
+ print(b["slug_name"], b["display_name"], b["installed"])
+```
+
+Returns: `list[dict]` with keys `slug_name`, `display_name`, `installed`, `installed_at`.
+
+### list_agents()
+
+```python
+from exgentic import list_agents
+
+for a in list_agents():
+ print(a["slug_name"], a["display_name"])
+```
+
+Returns: `list[dict]` with keys `slug_name`, `display_name`, `installed`, `installed_at`.
+
+### list_subsets()
+
+```python
+from exgentic import list_subsets
+
+subsets = list_subsets("tau2")
+# ["retail", "airline", "banking"]
+```
+
+Returns: `list[str]`.
+
+### list_tasks()
+
+```python
+from exgentic import list_tasks
+
+tasks = list_tasks(benchmark="tau2", subset="retail")
+# ["retail_1", "retail_2", ...]
+```
+
+Returns: `list[str]`.
+
+---
+
+## Setup functions
+
+### setup_benchmark()
+
+Install a benchmark's dependencies and run its `setup.sh`. Equivalent to `exgentic install --benchmark `.
+
+```python
+from exgentic.interfaces.lib.api import setup_benchmark
+
+setup_benchmark("tau2")
+setup_benchmark("tau2", force=True) # reinstall even if already set up
+setup_benchmark("tau2", runner="venv") # install into isolated venv
+```
+
+### setup_agent()
+
+```python
+from exgentic.interfaces.lib.api import setup_agent
+
+setup_agent("tool_calling")
+setup_agent("tool_calling", force=True)
+```
+
+---
+
+## Config objects
+
+Use config objects when you want to construct a run programmatically, save configs to disk, or pass them around.
+
+### RunConfig
+
+```python
+from exgentic.interfaces.lib.api import RunConfig
+
+config = RunConfig(
+ benchmark="tau2",
+ agent="tool_calling",
+ subset="retail",
+ num_tasks=10,
+ model="gpt-4o",
+ max_steps=100,
+ benchmark_kwargs={"user_simulator_model": "gpt-4o"},
+ agent_kwargs={"model_settings": {"temperature": 0.2}},
+)
+
+# Save to disk
+import json
+Path("my_run.json").write_text(config.model_dump_json(indent=2))
+
+# Load from disk
+config2 = RunConfig.model_validate_json(Path("my_run.json").read_text())
+```
+
+### SessionConfig
+
+For single-task runs:
+
+```python
+from exgentic.interfaces.lib.api import SessionConfig
+
+config = SessionConfig(
+ benchmark="tau2",
+ agent="tool_calling",
+ task_id="retail_1",
+ subset="retail",
+ model="gpt-4o",
+)
+```
+
+---
+
+## Model settings
+
+Pass model settings through `agent_kwargs`:
+
+```python
+from exgentic import evaluate
+from exgentic.core.types import ModelSettings
+
+results = evaluate(
+ benchmark="tau2",
+ agent="tool_calling",
+ subset="retail",
+ num_tasks=5,
+ model="gpt-4o",
+ agent_kwargs={
+ "model_settings": ModelSettings(
+ temperature=0.2,
+ max_tokens=4096,
+ num_retries=3,
+ )
+ },
+)
+```
+
+Or as a plain dict (equivalent):
+
+```python
+agent_kwargs={
+ "model_settings": {
+ "temperature": 0.2,
+ "max_tokens": 4096,
+ "num_retries": 3,
+ }
+}
+```
+
+See [Custom Models](./custom-models.md) for the full `ModelSettings` reference.
+
+---
+
+## Custom observers
+
+Pass observers to receive live callbacks during a run:
+
+```python
+from exgentic import evaluate
+from exgentic.core.orchestrator.observer import Observer
+
+class PrintObserver(Observer):
+ def on_session_success(self, session, score, agent):
+ print(f"Session {session.session_id}: score={score.score}")
+
+results = evaluate(
+ benchmark="tau2",
+ agent="tool_calling",
+ num_tasks=5,
+ observers=[PrintObserver()],
+)
+```
+
+See [Observers](./observers.md) for the full interface reference.
+
+---
+
+## See also
+
+- [Custom Models](./custom-models.md) — LLM provider setup and `ModelSettings`
+- [Output Format](./output-format.md) — `RunResults` and `SessionResults` schema
+- [Observers](./observers.md) — custom event callbacks
+- [Batch Runs](./batch.md) — programmatic equivalents of batch commands
+- [CLI Reference](./cli-reference.md) — CLI alternative to the Python API
+- [docs/](./README.md) — documentation index
diff --git a/labs/AgentStream/exgentic/docs/releasing.md b/labs/AgentStream/exgentic/docs/releasing.md
new file mode 100644
index 00000000..9ef4c0a3
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/releasing.md
@@ -0,0 +1,119 @@
+# Releasing Exgentic
+
+**Related docs:**
+[docs/](./README.md) · [DEVELOPMENT.md](../DEVELOPMENT.md) · [CONTRIBUTING.md](../CONTRIBUTING.md)
+
+## Release model
+
+Exgentic uses Git tags as the single source of truth for released versions.
+There is no manually maintained version string in the source tree.
+
+- A release tag must look like `vX.Y.Z`, for example `v0.2.0`
+- Package versions are derived from Git tags via `hatch-vcs`
+- PyPI publishing runs from GitHub Actions only for pushed `v*` tags
+- The publishing workflow verifies that the tagged commit is reachable from `main`
+- GitHub Releases are created manually after PyPI publish succeeds
+
+## One-time repository setup
+
+1. In PyPI, create the `exgentic` project if it does not exist yet.
+2. In PyPI project settings, add a Trusted Publisher for this GitHub repository.
+3. Use these GitHub values when configuring the publisher:
+ - Owner: `Exgentic`
+ - Repository: `exgentic`
+ - Workflow name: `publish-pypi.yml`
+ - Environment name: `pypi`
+4. In GitHub, keep the `pypi` environment enabled for this workflow if you want environment-level protections.
+
+See the official docs for the exact PyPI setup steps:
+- https://docs.pypi.org/trusted-publishers/using-a-publisher/
+- https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
+
+## Release steps
+
+1. Make sure the release commit is already merged to `main`.
+2. Update local refs:
+ ```bash
+ git checkout main
+ git pull --ff-only origin main
+ ```
+3. Create the annotated tag:
+ ```bash
+ scripts/release.sh 0.2.0
+ ```
+4. Push the tag:
+ ```bash
+ git push origin v0.2.0
+ ```
+
+Or do steps 3 and 4 in one command:
+
+```bash
+scripts/release.sh 0.2.0 --push
+```
+
+5. After the PyPI workflow succeeds, create the GitHub Release manually:
+ ```bash
+ gh release create v0.2.0 --generate-notes --title "v0.2.0"
+ ```
+
+## GitHub Release notes
+
+Use generated notes as the base, then edit the release text to keep it short and useful.
+
+The release description should include:
+
+- What changed for users
+- Any packaging, CLI, or behavior changes worth calling out
+- Any migration or upgrade note if behavior changed
+- A short verification note when helpful, for example that the version is on PyPI
+
+Good default structure:
+
+```md
+## Summary
+- Short user-facing change 1
+- Short user-facing change 2
+
+## Notes
+- Optional upgrade or compatibility note
+```
+
+Avoid:
+
+- Raw internal implementation details unless they affect users
+- Huge changelogs pasted into the release body
+- Empty releases with only the tag name when there was a meaningful change
+
+## What happens after the tag is pushed
+
+1. GitHub Actions checks out the tagged commit.
+2. The workflow confirms that commit belongs to `main`.
+3. The package is built from that exact tag.
+4. GitHub exchanges its OIDC identity with PyPI using Trusted Publishing.
+5. The distribution is uploaded to PyPI.
+6. After that succeeds, create the GitHub Release page for the same tag.
+
+## Verifying the release locally
+
+You can inspect the version derived from a tag before pushing:
+
+```bash
+git tag -a v0.2.0 -m "Release v0.2.0"
+python -m build
+```
+
+The built wheel and sdist should report version `0.2.0`.
+If you created a test tag by mistake, delete it locally before pushing:
+
+```bash
+git tag -d v0.2.0
+```
+
+---
+
+## See also
+
+- [DEVELOPMENT.md](../DEVELOPMENT.md) — local setup and testing
+- [CONTRIBUTING.md](../CONTRIBUTING.md) — PR workflow and legal requirements
+- [docs/](./README.md) — documentation index
diff --git a/labs/AgentStream/exgentic/docs/replay-testing.md b/labs/AgentStream/exgentic/docs/replay-testing.md
new file mode 100644
index 00000000..996be3b0
--- /dev/null
+++ b/labs/AgentStream/exgentic/docs/replay-testing.md
@@ -0,0 +1,229 @@
+# Replay Testing
+
+Replay tests let you verify benchmark and agent integration end-to-end without making real API calls or running external services. A recording captures a live session; the test replays it deterministically.
+
+This is the primary testing pattern for contributors adding new benchmarks or agents.
+
+**Related docs:**
+[docs/](./README.md) · [Adding Benchmarks](./adding-benchmarks.md) · [Adding Agents](./adding-agents.md) · [Runners](./runners.md)
+
+---
+
+## How it works
+
+1. Run a live evaluation and capture the session trajectory.
+2. Store the trajectory in `tests/benchmarks/recordings//`.
+3. Write a test that replays the trajectory using `ReplayAgent` and `ReplayBenchmark`.
+4. The test verifies the session ends with the expected score — no network, no API keys, no benchmark installation required.
+
+Replay tests can be parametrized across all runner types (`direct`, `venv`, `docker`) to verify that runner isolation doesn't change session outcomes.
+
+---
+
+## Recording format
+
+Each recording lives in its own directory:
+
+```
+tests/benchmarks/recordings//
+├── recording.json # Metadata: task_id, benchmark slug, expected score
+├── trajectory.jsonl # Recorded actions and observations
+├── session.json # Session manifest
+└── results.json # Recorded session results
+```
+
+### recording.json
+
+```json
+{
+ "task_id": "retail_1",
+ "benchmark_slug": "tau2",
+ "expected_score": 1.0
+}
+```
+
+Set `expected_score` to `null` if you only want to verify that the session completes without asserting on score.
+
+### trajectory.jsonl
+
+Newline-delimited JSON. Each line is one event — either an observation (benchmark → agent) or an action (agent → benchmark):
+
+```json
+{"event": "observation", "step": 0, "initial": true, "session_id": "...", "task_id": "retail_1", "observation": {...}, "action": null}
+{"event": "action", "step": 1, "initial": false, "session_id": "...", "task_id": "retail_1", "observation": null, "action": {"name": "search", "arguments": {...}}}
+{"event": "observation", "step": 1, "initial": false, "session_id": "...", "task_id": "retail_1", "observation": {...}, "action": null}
+```
+
+---
+
+## Creating a recording
+
+Run a live evaluation and save the trajectory. The trajectory file is written automatically to the session output directory:
+
+```
+outputs//sessions//trajectory.jsonl
+outputs//sessions//results.json
+```
+
+Copy the relevant files into your recording directory:
+
+```bash
+mkdir -p tests/benchmarks/recordings/my_benchmark
+
+cp outputs//sessions//trajectory.jsonl \
+ tests/benchmarks/recordings/my_benchmark/
+
+cp outputs//sessions//results.json \
+ tests/benchmarks/recordings/my_benchmark/
+
+# Write recording.json manually
+cat > tests/benchmarks/recordings/my_benchmark/recording.json <=0.1.0
+some-other-dep==1.2.3
+```
+
+Git LFS objects are automatically skipped during install (`GIT_LFS_SKIP_SMUDGE=1`).
+
+### setup.sh
+
+Shell script for setup that can't be expressed as pip packages: cloning repositories, compiling binaries, downloading model weights, etc.
+
+```bash
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Example: clone a dependency
+git clone --depth 1 https://github.com/example/repo /opt/repo
+```
+
+Place it in the same directory as your benchmark module. It runs after `requirements.txt` is installed.
+
+---
+
+## See also
+
+- [Adding Benchmarks](./adding-benchmarks.md) — how benchmarks declare their setup
+- [Adding Agents](./adding-agents.md) — how agents declare their setup
+- [Custom Models](./custom-models.md) — configuring the LLM behind the agent
+- [docs/](./README.md) — documentation index
diff --git a/labs/AgentStream/exgentic/examples/run_appworld.py b/labs/AgentStream/exgentic/examples/run_appworld.py
new file mode 100644
index 00000000..03600313
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/run_appworld.py
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026, The Exgentic organization and its contributors.
+
+## CLI usage:
+# exgentic evaluate --benchmark appworld --agent tool_calling --subset test_normal --num-tasks 3 \
+# --model gpt-4o
+## Python API usage:
+# from exgentic import RunConfig, evaluate
+# evaluate(RunConfig(benchmark="appworld", agent="tool_calling", subset="test_normal", num_tasks=3,
+# model="gpt-4o"))
+## Direct class usage (this script):
+# AppWorldBenchmark + LiteLLMToolCallingAgent
+
+from exgentic import AppWorldBenchmark, LiteLLMToolCallingAgent, evaluate
+
+
+def main() -> None:
+ benchmark = AppWorldBenchmark(subset="test_normal")
+ agent = LiteLLMToolCallingAgent(model="gpt-4o")
+ evaluate(benchmark=benchmark, agent=agent, output_dir="./outputs", num_tasks=3)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/labs/AgentStream/exgentic/examples/run_browsecomp.py b/labs/AgentStream/exgentic/examples/run_browsecomp.py
new file mode 100644
index 00000000..26176885
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/run_browsecomp.py
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026, The Exgentic organization and its contributors.
+
+## CLI usage:
+# exgentic evaluate --benchmark browsecompplus --agent tool_calling --subset main --num-tasks 3 \
+# --model gpt-4o
+## Python API usage:
+# from exgentic import RunConfig, evaluate
+# evaluate(RunConfig(benchmark="browsecompplus", agent="tool_calling", subset="main", num_tasks=3,
+# model="gpt-4o"))
+## Direct class usage (this script):
+# BrowseCompPlusBenchmark + LiteLLMToolCallingAgent
+
+from exgentic import BrowseCompPlusBenchmark, LiteLLMToolCallingAgent, evaluate
+
+
+def main() -> None:
+ benchmark = BrowseCompPlusBenchmark()
+ agent = LiteLLMToolCallingAgent(model="gpt-4o")
+ evaluate(benchmark=benchmark, agent=agent, output_dir="./outputs", num_tasks=3)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/labs/AgentStream/exgentic/examples/run_claude_code_on_gsm8k.py b/labs/AgentStream/exgentic/examples/run_claude_code_on_gsm8k.py
new file mode 100644
index 00000000..6e3cecb8
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/run_claude_code_on_gsm8k.py
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026, The Exgentic organization and its contributors.
+
+## CLI usage:
+# exgentic evaluate --benchmark gsm8k --agent claude_code --num-tasks 1 \
+# --model gpt-4o
+## Python API usage:
+# from exgentic import RunConfig, evaluate
+# evaluate(RunConfig(benchmark="gsm8k", agent="claude_code", num_tasks=1,
+# model="gpt-4o"))
+## Direct class usage (this script):
+# GSM8kBenchmark + ClaudeCodeAgent
+
+from exgentic import ClaudeCodeAgent, GSM8kBenchmark, evaluate
+
+
+def main() -> None:
+ benchmark = GSM8kBenchmark()
+ agent = ClaudeCodeAgent(model="gpt-4o")
+ evaluate(benchmark=benchmark, agent=agent, output_dir="./outputs", num_tasks=1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/labs/AgentStream/exgentic/examples/run_claude_code_on_tau2bench.py b/labs/AgentStream/exgentic/examples/run_claude_code_on_tau2bench.py
new file mode 100644
index 00000000..b8e7b66e
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/run_claude_code_on_tau2bench.py
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026, The Exgentic organization and its contributors.
+
+## CLI usage:
+# exgentic evaluate --benchmark tau2 --agent claude_code --subset telecom --num-tasks 1 \
+# --model gpt-4o --set benchmark.user_simulator_model=gpt-4o
+## Python API usage:
+# from exgentic import RunConfig, evaluate
+# evaluate(RunConfig(benchmark="tau2", agent="claude_code", subset="telecom", num_tasks=1,
+# model="gpt-4o", benchmark_kwargs={"user_simulator_model": "gpt-4o"}))
+## Direct class usage (this script):
+# TAU2Benchmark + ClaudeCodeAgent
+
+from exgentic import ClaudeCodeAgent, TAU2Benchmark, evaluate
+
+
+def main() -> None:
+ benchmark = TAU2Benchmark(subset="telecom", user_simulator_model="gpt-4o")
+ agent = ClaudeCodeAgent(model="gpt-4o")
+ evaluate(benchmark=benchmark, agent=agent, output_dir="./outputs", num_tasks=1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/labs/AgentStream/exgentic/examples/run_cli_agents.py b/labs/AgentStream/exgentic/examples/run_cli_agents.py
new file mode 100644
index 00000000..cbf518c9
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/run_cli_agents.py
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026, The Exgentic organization and its contributors.
+
+## CLI usage:
+# exgentic evaluate --benchmark gsm8k --agent codex_cli --num-tasks 3 \
+# --model gpt-4o
+## Python API usage:
+# from exgentic import RunConfig, evaluate
+# evaluate(RunConfig(benchmark="gsm8k", agent="codex_cli", num_tasks=3,
+# model="gpt-4o"))
+## Direct class usage (this script):
+# GSM8kBenchmark + CodexAgent
+
+from exgentic import CodexAgent, GSM8kBenchmark, evaluate
+
+
+def main() -> None:
+ benchmark = GSM8kBenchmark()
+ agent = CodexAgent(model="gpt-4o")
+ evaluate(benchmark=benchmark, agent=agent, output_dir="./outputs", num_tasks=3)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/labs/AgentStream/exgentic/examples/run_gsm8k.py b/labs/AgentStream/exgentic/examples/run_gsm8k.py
new file mode 100644
index 00000000..93466296
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/run_gsm8k.py
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026, The Exgentic organization and its contributors.
+
+## CLI usage:
+# exgentic evaluate --benchmark gsm8k --agent tool_calling --num-tasks 3 --model gpt-4o
+## Python API usage:
+# from exgentic import RunConfig, evaluate
+# evaluate(RunConfig(benchmark="gsm8k", agent="tool_calling", num_tasks=3,
+# model="gpt-4o"))
+## Direct class usage (this script):
+# GSM8kBenchmark + LiteLLMToolCallingAgent
+
+from exgentic import GSM8kBenchmark, LiteLLMToolCallingAgent, evaluate
+
+
+def main() -> None:
+ benchmark = GSM8kBenchmark()
+ agent = LiteLLMToolCallingAgent(model="gpt-4o")
+ evaluate(benchmark=benchmark, agent=agent, output_dir="./outputs", num_tasks=3)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/labs/AgentStream/exgentic/examples/run_hotpotqa.py b/labs/AgentStream/exgentic/examples/run_hotpotqa.py
new file mode 100644
index 00000000..eccf6563
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/run_hotpotqa.py
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026, The Exgentic organization and its contributors.
+
+## CLI usage:
+# exgentic evaluate --benchmark hotpotqa --agent smolagents_tool --subset distractor --num-tasks 3 \
+# --model gpt-4o --set benchmark.with_search_tools=true
+## Python API usage:
+# from exgentic import RunConfig, evaluate
+# evaluate(RunConfig(benchmark="hotpotqa", agent="smolagents_tool", subset="distractor", num_tasks=3,
+# model="gpt-4o", benchmark_kwargs={"with_search_tools": True}))
+## Direct class usage (this script):
+# HotpotQABenchmark + SmolagentToolCallingAgent
+
+from exgentic import HotpotQABenchmark, SmolagentToolCallingAgent, evaluate
+
+
+def main() -> None:
+ benchmark = HotpotQABenchmark(with_search_tools=True)
+ agent = SmolagentToolCallingAgent(model="gpt-4o")
+ evaluate(benchmark=benchmark, agent=agent, output_dir="./outputs", num_tasks=3)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/labs/AgentStream/exgentic/examples/run_openai_mcp.py b/labs/AgentStream/exgentic/examples/run_openai_mcp.py
new file mode 100644
index 00000000..08a2486b
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/run_openai_mcp.py
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026, The Exgentic organization and its contributors.
+
+## CLI usage:
+# exgentic evaluate --benchmark tau2 --agent openai_solo --subset retail --task 4 \
+# --model gpt-4o --set benchmark.user_simulator_model=gpt-4o
+## Python API usage:
+# from exgentic import RunConfig, evaluate
+# evaluate(RunConfig(benchmark="tau2", agent="openai_solo", subset="retail", task_ids=["4"],
+# model="gpt-4o",
+# benchmark_kwargs={"user_simulator_model": "gpt-4o"}))
+## Direct class usage (this script):
+# TAU2Benchmark + OpenAIMCPAgent
+
+from exgentic import OpenAIMCPAgent, TAU2Benchmark, evaluate
+
+
+def main() -> None:
+ benchmark = TAU2Benchmark(
+ subset="retail",
+ user_simulator_model="gpt-4o",
+ )
+ agent = OpenAIMCPAgent(model="gpt-4o")
+ evaluate(
+ benchmark=benchmark,
+ agent=agent,
+ output_dir="./outputs",
+ task_ids=["4"],
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/labs/AgentStream/exgentic/examples/run_smol.py b/labs/AgentStream/exgentic/examples/run_smol.py
new file mode 100644
index 00000000..2b616b68
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/run_smol.py
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026, The Exgentic organization and its contributors.
+
+## CLI usage:
+# exgentic evaluate --benchmark tau2 --agent smolagents_code --subset retail --num-tasks 30 \
+# --model gpt-4o --set benchmark.user_simulator_model=gpt-4o
+## Python API usage:
+# from exgentic import RunConfig, evaluate
+# evaluate(RunConfig(benchmark="tau2", agent="smolagents_code", subset="retail", num_tasks=30,
+# model="gpt-4o", benchmark_kwargs={"user_simulator_model": "gpt-4o"}))
+## Direct class usage (this script):
+# TAU2Benchmark + SmolagentCodeAgent
+
+from exgentic import SmolagentCodeAgent, TAU2Benchmark, evaluate
+
+
+def main() -> None:
+ benchmark = TAU2Benchmark(subset="retail", user_simulator_model="gpt-4o")
+ agent = SmolagentCodeAgent(model="gpt-4o")
+ evaluate(benchmark=benchmark, agent=agent, output_dir="./outputs", num_tasks=30)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/labs/AgentStream/exgentic/examples/run_swebench.py b/labs/AgentStream/exgentic/examples/run_swebench.py
new file mode 100644
index 00000000..0d7d9492
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/run_swebench.py
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026, The Exgentic organization and its contributors.
+
+## CLI usage:
+# exgentic evaluate --benchmark swebench --agent tool_calling --num-tasks 30 \
+# --model gpt-4o
+## Python API usage:
+# from exgentic import RunConfig, evaluate
+# evaluate(RunConfig(benchmark="swebench", agent="tool_calling", num_tasks=30,
+# model="gpt-4o"))
+## Direct class usage (this script):
+# SWEBenchBenchmark + LiteLLMToolCallingAgent
+
+from exgentic import LiteLLMToolCallingAgent, SWEBenchBenchmark, evaluate
+
+
+def main() -> None:
+ benchmark = SWEBenchBenchmark()
+ agent = LiteLLMToolCallingAgent(model="gpt-4o")
+ evaluate(benchmark=benchmark, agent=agent, output_dir="./outputs", num_tasks=30)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/labs/AgentStream/exgentic/examples/run_taubench.py b/labs/AgentStream/exgentic/examples/run_taubench.py
new file mode 100644
index 00000000..c2a2f1bd
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/run_taubench.py
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026, The Exgentic organization and its contributors.
+
+## CLI usage:
+# exgentic evaluate --benchmark tau2 --agent tool_calling --subset retail --num-tasks 1 \
+# --model gpt-4o --set benchmark.user_simulator_model=gpt-4o
+## Python API usage:
+# from exgentic import RunConfig, evaluate
+# evaluate(RunConfig(benchmark="tau2", agent="tool_calling", subset="retail", num_tasks=1,
+# model="gpt-4o", benchmark_kwargs={"user_simulator_model": "gpt-4o"}))
+## Direct class usage (this script):
+# TAU2Benchmark + LiteLLMToolCallingAgent
+
+from exgentic import LiteLLMToolCallingAgent, TAU2Benchmark, evaluate
+
+
+def main() -> None:
+ benchmark = TAU2Benchmark(subset="retail", user_simulator_model="gpt-4o")
+ agent = LiteLLMToolCallingAgent(model="gpt-4o")
+ evaluate(benchmark=benchmark, agent=agent, output_dir="./outputs", num_tasks=1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/labs/AgentStream/exgentic/examples/simple_test_agent/adapter.py b/labs/AgentStream/exgentic/examples/simple_test_agent/adapter.py
new file mode 100644
index 00000000..1501ea4a
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/simple_test_agent/adapter.py
@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2026, The Exgentic organization and its contributors.
+
+from typing import Any, Dict, List, Optional
+
+from exgentic.core.agent import Agent
+from exgentic.core.agent_instance import AgentInstance
+from exgentic.core.types import ModelSettings
+
+
+class SimpleTestAgentInstance(AgentInstance):
+ """Simple test agent that responds with basic actions."""
+
+ def __init__(self, session_id: str, task: str, context: Dict[str, Any], actions: List[str]):
+ super().__init__(session_id)
+ self.task = task
+ self.context = context or {}
+ self.actions = actions
+ self.step_count = 0
+
+ def react(self, observation: Optional[str]) -> Optional[str]:
+ """React to observation with simple response."""
+ self.step_count += 1
+
+ if observation is None:
+ # First step
+ return f"Starting task: {self.task}"
+
+ # Simple logic: respond a few times then finish
+ if self.step_count <= 2:
+ return f"Responding to: {observation}"
+ # Signal completion
+ return None
+
+ def close(self):
+ pass
+
+
+class SimpleTestAgent(Agent):
+ """Agent factory that creates simple test agents."""
+
+ display_name: str = "Simple Test Agent"
+ slug_name: str = "simple_test"
+
+ def __init__(self, model_settings: ModelSettings | None = None) -> None:
+ if model_settings is not None and not isinstance(model_settings, ModelSettings):
+ raise ValueError("model_settings must be a ModelSettings instance.")
+ self.model_settings = model_settings
+
+ def assign(self, task: str, context: Dict[str, Any], actions: List[str], session_id: str) -> AgentInstance:
+ return SimpleTestAgentInstance(session_id, task, context, actions)
+
+ def get_models_names(self) -> List[str]: # type: ignore[override]
+ return []
diff --git a/labs/AgentStream/exgentic/examples/simple_test_agent/setup.sh b/labs/AgentStream/exgentic/examples/simple_test_agent/setup.sh
new file mode 100644
index 00000000..5ddf8ed1
--- /dev/null
+++ b/labs/AgentStream/exgentic/examples/simple_test_agent/setup.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+# No additional dependencies needed for simple test agent
+echo "Simple test agent - no setup required"
diff --git a/labs/AgentStream/exgentic/misc/assets/cli.png b/labs/AgentStream/exgentic/misc/assets/cli.png
new file mode 100644
index 0000000000000000000000000000000000000000..fb4f5ad1de3f4b8cab4331ffcaa53a0a585db5e4
GIT binary patch
literal 1046661
zcmeFZXIN9+)-G&A6jW@8f`AmG@^Z>YCd
zNL*;kmMwel+`jp6%NC)HEn5T_yLR$F`50&Vl)u~R^H5KFOKI=%IsVBjNAo*R4Ggx(
z@{f0I5!iZg%l2PQ_)9
zx7@jT!z6I)!bFg-H93g9A$Z+haM%7byO!9emd4Cnlh1ZlxR1U+42U{}PF%YKd1mNT
z=fq9P^7%2nmYQ0lI5*dOPbTr>gm*%edi0^L1V^j)PZo1;-uZZDNxez*MEP>K?BHe=
z8veZbR*ulwzq*+ECxk#~BAbY7WFn%Y?KB-*Uv1g8>%{YabCdk;`i+kz^I0X3!_gKO
zDKe`&cI>~tW$VAUxwJRzFWqjQh_S4MMAXOSlnt#`R=r)PwdFUdd0*)+l{DZ$`TaRCx*p4~<+eiN_
zD*xto;p6MtV!Dw(EO8f$1rN_3{PcQXq4|x9*hw-4(|k&h-X0UUMPGQLO*{+mUc|eH~TynYX
zerYyV`LSvRq5^yud|1@THQhG$74wSqi2(8y<1iFm7lliy-8k?E!Pfi`*tKE1nw}zh
z^LMiUo+x_W#Jsw&?2G>EyNJG*h}0%R{7*|?uj{FOHM;tD2O|V66Sq#B6wq~8y&(~!
zi+>>)q$e04tmNrpk)(TAoIW#G*>M%tom>6{GWT^~q$NGj#lrEY!%ce-kJ|Rb@|H>0
z&v=Hc;&IT5|8xQzXKZbmni24tGR^r*?x)%tTR`;#OD(s+#s)hg&fNWSck)sJOoXE^
zj~iC5<-VMR+__VmbR;?4uP3C}#KXywg?fEBX=h}ShKgpqXI_{ymUUdZ`qEsR&UFjm
zVX-Hrn}bD
zNTa3QNxlQ3P|Xy|4ZOa96Ve8f-jm*VVn>+f{b1JSZUV$TnZD-tO6%3Ibj4
z2s;dZ=;Ek*^sGF3!!Npp)?1xX;o!s|sIzUWXNB7?PU3w;2V)QGy8F|QTQxpO?8&ab
z`YH_gIH|csYEYBpKg2i=HKX94RJxG~e)CQ)F1ov})EoG89aA6m{nA^Nv8#)^-{)=f
zS07gi_t8xyR9UFSuDjWdea?D@%Vt=^k^T({cS@R-&Ucj^$p!t#MDEE+Psqn78t5&y
z7c~kACBqRQl*efGvx`DoDbY$Z5gm))JeDN`C)Csd=VPrkxEEHpPKcG~3!9C0>JB>}
zD+WispX#DW{CwM?3JKs9qsSxbtXe?TF%)k7eU0FsG}d(oUt^Wh(?oTDr?HPd3A{T2
za`q!G==!r~1s6&wS`O7ht^44w7xfIJF4N1F;%zw+0*^_?DgJpAHQe6)d+75Rt)LYK
zAd3}8XZXz*m?)6~u}a-GZXy%Zv-S2I<#gkL{V$LpAr3dBO{B#~yWLzg_>fTH&``J8
z(RI^$#}-W?=B3J7R*)&)y|}b+H^`+PqORS(UP_TdiR@c0lg4bC4H0~rI%Q|W?<0@{
zY5D01qT!;KxrIgd?1|8}01HS!^q8iTF!@rYUGR51%Ocs;bMEI~S+PZ*aYh-iGFb4W
zQIgEiS!*SUZ&SmF+8z%=2+TZ-9pd#93~G_bn|ZH5r{j>8zGXm@R!(8)t2WK);-b5h
z^;toj+;lx)jogb6A!J(Gi73FHoUj51hR@6nDf*k(|LpDcalLYkYG8WDlq{M{tYYk|
zXV>Xzo7dYm&5ADyVgTz34dOllp2?_XWkUI~D4Q71OQh3LuK8Wsy!AiMk>xK`L0Dzc
zZ~bnLzB?Sb{nU0!8DiQ9whnUG)jxzkEccgM#dnil-|4tP#3|8kP^(!|NWkc7Wu*J{
zNf7cegfNSMuel0Mrr}NTqj8PM^C9f_bYHZrh}FTYM*XD6Sl?w8_pi-LrVl)X-lowR
z#kpSIwO@B?>@V3;-drxF&=@qOnZ~pm(Loar%|^E+-2q*~vzHExyr|tf-)33dPbC6<
z>ou7}BHHcfk2T$<0OAuBTc3giUo=0j>lw6=F^b7N|g%U@Usk7Uk~`B({AWTLsBK_{3b>
zxI{ZaFT4XvxU1i3QXP<|=|4QY7{!L!P-aEbk~oIdaqpL&v_ws^H%u=3oyg^FQ=&N4TCyD4JcfGxC;`(2}Cq%pH
zZ6VZ9ToJgeM~EPGZ8eMz0a?*ov&o%4npXzqxkK
zFvN2+L(_{UcDPf1SJ&$6@=VP|#?X=&c)2{YV+V2wvhhNRF*1P_hjWg|G1WgjcPSFu
z%3Nz(+zeOrpViJ|fu7JS#YXxg94D_VJ3YNNG%T~3Z7YMgWe3A%g0eM7tS>hOgWiG0
zqI@C|h)Ug#p&O!QARt~^WfhREVP5})C{$0b(_0u1Pb_+a1eTut^pt8LJec~bzkH%`
z(H}nd7hFo(Y^266Y$64|^4{FWpi!1=+I8yS7~{Uj^7ZUYRhV{?uY=Gb_RQp~YB->+
z_xLW(&z~L-c>6|k5(QtvZqf!#dQs2l>bYy>0AlgQ0EVxHF(I{&r+qfGu?G`oyNo&4
z^RaLFA;^RQE)QPaSzLOR(%&LAPv7r@?8op|Ul6PgY^sYu^fjEFck+gd`#49y{@{Ox
z6N`{9*b)PL_R7IG*y%lX%8;+XTA+U)FZouPC2GA
zMT-$x4#SB07FW2ej?(D-YoT9ji+au*N>1wp3QT$F&rwo&<%&Klw2bog9f7dz&@W{I
ztS-(kzO%|L94xg!;XP_*S@$=F1bOXwOnSs
zVfk_^6{3?7mZjogd357jPlQPjIarSY-feKZQ
zm4eMa46}y`!{D=D^K;6T1t21>R)H`}wKR8flL{zR<;b?Ll{1Z@8-7CbrO82lE7Uhj
zbF?;muW``{A8Jr$BiJ?X@{X`1D!xKxR0pjgh4S>oT8eZ$sciL$J?XXBHTUp2q_-rP
zkgJci8YYSlIciV6#m^r3Enw|B;f3F|pW5iEtR(mPchSh_vnSqkQd_5VP*A2YW1;}#
zknac*T8rDa@hR`KlqJ2hO3ROVHL`5!n=#vcv7!h5&`MmpfJx#@?i|#c8as$MMRa-&>>s&Qu($+ZKvRYEI%*`1>r@g>VqWs|L@&rtQ~F5>2j0Kv}-;_UOt>+|ejccq{wH+QV4
ze8Y$An%ePZxfV6RKm*wTp{puZK}DSuSL$=xn;1=N!dTErcQFw=2y;{+T6zZu9iCKd4Db^!Y!SBbCLDiFPl7ZU>xSyD*BeFrj_PotSJhAG8`Wd+S;7N>)g(
z#DgIOx?$^hwyxN&Cu$-%n6;1}kX;_Y#6FEO&mQ|Zuo-+E1cL@U4jYxA;JrIVAELB9
zqyM<@)cx|vCJ5QpdcXC^4jovLt>R<8xu2g-;d!g?MMHzrXiXDw&}|eN+m!=o>V8gu
zH%F)#ndzAoOLw3DAS8hHy*j#-w`^X8)yetsOmkM`lX!n~{MDsBc4ETWtY>mv#o&(S
zy(gbo!QXfU3sDM=<*pWLBcUCA9)XvuW7c9=T1SmRHqoyzMCe2dd|$|*Erh3vFjMf-
z(4crR$9r4)I=*{pd;CJ9=2EjWy&0S2{vQ?lpIrLuouk%akI2quU2evY{a@(*r*ywk
znb%j_O{R+`_V2jub!kepqg9maAE~*&J7}@R#hBdWRd%
z4j+4FX)+}nn^*En@6tzphQ@b)mH&Nkc>VkQffLgZy1U!Z;otcKVMeys?>rER4*rW4
z;kI=DP4#tq69FwvaC({$=)utT9GUce#q$)3KWp4x0H_o%Xpr=C(b#b`Q;<~U$T!^g
zc$MP4!gz>7$BVL#`SNcmoT+G1^)QM3VAo>sKvc@X&z6Rcy46WZrmsQ|&($I_7yZ||
z!zQ)PXzP!t!vV{t>$VF9&f;@URPrD;4d2o#JbS4c09*ke>_nl_qHzXB7!1+Fm~w&O
zv!Ds#^;ykDMqu6Ohdm5O!sDLGePi`zDGis}p?io#+Uv#&nng8IzT{>)vkUAc1g@zv)ThXyxok#fDo+$wzJ4nZy>`
ztpS&hfWDHR(&uWnp^t(5@X~ewk&WJ>R3|2BEhyY?N(dEal1C@lMlr3sippYa1kv$}
zX+c{j%r!jf4?oHRS-1t*La-pScjh`1r_vRRZ;1Mx&WZIlYWTQx+w^Xh72y#+NvY{;
zJvscXAn0r031zQ4^!L4gN~tARd~Oq`?v<8Sna*yTO|DTqhpkz(+ii0~nr+_dE*+#8
zl9688OwN~-m-lPpo=#T(wud|Xd>ijCH3Y?%uu7*E6H{U5Q=C)9>H(xy`^;ANwrF|S
zf1uXmJ>9Kdmrad~j(M?ORBu&y3IU
z#{L*Uhxg$2dl|HNwLTgbg7>v8zSKnfwgM3)Z{{(M@g^T?N;meme!II`J+H4u#*LTL
z(?I)slN6V`NsG1Be((@0m8FCBZ!c}7)6=z)2f}Tf;w>NT>UA}DcT>-Itsx+a3vBGi
zSzhZK`K^yN`EzmyvN}6avHdy7Js)u%_v+N}#kC!bG_X)TQBB~l%
z`M%8)=EO=N9xjdm&L#@FL)egoz79=BXKD7UjW&!gi`v!I2?o&>luBeA8>&75^zbG_
zqeKUFopcA@>YL>^p>_leS=Z{r3T(?7z@IheD%)zxi`?d@vw$??$hMBcq(Nfie7
zBUWLohxHV`m#3WBksZGp*i^>b?KGzB%1XFms;6z&U=(_KxP!1tF2Fu>m&tA=hL`D2
zW*=97=lX=-NM=E0VD4_jqA%-E%m62K=iP$=X>5bw4NLS~Nh0%X-_xV{>Aft7GymIb
z&%_E`0Z&zVcAj^$TGoL}n+;w|vqXkuNJN6)`S%K3o1rXD_guq2jhnb5nGrO?8w?9M
z8ZQobW?keJL+%)<|DLX~5l?FOet3F7s}=fM6OaipKXgx}u0L%|E(7^}T##8_M=TKt;!Pte9GwHlFY>-+yWh
zyXkqz2Ab)A6ZBm3U3!QE_vaA}jRjgb=K)D0WDMBnRzC8f2a)ReD~X{QLI7BB7HXCE
z8({97J{VEcN(?}Q*-%8
zjz#pzxgE#rhl|pd`7?M2w-?7${`4&lJ!v-(8hgjDB_M9DG&%7edaq2!UAu7^-*#RH
zxU-;~`KXSw+ij8}D&7gsY&g`yNikG!k-OT+-QiHJw8*(ueQBVg<3y&Zy2k?N4$QGC
zo!(<2!3C=&0;Q)`U)b{w^;Urg*UmbW#J$1`b89k33)qT#-=A5!%1mFQR(lLs$SmLe&~sT|C6mtV>sw>_+z}Uk
zJl5($diP)mMT|57mKJ(nt+w5`CcT$%elbJXt8f^0X&=_`ZqhgMZD&zD-go&qe2XtD;ldS!
z|6uY;$nwBdcWu~1@O0j>7)7)f)qjNdN*$a&_$kX@y=brIBFBi+(RM8@o)akBIv3RH
znbS3!6)!)XepI{)AMIJjMsh0frai{_!bz=tpoJf|izHQPVTx!1dhPF+&bbmdZG|qG{6nfUBls$CReE+47
zP6LJ?3%}H{#}?_0MucWnIWUc?zIG+_rKJriy+EtBXLQJc(c44@(_X8)2em|$fJ0U;
z4Sl6;FCbR76aI@X0cXSQ#fc@pYE%%Ibte5@_6`S?E__u;iyEmbH8+5
zT;o%fl3Kv=L!(kt`=%UPm!>>ZJOLygT<$6P$pVbOW_-pq`(q3Uvbj(ND~IK1m^+yQ
zfj#lV3oq1xnC;g}9jsxQosX#w<838q25)f4CB(Y&=h$%u5On6Dx*l!`r5Q_Dd@5$gv+TVNX5eSm5O{<>*kL0CWMH>D%4Ba#t3yjh)tdHP^z@l4ydQ-q*t
zv7s^i;h3(wvc?X>g8|1$hTOJU6<_!Fg?1`P$nE}MM5q1n#d+}Fshbfiy+=MD=P>kuQJwC;?tmaMcQ-8+LsRwgTo0{Q23`Z~E1jjcfbZ}P5aG#0t
z)k+0f5oD9Gn4#%Tn%~NBN@s-yAI=1uRl5q~iJ96jwIa~;473V84IRS0MJSt|T@g>|
zx)w~`yg~4rcC7cHH=+WU=1Z8^wQF_EFaB;CuoDfYJ@Ax7&J`)yo2iY^6;%O=;6>El
z_(ECBeMpeF9%m!8)WpidbJ(qMTqW7NDxX#YCNUE)N0i!xG`c^7AgIGDB6PJW0_4U7
zwInui%DEgGxZtNg5PY@46qGdl)U(8UP!dI{ZfZG>M(|=#35Zi5htxHGBCE&g}@;_Y7Fd
zw^AuNCTYoMCd*8TW_pJk~Q8-6OmgYYb|l{VgHclu*tB6
zL6U_Lw)Cts@vloWU#4bEEv-M1is&snhPfKo#)n3qvg4M8!t8mHs+ywn%;b}rZN+<*
z)yw5-7uH@LBSDok_7HMcsO1A~?vo9UaQ}vj?(vy|#K8><|6#I1T2@ZFfW3qt-G4q<
z-!s8nZYF&MCbqUZHLU%TVJXg`6R^IwH@Fz$IwAwgz#D81qH^lYau;_1xDJ!y6n!}AVi`-}FQ
z`dg%S_&LqK{bq)ym37~g{ZM0gvYU-$zt#+yLSAUMD(W+!*7;c!jk?tjoAVxJOR+%$
zSK%dqdiRgbZ@JN_n8W%S&lua;&5PLF6W$ftx0H*&OHZg23a{}~Uz1{uGD{)WVqJ~T
z^pp{Wvf*5eH>RcJPS`FHa_G`~_Tf4czt%eoq&Z8n`vD47iIHZMNZA|5+uLWs|W9biJ3V=+C
zDrJ^g56yGT{YTr&44tf=jQc;lF=;ef;8}OtERUBlrx>GFp%X`X3`)sQx-x%;GU-^Cz(Cyg=XPTZM}Ztr{yD)ZDQHOk
zvWgTVgjx!SfC94e%K_}0KI0lAmEbAzw2$$Kzr|9k(74V
za599qj?(L4PDaV@wJEVt(Rzj&?nP>Xl*NmWU0iZVtd!Xh|S+SxK$5esMr0)Fx~)
ztt5VR?V6>fbS7%=lHi4qwf*WN^$j2>U27pre3{^%444bqqgS
z!XG`Bi1JsVTuYI*Ee=^|_dibO%&vt`yCn%KN!LvdUG|uvTGE$tjKu>i7mSCYAp0JH
zQHidv@x|E-DQ;gQI~TrLVT(?MEL_X!(>1DLp83v)Yx7iM3w{{uyCczPD?>*?cnIz)v4*
zy5U{RUQZWChZ;&oqjNPwt(mh9j(yAhlLAioqGXhN;Pnwr?*B
z>is+*txH7eSpv%CrROt7A7G1R#g_AyNH$qgEUr2g>8KC4Bh19#Lp7HqwV1
zg{sRI;lsk)dsItZt=^4BKbR)71DUuXLE*m^EzYjflSvIqX@K&mJ~I<~r0Q%e2d4&;
zmlox&U0?Y82;0l)9jP5Mp1;>x(O7{D6{+!`0`v3pI?7h|);!5>Sp1rsJQo^j^HS4m
zz=1)9;Kt@=9g1uVVCwf?snGU)EKS~MoXJo6^?FnHR9)OBFK=VYG4+sV@pQ)7mg<@O
z&`I=lut)?Aweson)pzJp0yi&PCd*UZ0T{Z-7O$^Id^DH2Rn+nF8u#hYP$_MGz`JKz
zEUnFmXy|gMRtY_~*8^f(WdsUsIx;luGV$KIQ5*qpP#ZB=pI+^OZey-X7)O>=S6YN0
z@NN8Tc01AR0^`l{S6M|Fa(4?h*ug#M4k++6yM@&lO3xMQ-7qfshY(k_TKPz6vuRsO
zGx~+29pL_9!%K6NO||@XmS>Q-9?d^9+lRY5-~=CwpX?5JmI-?%0Ym89k+bt&XmV9t
zN@+5UeCXW>O$O8;+B3iK8I-{M)i*Pqt_CD;ZY3%v
znE9gg!d2UzLN?PrW{a?7N+s0;WDhfGKVHW62tGN0SL!l3`j~1%O9;MMP#=_K(lm4!
z!|BxInVuxInN%CVVoVD2^t>i-UI#*ShlpW@0`D)GTLpl2IXxK=Chfw|g6QK!uAzW3
z+s~l>;;Ro8E^UIlMeeWfTodk=idg~LzLJ%)xH^3@jgi$`W;h|V2cj+&x+-}#y`!Zp
zQ(;c>XN92v#IoMh5j@pJOO!_r{ao}d%zb3VY%U>q3u3R~CrBQe+a)crDJ3RJ!@R7-
zuhaJKb^sNIkc`KaF)AnPV>!~;X;(bz!k(1Ma`!}KkA`b*63Z}`%O1j_fsOs%xj$!;@+`n?`KMYu$`
zP%Dl8DIzn*s{*3^@lF$Q(o>-?2J%DC#IO2utkVJG}h|@9wn*9<}J;&
za@+2xFreyg4$^w~$d3lkcr=HT6z@Ui)Y>)pu(-%Ra)mVen%d(9Bq)8%{ZsuAi!UAbvrZ?S1K2r}>6fAPmvTIcgM_*HPZw
zGJ&uWu27abrbizua$S|2Km@vgBl190fcGbfGW9nkpb>234r{Xw7ggNpG-^Ou7D{@~l!5?M
z9~B<(L>8lU$OTaNh?P{XEASIrzIS^bZl=j1zxB!>eoL?AvF9q<^_}>u#dM8#2sEHQu!H
zWVI}v*)K;`!xFFm9Jb)+xD?`8@};zR;JIsLY0rcpqi4rw)L@_Pu$0N0E>Wu_ehcCm
zsK|eu@SShZ1dP*~DpujZ6G;EnNl1I<_FWW(})vLz-HGKgqA6XD_1IC~ktn0}f-e
zl;;sCK^m&Yr*3>He;}FFDss%Mt>Byfe9`XwjAHmK;RvYGMk4@Rs6RF!U1ifnzb|fqRM$zkwGCb#}BIGO&)!g~o$i-}1
z&s}19mG0+`LPWdXmsToh<>;Tms~&^+y;Yo#HoKvD;WWT4i~kk%<(_r`h~zhE87&R2
zrFmvaY&9tm*pT_MfMlHi1KVYvsS$GY!5jJuEmOAnddl=zXRWFPB)>r!1a(jFwz%Us
z#Luo=&4GP+4!gIx`l->Lu%VUOGtKt62di4);EIV08q@?{btiAfdjm!LQ7HV>Piac%
z+IS>m90JRvt8HwX7_sTC&fH~Jx%MrUtC!?1BQ2LUCa&{+BHj=AC~?vJZOI!4*3n3v
z9Fa)S2BEP+<tF8jh61`|B#+8Bb{LaVU@iupder|atzMOh+m~q!2_7_kE^-X_@)Z2
zR!MCnsGz!WfNga1R&Ar{*|rD3eMBI`^%}u
zfKvJ^+3UED(HaZUl}5xlNlN}%Q46NidiDIKedEy0#h;@hJ}|_?Buf}SUhNLprg`08skbAZg*%E^oEBr*D~Mxoey_~
zdTm~sk7C|2tR@_fZ*#f|o7>5oEr+K(_>dQOW@=yWhT`SciM1w4VwdgsH
zR^oC8Zl}ZQbdb6B@U0O}{{@pyu&;=A0bVBdssU(Gh=~P%-<|s?s+v)Q+a28`tmv
zUVyeDV;qAA9B*QEMERTm%^Z@trTg-v%D~+P-bKX6zXsVhAOLsGD?{*}IA`>!Pu5dyvcH
zsJX2WRkcTAmJ;;4Pv_cE^u0MgW0ysfqS}#@Pnha0D`PIO#law!x?r|2c|OT8bn%r>
zD06i+-iJG>={UHTk6AyC(zg%)LzE{uEPjys;9?8VEAt--0shs~$1p8SiBg(~PfXpU
zvh_bCtwXJrol?VUT^PirRcdDYI`A)7#?>7tHS*4cy}QGToF?6^?agK!8J
zC+LZjQKiUphkb1-9z|<0IOw%2Yjzc5ox$oK-*k~M>#z|CcGhmOcD?sgVvCi+ig6k$
zwD%6=z;cz!)Q0h1GdrqdBGRAA&!t}f?-Tn(q%yVIaKlx`j-Vl}wK0De<4(mJVupmL
zIs`6fji0k)ukzia4ZVLfisX1wpEbP;^`<%@O6^^&e4z?f1d-VmKVHM9qqQG4*m9dvL=JYwi
z##2$1kqh^yBK>KfAmBSK%_}Y^l|WB%7;XlK8vwHsg&K|aJwhg(U)flPlB#eX?AJDE
zM(c`;rei+*bMZ4fo-K{@ga5Y5EU70X%?dpzg;~7A?g4B9P=Q4E`aYs(ef&ZhkVEDf
z;6q6jNUe2wh%dnm+|o&(9Pr*8XtNC=J24Qz%@+!FtiZ}QCi@H=H>ZXbl=5D*OSv}k
zh5!U<4rQ6L4O$^?XXS+DwDP!Ri(|}(KPk&x7Ob4@5jY5!gX>7l$>p5wv{@dhY~gOq
zERW)dEJ4ar-G|rhmIn-WH@@{BD%r_iL&OurmNt~?C(|UF3k(DOqE)m4Tex^wV0Z9Z
zk!%N8B1>V@C~&HjSml}X*Nx_R@
zmVLBdZOL;O<5^NW^m`*mN9BZY1zpZ2^|#FU7w(cLRtflUO%gkA=q8^a(|GX{XdZ
z6q{m7kzC4~w$9Ygyc_uF+DwFn@2}aZdF&C!)`5zTTh^Itgj8tZ208y!Cpl@tNjqpq
zvH@nSBa**qA6A>;YX_JXvOZl5*r@0*Q{S)z@RjOw;Is6Q#kNF%{j7AyuSSv#YzK2|
z3@}t8=1P|JBdQ)%$z#Ti*JY-jh+ORkCLsMf`F+P`H{NvrQ^U;?y~Dr+)<@{l5UGlw
zGA^dIdnP`c1!bp`LsdW_pV0=-sBXAWPJe-~GU~Dk;5JcZfS*yuibXR*)f=sC)Owdq
za^U56=Z7t;6Z&j(mgPYQ)C*rfvEAyB8{N8LqlP&7Thh&St^idULv#CN>#cIn_N+=}
zf@iDH89?c$(z1DOP^H+|b``Cs)p*e(#@?o|%@56uxCTCBsULzHS;k)-frkrixDrD-
zmDZp@*jyN?3Jc-#sw)jTjMzb(mt;Oo9Pi&!Cjq{VW5$nNG)o#Kh5jJGS1xyPmz;R2
zYabq>yL@50jykQmRHTc~c%p|WH(pLjo)c*StO&O5uJNz}eRl=h9+@xI%t=*^nLP?6DUnYKs@
zS>f!U7+qd8xHn%Q>*eqGVq3N)BcF>Y1}<{2mHBkaLazbSMr|2@3%j~}lJwZ74}r)i
z7DgSQEYyB@z6@dAPEz&*pV`RJ0R{>Eo}gyEdz+LAKKStYJ?s%1JG6YU=e{Jnq|W2j
z{<=agEDb;-JEpOLCNtFUKUPZcLTdUmhY<}`r(W4azf+Vq`AM%ub2~!Usdg47(k&=T
zpT8A+%W^`*j8J--d|G|CRLxxZ{w@4q|BdXf>}!prxJ*@NP3TM0A7{PsCq)pw+4WyA
ztUm0+83K4{EXWoVNgjpkN|fYQ_3vyv@1F9IvzBz*2xG{4T^zWe)P))gz|hTmL1|5wgbSbas{jk@PKYi(-!r0K+wUTw<8yi1rLUvBR+uOCw)IjaJU<=-S#bJ~ZRe9u31>9BHF%{{sCHZ39)wAZ2Zw
zRv+-aKRH{`_qC;f0au_Z!s|9KfKFYK4CQJL10Ua&Ftw=6nv%{mGqt=#*$FbmOc*Lj
zi-1FqQ*NjKmQZ%W%WwDET7bD8?n9~35i2lW1G;TYd&WZD*GBnBDa>}Cc#z1(k@+8x
zC|Q%k7x>(@5hr;Yryjs9%4^2V_Z!j&ep`|B-QjS1cdLhWZAYn1{e@SdZI={sDbbyA
zmUs0k7u9NP#2#CTYH;2KT^}^Ah2veJoGa?3WlO!=JgN^#G_SgHkI
zJb2;3XSDgSiq5`~Q$mYJ=G2@D52~YDSfg@_L%1xb`0>@k**jWJ3+^CUL-uUsUH3MO
z*+O5GO>pmYxsAvfX5ky{soItyyy8dh`Spy{@F6ht&G?A!lg*IwHif6O^sI+N3$&+z
z3JK9?-0{r#qSwgqN2N$C>?>42DKWAjx!3L~AHRh`!Ho}>qkO~`1OXhC@DQa)wbb+H
zQG71y7!JjIiLhvBg3Fn~PkAYvXsEFdlf4Ki%d%C8c58fHqFwuF<~e94fZ9h(k)oac
zy|J9|dJtCA+#t1vxR_r2;{ZsAJ!IhtOD?^84jE~?){)%)G6;aABnK%1?#aBGi7MMnJ+D1QK`1C|FNjo{S?&_9lQRPy_~dw+gl6j=AL3E
zLNooI^`WU<_y+!L1KA6?3I;$ZXRIaR<5sz39}F^xF*!OL_~b{#4d@54I6iT
zm-vH(FH3Ia?WlYuwD;A8@u9EnYXT#4S2bSZdLU;>@Pi&Tc19kqnqb`WdX7?c
zMt|nnxq`8BMZpC@?0PA5mFPM0Iy^~e=?J`eNq+*C+*mesL47#5lf{N;_c2MF4i!)}
zBbi_BfGHCCoCKGpgyEkU6M-UQEb6{24JQDWS)$3ACyfZ|`4|4siA{R{&MFBrY%24z
zc+M?H%nI2lR0*ewR~HG)?LI&!80ecQ4ZrLvH?P$?Y@pJ7IQWBXZREm?4Q{OSE2RXZ
zd@KSKW)H%GV3A}4fF|9fHb`C65uXE-`XGPkfn$*0Wdoej=0tYs*YNqT!G^IclafoV
zhgaw>A(Rd_N0pddiW#*m@Wl?q>y{LwC%Dtu+{SJq%4lLOWujrVx6gl{I4-n&ie4d~0Tr8XKVoA0>vKxg}M8)wf7C(^pQz^y|l-zL}STWp;gKis!~mWRT^LOpIOYYhL9CI|He
zz2Gd8nVB(i#RCC)2Y>D99ksVLcSvk;FAmtZp0hZ@u2>xB!H*oOteI8qTMy%GBBak>
zYIA7DUv4hC-+;EVkYHBVqqLhCYT*r=g3i?VvQa5i4&XveDgtDtf)AcY-GCz||53
zt7}YX#mWy!NF0pe;Y8nP9JFZ3d#g;q-+-dXd#Q@Ao8_It*7{cuHXT!3%D*g>F
z$V;!7=3!<9kFPL@(gCW1=HE#)vzr%MorH<2CwOJ1{=LH`2sDHeh5*m)zsmVK7N(6~)
zaWW~R*Ix~Py(jdnvW8yQ-Y(jVEAds9qW|j2R|~YWC5`oGP0H!|wdFZCB#thaeO5_x
zZy`&BmU_fnMLcD&Gg!fSO7<(XX>OA&pX$}n%9sy
z=yNU&I8LUmqn^bV8FG*NX`lSrA(JgL>VGdODV>0`46t+B<f9el
z#aqHoTyIa7-1>XO&@v6X?}BRADITafu6pMyF-lEJmd@HV4lxWf6TWnDw?R;NB$w+k
z!6++wQEa=R@=&eman-wv@2}k0K3V!m(Vl(Ue%O6jrm@7DrBZ?DM@*C1x9c$?XWVaZ
zU0BLOJt-X48Mlq)Tb;nbh#qwSWO?JaYPxmDrDI!5Q*0H|U&Q^M#`;-C{_??FoR5%a
zX8|N2w^xoAypsMW_WJs|fcBN}c-WO7J8dE|_Xzxlwu^Tyru+5jv$CH*=!+$W+FBEb
zBX|DZUHiib|Dleh&%$z>+r6@s9-5T=*e;a{+WMl)xs$*Cm130H-$&erCP?;?xCYM&
zXr}VJzwG=kCx27^So~Z3{rz+PXIydP#Ib3x2hc@5zQLVO>ald7ZRHofmft*;ld(K=
zXsySm9o?rEY`HMef-R#-7A#s5?oj5z==Hun$M9y`xZg)7|JyK@Ca%=}*KYy*p9Aqa
zCKMQf{&4%)&>;q+iCmK7UnJ=^=GG`hRhqE?{&tF|TL$G5JI8HI7VIfXws5xz{sX
zT^8M{jWK@kSn5ya|9iXthcZ9CzP4^SJW-|Df-uQ`g%AWc9oM5D5R>qkP^Yw4a)J
z!~ID3zb(AypH2AH+=jYQL4Q~r{^n`-r`+I+O7X9r-2X}6*Ze}HqVrco{#~W!Pn_a;
zzNmJVTO9r$^fl&Jgs6+2o&6JR{>`I2+Q~2B={=zw{d>>g?<95ok?gitAqBZ#H2!WL
z|1f*qANZm=_T}k=|3Tk-x_pf^d6)91Lh;{**Z*PNVaE>hMfGxo|6>CGwmOFu@#_FH
zUmv9XUz^=O?eX$UzNnn`r|bSHp8Ojz{6FLU8~y!P(EfkM`!~A(uNwdVKjWR&oVGH*
z@0|a2hNU1E_s;T$eU9i);2%BWe-+oi=-E^44~3U=FS_X};4DoZ3v{B#9N#;|PD{WG
zumk%tf^m;m`6=XOG?D8ep})t*|MG@^j^@7)H}O^{EK*SUl#E{-`)Ysr%1T5V_{5k~
z)J<(wQdfZ&_<{X{Rm~q%q=hd980#HqI`W_aGS^jNk%|UUDJlH%~
zoE-V%B_KaJldO;7zv@i~k&2-}*kjEul8M%IW2bbIEfa9a;h+`z3r#8rPI^8Gc(
zcO3pveY5+0kCl^%HB;2B(B{(#&m2ju$^lD<#^B4Jkhxv<1{ZNB?JBALvR0}ve%apL
z5(N7Nzf{pO$4mPntKvjX;@`mRKf(H6j*<(W7lib<%2B}d7l@(D^-?Pqx2=vJ@m@ib_`d-oT|~ny7Ur#C85fbN;W8?9sNc_51fM
zhKNtHl%%R(Q4z^jO=As_Au?V+F8$kT#FuHi_KAeA4Q5|Uq5`dNEOzvd&9S;!w&uvR`lWgj{0&hcCJ%UPYW~H-Jr*KBovB+Wh
z+Qqg-CoxLrijN{wTIGFIi16mo+tPw!Ze)iL_vw*g?U3{!n3gMn9USf;*Jw*@IzH_7
z@HG0EqG^m@%1Z5NXa`HF6?mZuXEwOBLc#0NPfip>GtSm#3ctRld(UT8^6zx?pWwUx
zamQ`1n707gOQyEx75kD^bri17gjJplciWWP5u@NV>2JC%oVstTyolRe>K%LRTOr4w
zH$Lzj1u$A$B>fXQTQED~<-Hbpqj6yNaJKZflapOt3r2OB`?m~-7%#@7bGR|WIYDNJ
zyY+V_0CVjQF_qbnMnJ{BFzks;UFi@GeX}{Cf495BSjST4KynaKL)R-@?s+@$$C|Ik@XCyvg
zkLn?kvlKn<`NyS7Ausjgw#9yHLDpk>{ugEM{S5aXwGSssu*ixMqD$0h(K|tiDA9Xw
zA^KS1-{;bV5W6meqT2t4j#0m#h+2U#qS1b^q|pGxN;%nYlkR=P!7_=bZC8*L9ui
z=q$QUm0loS$p?K`SDty5B)P2ml}Hvtbg_wI)Mqr7?^|WMWPDGsOHD7lul^^H;hKNF2DBTEzlV}4Q@
z%otyuP#Kh+wO1DW3&G@zyDmpRo?p8}7i8HLPsFEY5LEnK$!MlZn^}N5!J4Kn(UnFw
zT;zk3M{EaFN
z_qH(t;SBq;ey2Y%j1tbjw@S$N0Q(xhX6p}9Gzh_Ella9m293@w`jsXw176$Zbc(ey
zA7!F}^W9lO1(wLZAy?$$%MuK4-7c?brStE$^iEtUAqs#n4Zpqkj2Y@Ee>Ymi8s|InU7BYceJ7ARZq9p>*$Gxx8yVMotfgn}qKGRvd;H4%0NT)X+&T~_
zC4A?P^mcD@oP|~=ysWQ`L!l~-keQ49fq@MW$;v~!!iKJ^*o7ku<7(nTyLZW-8e=UJ
zYc}+j@jxt{WKc-KaxJi^_Vc9_6BqK)JKeYBS40|Z_n01Y3r8W^8Cj_
z)Gv#H6RM|Qw9T;ZSC5giUjM$I6*N{u1{0`kv)fXG)EVi1|0*8Z0~sv*`tD*$Czy}x
zFlp2^2y+K(dYZAr?IO3;Mxo$+$qJLE3Dxb{Xj_Sn)zg3CG~#}I@m$Ek*d)5Xrf<>H
z(T}|2t7>9JocfxXP7;`GyWPr^>SlsfN3;CKde#-y54a
zr_~)PpQ!-M)i1y6%&4@7+@DiR^JNYK1E5E?Pz9u8OJdDb)TM!u!G{>fKaYgd&*{a#
zdtf342Yeegk>9V7J*$7jKSdS?B>WbrLPXjmm=1gBVQUhPH^i?>TrGdq@_RZ*-T1#o
z_wx!m6r3T`3~HRSF8!Y%1Xcp~eq!jQ?T!y7HiFDH*k2$$=4L+?{%o8b$>1TiWDC&v
zJT_%iAPaA6bNRLS8>PFQh|qm3SptS+{;!(jeO7-k3;Cm*4OWp?jx;*x#4@(oE{bx)
zeyYf(jGgZ`_GST|V7O1`E55rSW0a-97u7zTV|5Obj8c%l)i}zvU;?M9|7@1*%0(2RBW&%%aa#69C1vUpbHJ-fF;e`VY0P^$=C!_s~Ww~UsI1FDX4l+Zid1?GNz-J8m9TA&!_GD|AhYlSZsQ!az<1z=!har
z2Sn&ndW0ier626y(1?4Ptal0Zh(&lWQcjImbLr{NPhAZFLfOJc7@=q_)9SKvozIfA
zjcEkdqZ#~72OWV_ucXLv7H;P_b9SmS!OsD_0S^$k&=qQ+cVM^J4;OtcsC#yz8gLZj
zHs`nr=#13*(VK9
zBwL?Wi3ib?zfNx^6}JH8fkT4j$pM!-&z$5Aq=NLY>P|;~$kU)b(K^@G`Im-R-ziz?
zw)Z_<#sz{-gqOa4k+19`&mJ}%+x|}?aiCi(qYQ&+#q9neJ!-;A^m9Z_F|$sUJa`pU
z%74Vy7vV{V&9hwPE*sc4O@r$eks^fva8!+rI;ViZoRgA%U1dPsic$4hHrX!U^jYGV
zj<}Ak+FhCMEX?)?Dje8LM?#JZIK@6oit&L
z%>piu^~!Ea7Vh&hBMFx=5femO7JSPd@@c<G!Wk_a*2VX`-pP@cyi>y4|aUkGpQ?
z^XJcRPxLZvhx$HdeByLI9T~gOWK2IvQiqwf&C)X7w2yb(^^j@N>xk$Fk5y=2!6O5U
z=K+@inNC{rrCxUzezPZk(dD%u$FN`K)<0vLj|_uE@E*P>ul+Zpspzh~SfJBm2|V%b
zxa%avoN<4?vL+0B>K~=nvrCni2zM+gSvgJH@q!qYVR}y@2$U=fjb9g%+
z!M1ggMDV1+%|F@Ypp>x|-!kC2b4GJwSM_SqEc|}S0ALd+vIWY&X~+39xp3)_*O+*|
zq7k(z`1H_Y7e{{$x=`J-%M{eV1=;#d_S!H1S4m@sn6S61T)aHC%Qi46WiMz2({_g{M7vb3`zEay?mj{&U4!!1d+ST;0UeU}TN~X#7gA*cBfHF?Gt(cUhtw
zj5WyW55#;(?87R$OW(man-e#Eqp!@rleuX}zN{?=U~=0IN#O;C?q?wtUDmz0mD>6z
z9Y=LcY`EDJyzlleGWGA)N-NjVmV~xHm`oXeoIwzZr;Lne9)A?Kj(ak3iVjW2;IU&6
z$d5*}ucdUnB|86)kuSPZ@)2EcP%@M!x4JRIayBIeS;0S5Q5Q(lzBQQR++*1JRN?wI
zDq;Ot0cEzA=DcYTxVXQbKs2r?XJ2~zL!f2F_R74d+Z)q>_+$wl+gb|jvXHb{AB9g<
z%YA|#bvK`n!;|y+2)@oBDZ@~ZA0q=owj7E1zmKNui+^~47iJ7foRdvY_kV*YjNR~XDAsM0@b%^got
zow-N_D|2NH%bf&{{jI8H`Z|MmW4ji&{ZrL0=B3(*)XMJ<9SIzi%2IT)ol6zf>AYq%
z3h9hIIWTvoApi-^S`?-5X
z)_S5^#~rlow|8-OlTYOg=GtU5q$!t9~$ew
zf$E4?AB|ep1?it9K<2)ZjJ;gjTo(YE>Q`CVVv@k35eTjndj|Xs((_
z4@NcZZ&(H)vME#4-w!{q&N5^^OY%T8zSPrdu}l+^q1@Bw=uoIJ%-}ul{x;hM{%vD=
zk!ERNKXZ0uZ<6O`2^@VNtZGyfR&MdG`E{BQv6?ES6^e+(7$&%jeKQcNUl3MH;3iXK
z?6)n$!nsfiC#ZJ=C6D*zGE!ogO&b)xp}S6tKtr=lA<^iLTGynPhL0OLyNz^c7jlY^
zpl*vBdX!br`eX9YU?J3Lx<3e5W0is)t}1ONwd>VGBpC+$w{
zqHV9PpUNm|U{$9TdM>L^y)VaQtB=Bjv=F>fqg*;YYp>nvmGA$70cHz&%M!oFvMnTKlz(dhe0xivsPd~KD37p`ngY)uA
zUd9KA<29_+a-)=@d$aSgj_Glw+ei8vTYbWypbx_JHvsNWQ0Gfk{gGwzmF6{D<4iCE
zyIGh2&kyYvsMYFP`%&RDF6Ro{%I*Fq9kTR9DE&$?S3Rhawe0&qr4`!9%MO4yYLtJc
z`Qy|Tdy;i!r~v
z@!*axlpG|kH-9vlix3ZZ-X1fs%BkI{u0M>~dKj#Mu0HC8fiV}y%b-k`ctSezTTowCw4!-oK-e`8iO{z2)jyV2aH?3|R1*&On_g3D07WWDm|OCT~slP?P`b+9LEnwMG2?
z9bFY$!MH_QAQu^PMfJx?*S)46R}`R=#!0wo{Vc!f&3YYgdG`+b4s+fqwq&AE@Vpq%
z2k!HG$+OkN;)cklQ;1Fn?2t|hr-7uckUe!ZlNjatclunXDV6NdY8}G8JP8+<#ntLg
z+qZph;{rJ+`&3!H+f;f6H@Ow;$79nf`rtnTeR>aYv6X82#)|%$QJuFIS#_?9=H&}U
zPT~75<-#p*)1>otf1=F@FV8s5q)d~r@HQ+wE3iTD&foWr(=%ThZ(CneaWTLZ_EIRC
z%=#tkzlX3|HZ=vM-=84T0a*m5hE)Kn=L8L!rwkJG3OTCpUcNGQ@$p+O9O-U~PO?q%
z^D$mDU98lFoFzq`-+~5T{?KwkOq@Hbq_GtOc&>ty5x?{ZGs`4PWm6y%GUrj9{q>2A
zDs4wgnhm!c3P)po-z?S<8mb1c$E$6=!Hzc^iCkLQB$JW9>T&EK<&PFB0ADR6rg5tU
z@PAz+w?^wpU{S^$A1iu)sX_0TX}w`x0ui4A^)N!<_P{xV&_I(r{~?#^ju}b0xne8x
zxY=z-!kR1iLf{aB`zD<$bsrl+;15{@rL7x_E!pjtD?D?N%<*K@`|0(ba0H}e57XN%
zm^%}*5IXeITkptJC}Irbt;(lsv|K$dd}`?bN70>;=Im3AITM(yx4qLjzdifIyBlf!
z_d$XYUkL0Eu$G8}W$R$#8GVQ{aDKQb5C+y0SUPurNiX`9{|*3-G+*919Cn27Nlv3D
z?!QHom+}jWXUs$)R*w}!BqrUJh4Q}m3TsaV?EAP>+G%_=Unw^q)vr1+;XX_swWutr
znMHmPj>)y2uwo}nVI~a0BMf2wqPx8$w%Rs{&I_bn9(V13L^w1lHr>2@o`?Q+h=r^l
zC9Q&7rLJZY{;WNu7USL?ulqD6kso}~L2gqaG`O@*GZj^QB;D?m!GPW5H)AX%%h}%v
z8xvpqYel~Q!)ziznkWrT;}nnM#fVP1LCe@?55_J_m*SKqBQ?sJu*KBQcqksk3=!WP
zNA-a?KR0teHght6TUz|!oeBa|0js6lx>}p%?U6NASa|uyd8AE!za(>9fN`l2pQ{*l
z1+MD8T@M^^X=?ka8Iy|&Lx=s{RvP#~VgxwTK)Ke@;9c|{CBrlYZ9RpUw2oRNUgjoB
zJn2e-aooGzBE8~x&}$1#9^nGxnSnOe{&NTkP~(ahla5V|1#|uEI(=;M(p%%n`U6Yb
z?BSbd#G|`>U9%KT^UCSZ$?XftKHz3O3)D9nfh5d`VDlzBxMGjrGf9MD?sd{)ZVC!$
zMrocfy>npGCD>6}Raiog#rWIKF>a>VJfAeQZ+yX%@2ctb<82vM1Jfk$<-S?$Y)%kt
z$df;!e=_=k_*2|N*ZQ6$Sp`8U#s|lZYY)lVJz-2ew4>hiaN0XOV2nG{=2gFJ>`k;&
z?m3m}82D$=z2q1=jxy_HP;Lv4PR#UYs(U1z+8tpt(x01ibLe*RXBBMOReCxL@k|q0
zm(@$amZ0>oXgCec8ty&IECMdv{R!o_F*n>ZUs=JSH*v@dsm^+I61P8WjJ~f!D4Q?_rH}XK^?!*qX^P~HK=N+&qw&{EQt|bpek@J~udmupV$&KO
zho4u3zfn5oj)rZyI{w74u6`PN!vmHJ_3j3+d!evNE|$*gU`LKKHCw|Qy>j1
z6!CuOq`pr))amPFZ63FtEDvG(QwO-vL!^V3_mvlSmE?Q;T?7&jQB8GKp
zkj|}UonethSVuX`POtoELtz9ep_9tGE}j{$C`hz>V}&mc7^3r!@7Zr7!8o^X^j__*RW)OL(3oKd%v%{
z2dt~h|8$Z+4Z)&Ig-0phuv?v=kzJtv0E@sUBFC9);==S@x+DqREehEJ+k{(V^=Y2w
z>QP1i2sX%Q{Ms@XjyI4G-Hzy9t8z_ia(&%oK?}&Y5LucB=OV+c41@NshdMjb!}|x{
z%*n>={&Jy-dq)XKRj%q{|S
zT?Tv+QZHpignWdSFve8+8IKE&(bdbv`plgcH>lKJQz|U6&=ah1zAO4F%JMDMRvDB!
z+#Vf#BDN{ncY8eK4rSgNte*WvUHknrT|A2%5dL_lzy+P0cl}+d=iBuomK7Z+<2>$D
z*<0!K?P}(4%^x34+1)GQjCJQwi${-EwwDN-KD*cFmQRd^)14!nE(W@C5MP$8^yDe_I{Vn~F}$YR+|;E|!fM$pN?Q_l~c9-r=H
z=xzvCZ%Mnl8{ho&QFP~LiIfPF@c&I^Wu@xE=oQ_&{|av^
z)Wx*pZpLWR`riI-f1>4R2@GHEe7jW>odV)G|HZ-7!<7l_ba1G=>u54FgN48T2G1L&
zG%L}-%9`LIcZ`q}bTJ8W=hZqzIDvN>v%Yt@G#SEYg)H&h2X4n#F%B0uU)(%z5LP@8
z;LoySOu*i6%S&3W$LH&x0b@xTc^75`AdxTvTbl+Sz8)b7NzPo`r+aa8h{9%S2bP`3
zR)-Q7hQSZtor}vdw0&r@wZ7Cbb75I+DZ4~pmQoTi3kIp&4dFD4PFNajNp0>s*5Cq3Tn!Bb;U
z)m2%p~*%KdZoJEqBXeH}q8n6iyX(H=BXFedA46(l6J$IDV)E
zmY9u20|eoZQzxh62tSAXm*|dUel@GrLWXE7s(KP`Bqb9a8IT3HEsllnuP4rpIYF~_A*Bcn{@
zo1zoO`sN|%C6RxM?t5${A&mgYgNEpfdenwU=Fq>t_M9;-ui{{2hvK_ssYUA%Yb0fuAX(A1NnN$nrRalswc|2NPx=u5&pD;>5I2@@4gprs{*O8Zs;oEG1Ts
z$G-j~D2qz=AQ0^WX7E1ZqJV4R;1|CB^|!eDx!7L?dq$!k&}bK{A8tQdhzLU1s+ioe
zKZ=lXW_^G6gd4K)li`*>7?%iz+x4DVCk;y$Zo|tXHhu@G
zDUR^JR!>CCfOJe0-sILx^2KiUDdu^sQC3rc8`E~7Q&VVDNUo8
zJ@HilK|E2BD=|$W3ZHmEZ5x-GIZEvyrF8*|`=^pKaek`s>7cfEsU;b~ZDx{$&`!^d
z^7uPY(t?-5adAoWonwXVpxRl0h6-)apIN6V0kvP{&TLGi+_q{5sHnQswK~C+UV71q
zf6u3BJIFvNb_V0at<<}V0XRqT1dO$@vj)xyAM@RO49%&e#j6>gyFWp^1P%-P9p{#F
zD>O3WV3Nnn!hhYwh{sKV}Rfe|0rp@5as+*pv6~`U~SIgb%S8
z_vXY`bI@Kvlx=mQG_PCS3MdVX42)C5Hm$}KZK1%BS2x+=`Q75$Blg(Aw3ANS%Cd&c
zNhkV5jNxmW12Wc&zwSpNpU;WMClnKzdc$!sVoU#U#LrzX}8ZzpGD&vamV0$R6#E8#qL2iOasUMukH2Xx0aNrgqdUu@b;YOZ>+9-
z$$_8n*pgZ#7QgyrNgq^D1qLF}lO~6QO&){@$i#Gr>0w~^p`!JI2ktGM4H?k;)YTs!
z&4eR(wHb)NNcOZopS8oH5q=)a2(574D-9=et
z5CRqw7PfnP?6ca3yWHp%nBUQ19j<+3-bhc*W|vE@gI}0@Ny}m=V4l%BHbdrg(2n9W
zZTF9%Vzzj^hN^X9rtCbrMk`k~kpJBTfIF@$RVX^P7MyqzV8Bv;-B5J!B|5*&&Nv*V
zWZ4nWugua1_Lked0L$E1E}ff-->(e2vJ
zxr;mV#npbzbG;`D4pbYZKB$8xkV?0P0ec6KsEE>+`P=C*oT=$cPf)Yl_93eIvf%{(
zmF=;&9{aN--hXk#-DF^{8Nq?(4e}YHCtob}ExazN(nHiu#x^S>BH%nJq+A<<<7Cgu
z(%;-&G;F4+xI_}|8UA$q)Cd^rQV+8$SW^8h_A7^3PQy8l373F)=~DN{XJbD5e3D=)
z=0=^94tkxB_j+x0dxp8%c>u2Yg*4!}YF=sndC_}sz@autxShz$=rnE_L`JRS2bZxa|#4Qgllk+>O3(kp$O1z5YoK
z(lghJf_|V|sxRfkf?Vc}+?25(GQ^Xl%{Q1Jyso#+Z{?4a?8N)WJGGBxlyuRFR33t(GP)Ni~xZ@MNquAo^2cn(v;SJLI
zx?J@JYn@a`89W&54X#zJkAc^f?=*L
zXjjeDarr;{{#Qg57WG5f$9Wk<%U&RACOZ#)H72RQs6Pn4ubB`x;8p_h-_0oPv+WfF
z72EtbBQDMGy1N+_pAVjV+fZv9W7-$)bBm4qCF1aCc*A|6UNulMdQD4RY30v8!8+FU>gT
zDzZmpdPa=g?Q$h)g@KW+Pvv)^#zJZJtd;T}q;Ud_$_
zOP{}Z^!1drsvC|Dmzt7&11ZvJ{4FziVr-Cji4hNI*Nygvmbp1E-U3LJZn!Q%Lxb<~
z#HXhh*WDGz7>bi(Zn~(R6I$P?{JAu%TzI0Q)>vhfm{@78#p{D<4%4(<+Ri5@N6tBC
z@aFY`59<0=$w>(_s%9+m&q6%qs0Axu3CpJr?5t_EH?yO*FfA{rBGUP~gXy~apdHQU
zC&`@?jGz+}8@LDtp0--VNK{#*SmjtX&SSn#Gq4%2bJ62P%ofq7eJ&Zd@m
zlTFnar1B^96rS;ejrnZWqG9k8vkuxQ)`(A3q1P^eZLvE-uQp5Ff?>7&vl2J66iDHP
zZX!Be$Uu#oUF}-L~G#0sGMGb&k
z?$O_c^+(DcXH#W*QI2uyM|*NA)yK{n8^lrC1$~2%#AJYmoU{S$x!iW)O#BmDdZu_F
zgV)a7lR`u8^Ng&oZ5}_e8=UCyLS?H(Z(_V^vwu(;fqdc(iQWxFEu6<^fGYfnyST5U
z2=n_{LT!H$m&Vb7H55|bt2-{RYkmIyHumxfE)#pfw!odtOil`V4&70%lDrXj6K7nt
z;W+}i-Y}HiDK&FCE_qtf;+Zy6X&l~BzoI+PX8ET4q~qKabp!nI#L;Npn4vNr%@AC4
zw%_o@ysa9Jd`JeRj>Y<@bTF;lYFCR{A5aPz9cGG|Q7lz>867H$M|*vaL`8-1Gp$*r
zfDY!gx&N*(RVdW*ZkeCdj`D;~)_!ZIw&H!7d|=uiPS}Y^VOgZs)609?lmGQD&bF`_
zZuKqd<}c$2gp)^|d`mjfz~Cxq5G4H)Qz)lSUuBvAYIm$zJ)CU2xGNko
zi%iC$4DCqrOBiox%Eph##_1l?d9iMcfpub`P{Dxix(1$-JXY#^ZCw>K$gqS
zL;xo+A;Ys>Gx_N>S;`bybk!MAj=YLi2bn`#j_|%z6V8iw1j9CypFShh#E)a?$#_?gx;d5H!x=k%vjRM(
zI;vy&FzzuQi`<@iM();TX?!U60(<6}tf`2T1NG2tCkUdXnJ*L#G~W7W%>?HvuI0)O
za&_~9zPh&@BFoKTDb;Ut{&wY@G}>yt>=DnhIFcgi{mVw>mBmH0Tv$;@_w+k8nm!#y
zq9GcXbc*X@e!$Yz$~@t-r^Da1owF!rb&U=|Aj~Dvpb7~_)|zn(d&L9!_aMbIX)T2k
z&2IM0c0$xsR#}|i!*M=_{gc_@aCMyXYE1a+pbq`d*T5H?#4HD_u8L^V=xt)aFVhwqE21#ZvAAul>)WpWD%UrQ+};v2?wvOoP>Ro|%dw
z3t-LtGI%>#VQ#!Sy)_TO!7&Ii8<`jKS9*a1)DM!dR%Lj@jJ9uO9n5}lXmXa4jXY}bi
zt#q{7ymiJLai2Ubc*dUCdBrc-Y8loU{|CvNH!}L>@0O2W^XdCxKf6he+4D0~6<;{+
z3!a~GhhD*j-qM~&A^Hke=GXkU#rphad#Ilg(aNeSDcd8%R`+X*7jDjL;p5G|Qbr16
z{7rCF&b>_*_d%gismBVsIq7+Vq!%8RGpJY6-|ZMQc0+W&dynxQ>RLB=xpz6WU@d|<
zLlyWC047gsjKC7H&5V3XL)Q~eg)UCaV2!1+JHN)-m5-s>`tb0~oRIV|wO81QC;$0t
z>Gr>YO8(j%vHxlA4k66-!$|>ahGm{O@#5vrR2VKE-t{mUVUK-tB%IPY)&$KnQgm@*
zf}RZ(Gmyy6Q53m!y0#Msb~RCR&&TL6|=^%w0J_Exb(u>F=>ZYa2nvFitmgl
z?z;pG!eTp2mokO%>oe?1Aa$)av!JeWF<5Cm4#$9zoY7u{Mn&aA#INQpTIyO(XADxh
zq-NHl1%N~MMpOk%$&eK>B)m-6l^7J;d#1;0^g*cex|;3P7T>f2r>45
z(Ue(}2F*MmI%Fo0GD)J0lb&EFRe*J7qUp-1>bU+bASs;5wa&UI@5E_^OHrA*Qv1Yy
zF;jZ%_Pzd4$q%Y`(?A<+$rtzc7*nUowaPl0P3LV5h(RyOh@|%P0quw{arbY3Fbu
zhJ6{xZ5c2*Y+GHNyw&=*7ugIKUOie+2NyF>==`4A)nR})lHcA~OFtFl52nFliNuSe{ds{|ez?bd0sguF(y(V8^G^szqTAQc~cmqjem>cjrJnEb%$ufMP(H>C`IUXu^{bPBA?w?)YfI
zKgDMqp1Z*WXrRn&k<UD#NSun27DD0@;
z;(&LeNzOo+3&2N62yu0|BW#nv
z+(W(ky1OF`NOVLi<8SnHJ3;Ka2gf3bME}2G_^+U@96lvka2uZyRg+pHtJG(q|!*4pA`@+17Y^K0jI9*C0m!Dm6QSpc4vZ2kRN
zT^Q1g!_3z?`-kD}O$?SAz%tQT%C}F4TGcCxQ~P0=rQ)OYJ^r?F4NG?qHmdD!(>7nU
z`5CXa9r~xZa{g$^9Zc1&ev>M&3n-^Q=PwpXSHaQQYY;`zt$KqpEc5deu3Y3;N6p*_
zA~=dRdqWGI)j_EUmHLKDpe~1y%J8_aPe3nk@djS8&SwviQU3F@?W;g@J~gI=q+^w4
zch#C-i?sD?r*{JleUz%I=6tV_GO@;X6qV4%^tOYChzgnC-gqg3_>Sx?>B)TREEzb|
z#Ugou&u{z`C=^6#Dol!GCKST#aQ@O2?~tT;5Hz|pI1}i&Aw%udcX$n!r3+G~uPOSv
zPd{X&mXdGHF=n84D>osHwKHHYBJ8+DU@$;=5%=AtGp*n)y{8&7NGHap8`88Z_!&kfjje`#
zkz&H@E_o1y@egLiLx@5LD616*@##i3axHIPcY6=zBw$`^a5qwqja<`ab-(&-RXXe)k^Hm$4*(C|H*=Dom%Zy{1ZbRoyn1DYn+(RTfari`
z^t}#C9R4r*WgdAYdWF<_VN!}GBD}mD5qzTDNa
z3BM`94+&lqjr;M7
zaTBHg!r0x@duw47gwQ79pm{Y^n0iN{nVL2qK7rv_(f{KGu#-C6q&gb)
zHpqTb1`8#%d_YuEUf%mZF!$}D+?|&i@Vv#SooF~s=tJYPNV{0LcclWz@8y)bnLWTm
zLxX%fQyW$BlH=)np9v6Tb>Iid)KggXGhom&oE3WTvVY}f5<}e!xf`N4%L&cWQ!)Zi
ztC)0l)!EJbJwF2fGQv3|oDv(By}IH8!kK~`3GUFA^UNC^Q7AW(?4+;ecE$gQwEkTc
z73+OJrEQvRI~X)T2!4Vhs2Gt={74STRjV)qu^qf8?rhJ&Am7adzP$bvwjg
z>2yP^(xKigJz_U75PZTf!;tbWrmpLjm-RiPg6dz3462;NcDj%Ch`&axPwQSDLJcJ_
za_#I_kNfDzcj@wqMvJU8`IMvpso*CwW3Vtcr5jSGs>aiGMc<3ATWad>msF?DVim7T
zc^`k_Q56_py|}jL(ifbsJ1#M;#H?Utjs+|%w|8ozd!!=qG)jtUwW=zOy;~X}`zXE<
z?Z?s`vd7M2Ix0a~FWy3L$~bUZp1%7@K@5B1=7>3UC^6QFKq@85V454`Zs`u`>|KNh
zS4(sM;e2X&&VDY)b#-)iZcDwv$wsa7@y~h4_#MaH7{CZ6qLw}ng#}aOySAb8uf&sIXXhy
zGST9I`F6n0x0z&Ip=f3i+A94wvEBmF0v)5Yh$;-aV`4fxSUd%8ZArg$66(`cyJB(>
z*u85|3D~MJ^rpSLkdA65_Nejed-OThtzg_TWyYK0?~SU__4L$*B8CO)N7
zNcvJF{L+x7sNGC9<3y6Yo>ak>2o-_~9k;*orqjvi{^Gif-a`2v!86;)R%DKI#s@?$
zUq2{5BA?aXFQ1t`J2LEvDML}lgMM+v0tvSS=HkI;Ws`4XM>OVi0c1Zqat&lQMG_mt
z2xB^(ZsJyC6L;pRRVQDle1c3XP3}gYUJSlb)B6Tra%!0GLbt8q?4;X>
z5YN5BNEzfA_T+3|@bB8iSm%Fgp+;}-qIkTn?+KLoRg*h{oMWQ=!8ME~DKvF*Ei{H*
zGu&<9s(^En?nQK8)@3uEhzGxwC-2wTj-XWzf99&pZhE!cWN(SPDBmFG=%DM2^g*+j
zHO4nj0?yom7q`IMfqcWWFO)hXS|pq8iEj_yDsfD}nF`tF-lF|Kuc^&BRVOKsZ^@2$
zV-%O(o*(wra13U&XW?nCXL^cm3j{kXcr41CgRoMT{*QMC36W
zWLs~`?YyKnd(S)GEIZt`8#h!cP323i?+^=Q&P0I-LGz-VXHQ
z2&A^xZoWGO&8+2#v{3D&bevYtcvIj!(v`&QaX5YNNX$GS7aOr%p%tQ7Kp^t!l_O*s
zeslN)jJbBmhisjAKzYEI1@d
zHI7SV+dDvjT54L#{Y}#8N)+@mu}zvK<+&}>6xsc(Kv0YD7&6YRvVTmO!Fh#0#d+I}
z#URWaJXl=>Nx#6)-RAcc44&L1XeYj;zKacrz31cDxr@QGx^7ujbL#RW7#$M2h0ut8
zQn!`}C4rGeirJW!?z5wc1Fq|**emGyzSyzI(IJa@4dhf%op?X!MntoG!?5lxOU5y3
z%{cS^Y%=VpTWIxw_t-R_odU}fqKuZR_jDnS?}mlQl(dN>`WHs|o5~~Ot~m(gTY^yp
zeDzNs1@?T9xVXIQ$dhJMQ7!p8#!?%j$I24KTel^}60vg4Rs=R!9Lm$Y=HBH(YV$Cp
z3DvOsFF*YpPWc%E)l{4!rlTClOxl}wG3W1mu>8syjDL)$*(kh@B*D<-F{6fJs$4%Xv
z-k7k<=$cq7^{b;_52s;Sg8^G;@l0Q{F34efB0HXq+@g+xY2b?gcMCCmJ#vUV?YapPR!^LHj5hSrCGGejUbx!oX(hSZ63liP~E}JZ0c+K@?dO|
z1!((LKIU~I92y?4_N1N`;n;$*u%rv!qov=Bet!v^Tf1xDda$r?vpZ^l`+F79EJx`n
z=~K=1vwtyBhAdD6p*EQT-`Ztp7ckbAb$KmpXFKMm5|t2Fn(KjVPLP;o;=
z&XYO(ges*F6ib$Ipe0+*PEXi_uYU?~=eRA-0%Ef$0)@B1RI~bh7z9%O>wM?zTx;3L
zKiPnJovu}pC2&RQBO&u;#09}&;E%(9vLl@~w`nugBQCQKZ`f+jswDQWKDhl=n-APo$65xn`Q9wM4fp3_A$V<3^vF+{_>z
zol`)$576qgzS`N_75_~L@c#l5&)pvDj5EI}rR@BX``lT^jyA)Y(@dWJQv_Kb6F;BL
zx1wGTCi(YaZo0l{gAl=arUJ#Z(a3jE>I7rPk34lPE%_D9Sa?saHhYU!h5?&Qs-2TE
z7}T(ARJT5nJYmY>j;QmA$@@EV3xfTs2Z|-8IwVkwcDxj8Kv{7P6m{vqqh<^v-0^mf
z<&NMed8wmQ=&l#q-}5w=DZ2jcah%O7U6;6@90q9-p8;ReRor=Fb&Nk(v9abRQs@18
z`jqwJq+*`yufRj``+0r1i4t}7h0}~CYAMoou;jT84-bhVyRzy_UH8j|?fVI}{#2W_
zO8Lv9m=*<(^jducH@~7fNxxuhsdc$Co@Lp4k2qVsFmS+CBtv2DXY_I3y8)C?g6Ja&
zVGDC7QpEBajwz+W@dG@|ktp>STZo0P8hAK-m(?<pjn><)=svIYXw-6jVu2z<-f
zG~>?$GAj29kQ;rKT6J+{{eQ479Z^NPIgWesUY7S}Zt||L{=?DFH-d+m_q*6>+YZ9U
z_C6}?u1bNQ49)EYPJ{m;_h&E1zKUKS!;_^1b|^~EG@YjLZ(+Z=$zPWD-N?5~I67@3iId1m(qY5
zW;{lxHEE>cTNSfGakND@Wcg5R6SIK~X*{ydMkd|-0A~@}S;otqwU;!Yjsm~wKgYB^
z^P!sNV!eI5*7-ZZcG7|mh%GdHLiX^#MCkMj_g-d+4&+^qWGqqVelki_QPlkIZ$R5U
zdB1-ibazgcN)+?BSA$$(DT66!PIjqr+yb=oEW@(|M>uPQ>T~^9h@d{Ncii{t=L3_G
z`gs8B=vt3|`|6MnM?{mQr@CQds2@`i6xTxZUIl<@I_~U6=JMZlWA+yr(^#B_6!)hg$lGG0Bp6P?kj>GnZv3&@QG+coKeW^3P8Q9ZJiULcIJi-URj&!>bjm6c;7~Qw7dd
z>vq~mf4sRrh}G39
zXd!pbg^vSaY}@8rf}-%)n6#|Ntn7*br~sl9dL1#2*wSL>5egltJn+)?rzbwS%obJP
z5o-DvU;9KR;uL%MXM6Rd;%iec{5KuZoi@BGR9kbv!#6r7qx>u`XpvEQ@x*@4)&ze1
zkx$7he@XqB<}#&oWxZ+854i@&>W!Z#6QULU@7$BGb_l|pHhY!Gnv$Bw_U_MG=X*}v
z`M(sI_1&~rsaICm9)m#`VrGTRdu_vnp8@xmp1qQt@o9PbJgw6*S!`SSYiymlIp2W@
zvSVY+*nps{_O8QEfMW$MAyEzbO1zMVry+k-AFye7^nyT6fmTo_bbf_?OKAJ|PR5Be
zAEgUf41$sfJ|RDWU*%AOtvwO~#F|g~t@){>kqJ1llnLIi<9)0*pOizvJm%aVB)2^=9_GP#U&
zvk@y$h5R3+nBK=bP82!|pe2eF*E
z@h|6#ds{Eb&Sd+e&FS@JmFXyEwF2LDrdw@hcU?(*Iiphl2ZdwKWBTC9*E&$YJmLhh
z>eWXIM>GNLtM1NURjlE;>I(jwJ}rz$Qj92y2{O8=mp4%ZHHD;cG@>(&G(@(JRCCb#
zhelUx@NAnM*uEpdYpW&HR;K^r{)yt4IQSb^u*GqC(
z;wJ|Vc@09D+_-noquHi1#W3yS2iQ4X!MDIdrp%lRYnDE{fT@`qOWHDR;P5xF$O6yXdA26*$fw_-dhy$CmYxn*y5f%
z9Kt^Q1!uiVs~$(J7)
z@-|j3E@{i&qr8m=6Ik~#b~RE=KtgD)vH)lE1EnR-{+?;6&M%5yVJnc_>8o6Ga
ze8*a+T>po(w+?DM{PqTm7bsGoxE3f-9E!VBytuo&ySo&3FU5)kw;-Xo6)6_nr6srp
z&*nF~@63De%)7I_`)?*QNxnJHIp;YaIZAAsnbd)e0Gs)WX4_=U#D=yVFL}vm)1i8;
zdgD$4kS^u-MYggOQM>1SJ}FxT3EyyWUJ0e|EHVfQvn)~Vu?&maQi3N9bwtG;zcCA?
z#dfl6o!$K=^2b159wG{4zkJFNe0JF2{6J7=9e8gh5At~g&vBQxx$LekURWO+{o#V^H&@DGPPD<_?-mS8*ECq~sJ{CL@(3aSrNdK3T28(sT0txM
zO>GjhQ`8(rp!b`1PXDY2&KL`6D4-NDMZzUqEe`Cz-^$ACeqd@-UYlNn9`6!TqB=FmjK$f?ib{`Uh^LqWF{8!%Lg_PW06jj`+EZAk!>g
zuI>5@0rA;-!i&X*v?#l#ioZZ@t|?W2WkEKvhe8EGi9;5f)XLvLRbi`mZ{6sW;DRAi
z^I{`%95Mc}n&fm<{|Mw|h)2Ly1T;vcU|pE4llpj9#BBXB5PNfX$&4a1WnA!bxiYv1
z4p|f$AR_uNLDeo<&0O3{-kLM)L+8VuMcIQ+o1Sr_@Rz{MDVC7#hNL%#1k~$DMO2#?
zER+XZUVcw@S+11a2n9_yGi)5gAopQMU*%&eCHGrU6uC+>g$8A`zk;4{#;eb5Xm~bo
z`h(3Vf$OP_qYrA;qRer-mkxkQC!$S!i{9XgtM16v#Ridu{tM;6Eg=_fMPoYwYB(AT
ze^x0o_*{lEbwi3dcaG%|A}v0?7A
zj31E1-UCmupP~Y&C|$RMb^86kEv+@XMQB;DSLrvr3THV*(Dk`^uvYzbb3--0WO@cA
zThpnH1-k8cC&eYB$x+XuLX|(^-2_w%t>~DRevyTKncs{@-qfx*pUT@;%Bh>;5q%nC
z;CF2b`ov#schTLR5|?>+!C|>Sby#%q$;>#$qaf%9V+iDxMsjYr@|Sg}-xMSc#UsR%
z|85h_G<~SY(sl%UEjxP~#J>uS_jNiMo(U$fRb#xL
z>U9JyNDQKHUg4TH6|{z(R?n09s75^C13nAQ!h17La6~<)oRFg}oY^GwxPO&5Vo~&H9N{(dvS*QXvc0xPoLw8g!yPGm_!ok}
z6&NNde88Ee>>*-QdQ)`IPApc7c50BjD>wagef|a`*AQn;H_!wLavc_QcTd2RY%;*{mFlo&D~XvbR1mAlw82Mr|e-9m5kI
zgn$-aUe7HW*$-(9@wHNL#ZPlHfP>;kU{k87&|hfufmG{*K?LZ((~nE}mmkG>d6wUa
zqKYQFZ4y7ZGT+`XJQgoHsS1@rCcZt&dbpxE`40RFb959TQ+Dsa1en)-o;LvQEzCCUXlArH>gQxt?Qu)L4
zR4(^OUFXq9VR(_EL_(7^V=?_CC7Q(0-x@)7kKqa?CvZzNnm=vorVtDW>LUsk=&O1oLi=fDeLHfhDP1btE$eB20`PI@{~4I~e&{ZNy4B2FUEu&Kcg
zsRu2Nd_?`dF@k~jX$UAz&ia0`8QX$~=sn9&n`vDc^la$K4%&D*K6O)^!U^Xyr^*Y%
za9$eesSmAEuE!&%ERtWwWjs`y4_Yfsk3FhlH4WPE-=FQ_aiMr#iE`d&SSW70cy?cq
zSt&826!-$U+4dXVz74pln`@!;>Sg7i1D%**5|Hn?J}6wcc6h-9)`xSr3?C>qTIW#Pod!T)
zy;DM2WN^K4Fp!?6mYbwEcF#t%0GIGE^zl6uLjjh)2@Y>Dtq9Waz4f$q0q+F_>b4up
z{F5?L9J!+Sr*FQ^0|bVqLF-9;HbFi6x;XICtwzFpYaP0#k>yRh7+*FQat&}Suk4!Vel$2OzHgMdj?8m_POInVhGc-@pc1VX0WyyKp&E=Rpi@w`8s
zI{YArT(!+)AZ`|)MxHJdlR7w7-{WJ4Z8MlsKn75fv<)A`j6x@pLOgMN4N{DdHt6O?
zW{T%J{UML?nHhE4?La346CDKABgZ4c7s`DdaLkvybe;s@$BZGmdv}D7nJ$p}HC9tO
zzCtRcOmQ=kJW7y>lP~D6_JofYjg>d_hXG;Y!h=q$PJ~F5de->@jB%O>_RR89(jueh
zWI27h1%cf2A#=N%b=rkhWe}`W2aMR~r|NbOek5YMlm@?j?S!X2X$KZz8$zPmNGZ`G6z=MB6uab9w06Z7y@B1-ciNK9`8R+~*_}(lV!7Ob
z7{G2W17LT#lBScw%{)E%okmW?i8J!$@#w6Ts0-s2{ff0d_Youz!!9S6=KRwBJlKGP
zfqEg|L1XUj@{`<3-A$K~A3LO*
z0xzJq@~3-elk-%JI-4-tQ9s<7M+OhaiQ~6F?%`Kz8bL!$<4u8UCdcWPMWnZ_$iRG$
z$f%PF&51~T!i|u-qHMh3ycmCpYVNtvWvX8T@dt#yck(2n-W