Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions integrations/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ via `POST /api/v1/admin/rules` (no `[policy]` provider).

```bash
# From repo root
just integration-test
just integration-test # defaults: fake-agent / no-auth / calculator
just integration-test codex no-auth calculator # positional arguments
just integration-test-matrix

# Or directly
Expand Down Expand Up @@ -39,7 +40,7 @@ Harness LLM billing credentials (never committed):
| Claude Code | `ANTHROPIC_API_KEY` | API billing |
| Amp | `AMP_API_KEY` | Or existing `amp login` session |
| Grok | `XAI_API_KEY` | |
| Cursor | `CURSOR_API_KEY` | |
| Cursor | `CURSOR_API_KEY` | Optional if `cursor-agent status` shows a subscription login |

Harness→**Atryum** auth is configured separately (`no-auth`, `oauth-client-credentials`, `oauth-dcr`, …).

Expand Down
17 changes: 9 additions & 8 deletions integrations/config/harnesses.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ harnesses:
mcp_transport: codex-config-toml
mcp_config_path: "$CODEX_TEST_HOME/.codex/config.toml"
mcp_config_template: |
[mcp_servers.calculator_via_atryum]
[mcp_servers.$ATRYUM_MCP_ALIAS]
command = "npx"
args = ["-y", "mcp-remote", "$ATRYUM_MCP_URL"]
skip_reason: ""
Expand All @@ -35,7 +35,7 @@ harnesses:
- id: claude-code
display_name: Claude Code
cli: claude
invoke: 'claude -p "$PROMPT" --output-format text --dangerously-skip-permissions --mcp-config "$CLAUDE_MCP_CONFIG"'
invoke: 'claude -p "$PROMPT" --output-format text --dangerously-skip-permissions --strict-mcp-config --mcp-config "$CLAUDE_MCP_CONFIG"'
api_key_env: ANTHROPIC_API_KEY
api_key_env_optional: true
hook:
Expand All @@ -49,7 +49,7 @@ harnesses:
mcp_config_template: |
{
"mcpServers": {
"calculator_via_atryum": {
"$ATRYUM_MCP_ALIAS": {
"command": "npx",
"args": ["-y", "mcp-remote", "$ATRYUM_MCP_URL"]
}
Expand All @@ -60,9 +60,10 @@ harnesses:

- id: cursor
display_name: Cursor Agent CLI
cli: cursor
invoke: 'cursor agent -p "$PROMPT" --force'
cli: cursor-agent
invoke: 'cd "$CURSOR_RUN_DIR" && cursor-agent -p "$PROMPT" --force --approve-mcps'
api_key_env: CURSOR_API_KEY
api_key_env_optional: true
hook:
type: command
source: cursor
Expand All @@ -74,7 +75,7 @@ harnesses:
mcp_config_template: |
{
"mcpServers": {
"calculator_via_atryum": {
"$ATRYUM_MCP_ALIAS": {
"command": "npx",
"args": ["-y", "mcp-remote", "$ATRYUM_MCP_URL"]
}
Expand Down Expand Up @@ -109,7 +110,7 @@ harnesses:
mcp_transport: grok-config-toml
mcp_config_path: "$GROK_TEST_HOME/.grok/config.toml"
mcp_config_template: |
[mcp_servers.calculator_via_atryum]
[mcp_servers.$ATRYUM_MCP_ALIAS]
command = "npx"
args = ["-y", "mcp-remote", "$ATRYUM_MCP_URL"]
skip_reason: ""
Expand All @@ -126,7 +127,7 @@ harnesses:
mcp_config_template: |
{
"mcpServers": {
"calculator_via_atryum": {
"$ATRYUM_MCP_ALIAS": {
"command": "npx",
"args": ["-y", "mcp-remote", "$ATRYUM_MCP_URL"]
}
Expand Down
3 changes: 3 additions & 0 deletions integrations/lib/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,9 @@ reset_case_runtime() {
MOCK_OIDC_PORT=""
export ATRYUM_PORT MOCK_OIDC_PORT
unset ATRYUM_URL MOCK_OIDC_ISSUER ATRYUM_BEARER_TOKEN MCP_REMOTE_HEADERS || true
unset CODEX_TEST_HOME GROK_TEST_HOME AMP_SETTINGS_FILE || true
unset CLAUDE_MCP_CONFIG CLAUDE_CONFIG_DIR CURSOR_MCP_CONFIG CURSOR_RUN_DIR || true
unset HARNESS_MCP_CONFIG || true

mkdir -p "$RUN_DIR" "$RESULTS_DIR" "$HARNESS_CONFIG_DIR"
}
Expand Down
86 changes: 69 additions & 17 deletions integrations/lib/harness.sh
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ harness_available() {
if [[ "$harness_id" == "amp" ]] && amp threads list --json >/dev/null 2>&1; then
return 0
fi
# cursor-agent may use subscription login without CURSOR_API_KEY.
if [[ "$harness_id" == "cursor" ]] && cursor-agent status >/dev/null 2>&1; then
return 0
fi
warn "skip $harness_id: env $key_env is unset"
return 1
fi
Expand Down Expand Up @@ -105,8 +109,8 @@ EOF
export CLAUDE_CONFIG_DIR="$HARNESS_CONFIG_DIR/claude"
;;
cursor)
mkdir -p "$HARNESS_CONFIG_DIR/cursor"
cat >"$HARNESS_CONFIG_DIR/cursor/hooks.json" <<EOF
mkdir -p "$RUN_DIR/.cursor"
cat >"$RUN_DIR/.cursor/hooks.json" <<EOF
{
"hooks": {
"preToolUse": [{
Expand All @@ -120,7 +124,7 @@ EOF
}
}
EOF
export CURSOR_HOOKS_FILE="$HARNESS_CONFIG_DIR/cursor/hooks.json"
export CURSOR_RUN_DIR="$RUN_DIR"
;;
esac
;;
Expand All @@ -144,9 +148,40 @@ EOF
esac
}

render_mcp_config() {
local template="$1" mcp_url="$2" mcp_alias="$3" dest="$4"
printf '%s\n' "$template" \
| sed -e "s|\$ATRYUM_MCP_URL|${mcp_url}|g" \
-e "s|\$ATRYUM_MCP_ALIAS|${mcp_alias}|g" \
>"$dest"
if [[ -n "${MCP_REMOTE_HEADERS:-}" ]]; then
printf '\n[mcp_servers.%s.env]\nMCP_REMOTE_HEADERS = "%s"\n' \
"$mcp_alias" "$MCP_REMOTE_HEADERS" >>"$dest"
fi
}

render_mcp_json_config() {
local mcp_alias="$1" mcp_url="$2" dest="$3"
local py="${INTEGRATIONS_PYTHON:-python3}"
MCP_REMOTE_HEADERS="${MCP_REMOTE_HEADERS:-}" "$py" - "$mcp_alias" "$mcp_url" "$dest" <<'PY'
import json, os, sys
alias, url, dest = sys.argv[1], sys.argv[2], sys.argv[3]
headers = os.environ.get("MCP_REMOTE_HEADERS", "")
entry = {
"command": "npx",
"args": ["-y", "mcp-remote", url],
}
if headers:
entry["env"] = {"MCP_REMOTE_HEADERS": headers}
with open(dest, "w", encoding="utf-8") as f:
json.dump({"mcpServers": {alias: entry}}, f, indent=2)
f.write("\n")
PY
}

configure_harness_mcp() {
local harness_id="$1" auth_id="$2" target_id="$3"
local template path transport server_name mcp_url dest py="${INTEGRATIONS_PYTHON:-python3}"
local template path transport server_name mcp_alias mcp_url dest py="${INTEGRATIONS_PYTHON:-python3}"
template="$("$py" "$INTEGRATIONS_ROOT/lib/registry.py" \
"$INTEGRATIONS_ROOT/config/harnesses.yaml" harnesses "$harness_id" mcp_config_template)"
path="$("$py" "$INTEGRATIONS_ROOT/lib/registry.py" \
Expand All @@ -155,19 +190,20 @@ configure_harness_mcp() {
"$INTEGRATIONS_ROOT/config/harnesses.yaml" harnesses "$harness_id" mcp_transport)"

server_name="$(render_upstream_block "$target_id" | awk -F'"' '/^name =/{print $2; exit}')"
mcp_alias="${server_name}_via_atryum"

if [[ "$transport" == "direct-jsonrpc" ]]; then
export ATRYUM_MCP_SERVER="$server_name"
return 0
fi

export CODEX_TEST_HOME="${CODEX_TEST_HOME:-$HARNESS_CONFIG_DIR/codex-home}"
export GROK_TEST_HOME="${GROK_TEST_HOME:-$HARNESS_CONFIG_DIR/grok-home}"
export AMP_SETTINGS_FILE="${AMP_SETTINGS_FILE:-$HARNESS_CONFIG_DIR/amp/settings.json}"

mcp_remote_headers_env "$auth_id" || true
mcp_url="$(build_mcp_url "$auth_id" "$harness_id" "$server_name")"

export CODEX_TEST_HOME="$HARNESS_CONFIG_DIR/codex-home"
export GROK_TEST_HOME="$HARNESS_CONFIG_DIR/grok-home"
export AMP_SETTINGS_FILE="$HARNESS_CONFIG_DIR/amp/settings.json"

case "$transport" in
amp-settings-json)
mkdir -p "$(dirname "$AMP_SETTINGS_FILE")"
Expand All @@ -176,7 +212,7 @@ configure_harness_mcp() {
if [[ -n "${MCP_REMOTE_HEADERS:-}" ]]; then
amp_env+=(--env "MCP_REMOTE_HEADERS=${MCP_REMOTE_HEADERS}")
fi
if ! AMP_SETTINGS_FILE="$AMP_SETTINGS_FILE" amp mcp add calculator_via_atryum \
if ! AMP_SETTINGS_FILE="$AMP_SETTINGS_FILE" amp mcp add "$mcp_alias" \
"${amp_env[@]}" \
-- npx -y mcp-remote "$mcp_url"; then
warn "amp mcp add failed for $AMP_SETTINGS_FILE"
Expand Down Expand Up @@ -205,17 +241,31 @@ configure_harness_mcp() {
if [[ -n "${MCP_REMOTE_HEADERS:-}" ]]; then
grok_env+=(--env "MCP_REMOTE_HEADERS=${MCP_REMOTE_HEADERS}")
fi
HOME="$GROK_TEST_HOME" grok mcp add calculator_via_atryum \
HOME="$GROK_TEST_HOME" grok mcp add "$mcp_alias" \
"${grok_env[@]}" \
--command npx \
--args -y mcp-remote "$mcp_url" >/dev/null 2>&1 || {
warn "grok mcp add failed; writing config.toml fallback"
dest="$GROK_TEST_HOME/.grok/config.toml"
mkdir -p "$(dirname "$dest")"
printf '%s\n' "$template" | sed "s|\$ATRYUM_MCP_URL|${mcp_url}|g" >"$dest"
render_mcp_config "$template" "$mcp_url" "$mcp_alias" "$dest"
}
return 0
;;
mcp-remote-stdio)
if [[ -z "$path" || "$path" == "null" ]]; then
return 0
fi
if [[ "$harness_id" == "cursor" ]]; then
mkdir -p "$RUN_DIR/.cursor"
dest="$RUN_DIR/.cursor/mcp.json"
export CURSOR_RUN_DIR="$RUN_DIR"
else
dest="$HARNESS_CONFIG_DIR/$(basename "$path")"
mkdir -p "$(dirname "$dest")"
fi
render_mcp_json_config "$mcp_alias" "$mcp_url" "$dest"
;;
*)
if [[ -z "$template" || "$template" == "null" || -z "$path" || "$path" == "null" ]]; then
return 0
Expand All @@ -225,8 +275,8 @@ configure_harness_mcp() {
;;
esac

if [[ -n "${dest:-}" && -n "$template" && "$template" != "null" ]]; then
printf '%s\n' "$template" | sed "s|\$ATRYUM_MCP_URL|${mcp_url}|g" >"$dest"
if [[ -n "${dest:-}" && -n "$template" && "$template" != "null" && "$transport" != "mcp-remote-stdio" ]]; then
render_mcp_config "$template" "$mcp_url" "$mcp_alias" "$dest"
fi

case "$harness_id" in
Expand All @@ -235,6 +285,7 @@ configure_harness_mcp() {
;;
cursor)
export CURSOR_MCP_CONFIG="$dest"
export CURSOR_RUN_DIR="$RUN_DIR"
;;
*)
export HARNESS_MCP_CONFIG="$dest"
Expand All @@ -259,10 +310,10 @@ print("|".join(item["verify"]["expect_substrings"]))
PY
)"

export CODEX_TEST_HOME="${CODEX_TEST_HOME:-$HARNESS_CONFIG_DIR/codex-home}"
export GROK_TEST_HOME="${GROK_TEST_HOME:-$HARNESS_CONFIG_DIR/grok-home}"
export AMP_SETTINGS_FILE="${AMP_SETTINGS_FILE:-$HARNESS_CONFIG_DIR/amp/settings.json}"
export CLAUDE_MCP_CONFIG="${CLAUDE_MCP_CONFIG:-}"
export CODEX_TEST_HOME="$HARNESS_CONFIG_DIR/codex-home"
export GROK_TEST_HOME="$HARNESS_CONFIG_DIR/grok-home"
export AMP_SETTINGS_FILE="$HARNESS_CONFIG_DIR/amp/settings.json"
unset CLAUDE_MCP_CONFIG CLAUDE_CONFIG_DIR CURSOR_MCP_CONFIG CURSOR_RUN_DIR || true

install_hook "$harness_id"
configure_harness_mcp "$harness_id" "$auth_id" "$target_id"
Expand Down Expand Up @@ -315,6 +366,7 @@ PY
invoke="${invoke//\$GROK_TEST_HOME/$GROK_TEST_HOME}"
invoke="${invoke//\$CLAUDE_MCP_CONFIG/${CLAUDE_MCP_CONFIG:-}}"
invoke="${invoke//\$AMP_SETTINGS_FILE/${AMP_SETTINGS_FILE:-}}"
invoke="${invoke//\$CURSOR_RUN_DIR/${CURSOR_RUN_DIR:-}}"
local harness_timeout="${HARNESS_TIMEOUT_SECONDS:-180}"
log "Command: $invoke (timeout=${harness_timeout}s)"
# shellcheck disable=SC2086
Expand Down
Loading