staging <- dev

.github/workflows/benchmark-live.yml

@@ -0,0 +1,146 @@
+name: Benchmark Live
+
+on:
+  push:
+    branches:
+      - staging
+      - dev
+  workflow_dispatch:
+    inputs:
+      target_environment:
+        description: GitHub benchmark environment to use
+        required: true
+        type: choice
+        default: staging
+        options:
+          - dev
+          - staging
+      server_url:
+        description: MemWal server URL. Empty uses BENCH_SERVER_URL environment variable.
+        required: false
+        default: ''
+      namespace:
+        description: Memory namespace
+        required: false
+        default: benchmark
+      remember_text:
+        description: Remember benchmark text
+        required: false
+        default: benchmark memory
+      query:
+        description: Recall query text
+        required: false
+        default: benchmark memory
+      limit:
+        description: Top-K recall result limit
+        required: false
+        default: '5'
+      remember_runs:
+        description: Remember runs
+        required: false
+        default: '3'
+      cold_runs:
+        description: Cold-path runs
+        required: false
+        default: '3'
+      warm_runs:
+        description: Warm-path runs
+        required: false
+        default: '10'
+  schedule:
+    - cron: '0 9 * * 1'
+
+concurrency: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  memory-api:
+    name: Memory API Latency
+    runs-on: ubuntu-latest
+    environment:
+      name: benchmark-${{ github.event_name == 'workflow_dispatch' && inputs.target_environment || (github.ref_name == 'staging' && 'staging' || 'dev') }}
+    timeout-minutes: 30
+
+    env:
+      SERVER_URL: ${{ github.event.inputs.server_url || vars.BENCH_SERVER_URL }}
+      NAMESPACE: ${{ github.event.inputs.namespace || 'benchmark' }}
+      REMEMBER_TEXT: ${{ github.event.inputs.remember_text || 'benchmark memory' }}
+      QUERY: ${{ github.event.inputs.query || 'benchmark memory' }}
+      LIMIT: ${{ github.event.inputs.limit || '5' }}
+      REMEMBER_RUNS: ${{ github.event.inputs.remember_runs || '3' }}
+      COLD_RUNS: ${{ github.event.inputs.cold_runs || '3' }}
+      WARM_RUNS: ${{ github.event.inputs.warm_runs || '10' }}
+      BENCH_DELEGATE_KEY: ${{ secrets.BENCH_DELEGATE_KEY }}
+      BENCH_ACCOUNT_ID: ${{ secrets.BENCH_ACCOUNT_ID }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+          cache: npm
+          cache-dependency-path: services/server/scripts/package-lock.json
+
+      - name: Install sidecar script dependencies
+        working-directory: services/server/scripts
+        run: npm ci
+
+      - name: Run memory API latency benchmark
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          missing=0
+          for name in SERVER_URL BENCH_DELEGATE_KEY BENCH_ACCOUNT_ID; do
+            if [ -z "${!name:-}" ]; then
+              echo "::error::${name} is required"
+              missing=1
+            fi
+          done
+          if [ "$missing" -ne 0 ]; then
+            exit 1
+          fi
+
+          mkdir -p benchmark-results
+          services/server/scripts/node_modules/.bin/tsx services/server/scripts/bench-recall-latency.ts \
+            --server-url "$SERVER_URL" \
+            --delegate-key "$BENCH_DELEGATE_KEY" \
+            --account-id "$BENCH_ACCOUNT_ID" \
+            --remember-text "$REMEMBER_TEXT" \
+            --query "$QUERY" \
+            --namespace "$NAMESPACE" \
+            --limit "$LIMIT" \
+            --remember-runs "$REMEMBER_RUNS" \
+            --cold-runs "$COLD_RUNS" \
+            --warm-runs "$WARM_RUNS" \
+            --output benchmark-results/memory-api.json \
+            --no-color
+
+      - name: Write benchmark summary
+        if: always()
+        shell: bash
+        run: |
+          if [ ! -f benchmark-results/memory-api.json ]; then
+            echo "No benchmark result file was produced." >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          node <<'NODE' >> "$GITHUB_STEP_SUMMARY"
+          const fs = require('fs');
+          const json = JSON.parse(fs.readFileSync('benchmark-results/memory-api.json', 'utf8'));
+          console.log('## memory API benchmark');
+          console.log('');
+          console.log(json.markdownTable || 'No markdown table found in result JSON.');
+          console.log('');
+          NODE
+
+      - name: Upload benchmark artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results-${{ github.run_id }}
+          path: benchmark-results/memory-api.json
+          if-no-files-found: warn
+          retention-days: 14

.github/workflows/benchmark-smoke.yml

@@ -0,0 +1,68 @@
+name: Benchmark Smoke
+
+on:
+  pull_request:
+    paths:
+      - 'services/server/scripts/**'
+      - '.github/workflows/benchmark-live.yml'
+      - '.github/workflows/benchmark-smoke.yml'
+      - 'docs/relayer/benchmark-ci-setup.md'
+  push:
+    branches:
+      - main
+      - staging
+      - dev
+    paths:
+      - 'services/server/scripts/**'
+      - '.github/workflows/benchmark-live.yml'
+      - '.github/workflows/benchmark-smoke.yml'
+      - 'docs/relayer/benchmark-ci-setup.md'
+  workflow_dispatch:
+
+concurrency: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  smoke:
+    name: Compile & CLI Smoke
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: services/server
+
+      - name: Cargo check
+        working-directory: services/server
+        run: cargo check
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+          cache: npm
+          cache-dependency-path: services/server/scripts/package-lock.json
+
+      - name: Install sidecar script dependencies
+        working-directory: services/server/scripts
+        run: npm ci
+
+      - name: Typecheck benchmark scripts
+        working-directory: services/server/scripts
+        run: |
+          ./node_modules/.bin/tsc --noEmit --skipLibCheck \
+            --target ES2022 \
+            --module NodeNext \
+            --moduleResolution NodeNext \
+            bench-recall-latency.ts
+
+      - name: Benchmark CLI help smoke
+        working-directory: services/server/scripts
+        run: |
+          ./node_modules/.bin/tsx bench-recall-latency.ts --help