test-e2e.yml

name: E2E test with video recording

on:
  workflow_dispatch:
    inputs:
      ref:
        description: Branch or SHA to test
        required: false
        default: ""
      test_filter:
        description: "Test class or class/method (e.g. UpdatePillUITests or UpdatePillUITests/testSomething)"
        required: true
      test_timeout:
        description: "Per-test timeout in seconds"
        required: false
        default: "120"
      record_video:
        description: Record the virtual display during tests
        required: false
        default: true
        type: boolean
      runner:
        description: "Runner OS (Depot runners for GUI activation support)"
        required: false
        default: "depot-macos-latest"
        type: choice
        options:
          - depot-macos-latest
          - depot-macos-14

jobs:
  e2e:
    runs-on: ${{ inputs.runner || 'depot-macos-latest' }}
    timeout-minutes: 20
    env:
      TEST_REF: ${{ inputs.ref || github.ref }}
    steps:
      - name: Checkout
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          ref: ${{ inputs.ref || github.ref }}
          submodules: recursive

      - name: Capture SHA
        id: sha
        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"

      - name: Select Xcode
        run: |
          set -euo pipefail
          if [ -d "/Applications/Xcode.app/Contents/Developer" ]; then
            XCODE_DIR="/Applications/Xcode.app/Contents/Developer"
          else
            XCODE_APP="$(ls -d /Applications/Xcode*.app 2>/dev/null | head -n 1 || true)"
            if [ -n "$XCODE_APP" ]; then
              XCODE_DIR="$XCODE_APP/Contents/Developer"
            else
              echo "No Xcode.app found under /Applications" >&2
              exit 1
            fi
          fi
          echo "DEVELOPER_DIR=$XCODE_DIR" >> "$GITHUB_ENV"
          export DEVELOPER_DIR="$XCODE_DIR"
          xcodebuild -version
          xcrun --sdk macosx --show-sdk-path

      - name: Download pre-built GhosttyKit.xcframework
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail
          GHOSTTY_SHA=$(git -C ghostty rev-parse HEAD)
          TAG="xcframework-$GHOSTTY_SHA"
          URL="https://github.com/Jesssullivan/ghostty/releases/download/$TAG/GhosttyKit.xcframework.tar.gz"
          echo "Downloading xcframework for ghostty $GHOSTTY_SHA"
          MAX_RETRIES=30
          RETRY_DELAY=20
          for i in $(seq 1 $MAX_RETRIES); do
            if curl -fSL -o GhosttyKit.xcframework.tar.gz "$URL"; then
              echo "Download succeeded on attempt $i"
              break
            fi
            if [ "$i" -eq "$MAX_RETRIES" ]; then
              echo "Failed to download xcframework after $MAX_RETRIES attempts" >&2
              exit 1
            fi
            echo "Attempt $i/$MAX_RETRIES failed, retrying in ${RETRY_DELAY}s..."
            sleep $RETRY_DELAY
          done
          tar xzf GhosttyKit.xcframework.tar.gz
          rm GhosttyKit.xcframework.tar.gz
          test -d GhosttyKit.xcframework

      - name: Install zig
        run: |
          ZIG_REQUIRED="0.15.2"
          if command -v zig >/dev/null 2>&1 && zig version 2>/dev/null | grep -q "^${ZIG_REQUIRED}"; then
            echo "zig ${ZIG_REQUIRED} already installed"
          else
            echo "Installing zig ${ZIG_REQUIRED} from tarball"
            curl -fSL "https://ziglang.org/download/${ZIG_REQUIRED}/zig-aarch64-macos-${ZIG_REQUIRED}.tar.xz" -o /tmp/zig.tar.xz
            tar xf /tmp/zig.tar.xz -C /tmp
            sudo mkdir -p /usr/local/bin /usr/local/lib
            sudo cp -f /tmp/zig-aarch64-macos-${ZIG_REQUIRED}/zig /usr/local/bin/zig
            sudo cp -rf /tmp/zig-aarch64-macos-${ZIG_REQUIRED}/lib /usr/local/lib/zig
            export PATH="/usr/local/bin:$PATH"
            zig version
          fi

      - name: Create virtual display
        if: ${{ inputs.test_filter != 'DisplayResolutionRegressionUITests' }}
        run: |
          set -euo pipefail
          echo "=== Display before ==="
          system_profiler SPDisplaysDataType 2>/dev/null || echo "(none)"
          echo ""
          clang -framework Foundation -framework CoreGraphics \
            -o /tmp/create-virtual-display scripts/create-virtual-display.m
          /tmp/create-virtual-display &
          VDISPLAY_PID=$!
          echo "VDISPLAY_PID=$VDISPLAY_PID" >> "$GITHUB_ENV"
          sleep 3
          echo "=== Display after ==="
          system_profiler SPDisplaysDataType 2>/dev/null || echo "(none)"

      - name: Install ffmpeg
        if: ${{ inputs.record_video }}
        run: |
          brew install --quiet ffmpeg
          FFMPEG_PATH=$(which ffmpeg)
          echo "ffmpeg: $FFMPEG_PATH"
          ffmpeg -version | head -1
          echo "FFMPEG_PATH=$FFMPEG_PATH" >> "$GITHUB_ENV"

      - name: Grant TCC screen recording permission
        if: ${{ inputs.record_video }}
        continue-on-error: true
        run: |
          FFMPEG_BIN="${FFMPEG_PATH:-/opt/homebrew/bin/ffmpeg}"

          # System-level TCC database (where kTCCServiceScreenCapture lives)
          SYS_TCC="/Library/Application Support/com.apple.TCC/TCC.db"
          if [ -f "$SYS_TCC" ]; then
            echo "Granting screen capture in system TCC database"
            for client in "$FFMPEG_BIN" /opt/homebrew/bin/ffmpeg /usr/local/bin/ffmpeg /usr/sbin/screencapture; do
              sudo sqlite3 "$SYS_TCC" \
                "INSERT OR REPLACE INTO access (service, client, client_type, auth_value, auth_reason, auth_version, csreq, policy_id, indirect_object_identifier_type, indirect_object_identifier, indirect_object_code_identity, flags, last_modified) VALUES ('kTCCServiceScreenCapture', '$client', 1, 2, 4, 1, NULL, NULL, 0, 'UNUSED', NULL, 0, $(date +%s));" 2>&1 || echo "  (failed for $client)"
            done
          fi

          # User-level TCC database (fallback)
          USER_TCC="$HOME/Library/Application Support/com.apple.TCC/TCC.db"
          if [ -f "$USER_TCC" ]; then
            echo "Granting screen capture in user TCC database"
            for client in "$FFMPEG_BIN" /opt/homebrew/bin/ffmpeg /usr/local/bin/ffmpeg; do
              sqlite3 "$USER_TCC" \
                "INSERT OR REPLACE INTO access (service, client, client_type, auth_value, auth_reason, auth_version, csreq, policy_id, indirect_object_identifier_type, indirect_object_identifier, indirect_object_code_identity, flags, last_modified) VALUES ('kTCCServiceScreenCapture', '$client', 1, 2, 4, 1, NULL, NULL, 0, 'UNUSED', NULL, 0, $(date +%s));" 2>&1 || echo "  (failed for $client)"
            done
          fi

          # Suppress Sequoia's ScreenCaptureApprovals prompt by pre-dating approval
          APPROVALS_PLIST="$HOME/Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist"
          if [ -d "$(dirname "$APPROVALS_PLIST")" ]; then
            echo "Pre-dating ScreenCaptureApprovals"
            # Set approval date far in the future so the monthly prompt never fires
            defaults write "$APPROVALS_PLIST" "$FFMPEG_BIN" -date "3000-01-01T00:00:00Z" 2>&1 || echo "  (failed)"
          fi

      - name: Clean DerivedData
        run: rm -rf ~/Library/Developer/Xcode/DerivedData/GhosttyTabs-*

      - name: Cache Swift packages
        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4
        with:
          path: .ci-source-packages
          key: spm-${{ inputs.runner || 'depot-macos-latest' }}-${{ hashFiles('GhosttyTabs.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}
          restore-keys: spm-${{ inputs.runner || 'depot-macos-latest' }}-

      - name: Resolve Swift packages
        run: |
          set -euo pipefail
          SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages"
          mkdir -p "$SOURCE_PACKAGES_DIR"
          for attempt in 1 2 3; do
            if xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux-unit -configuration Debug \
              -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \
              -resolvePackageDependencies; then
              exit 0
            fi
            if [ "$attempt" -eq 3 ]; then
              echo "Failed to resolve Swift packages after 3 attempts" >&2
              exit 1
            fi
            echo "Package resolution failed on attempt $attempt, retrying..."
            sleep $((attempt * 5))
          done

      - name: Run UI tests
        id: tests
        env:
          TEST_FILTER: ${{ inputs.test_filter }}
          TEST_TIMEOUT: ${{ inputs.test_timeout || '120' }}
          RECORD_VIDEO: ${{ inputs.record_video }}
        run: |
          set -euo pipefail
          SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages"
          ONLY_TESTING="-only-testing:cmuxUITests/$TEST_FILTER"
          DISPLAY_ENV_PREFIX=()

          if [ "$TEST_FILTER" = "DisplayResolutionRegressionUITests" ]; then
            HELPER_PATH="/tmp/create-virtual-display"
            MANIFEST_PATH="/tmp/cmux-ui-test-display-harness.json"

            rm -f "$MANIFEST_PATH"
            trap 'rm -f "$MANIFEST_PATH"' EXIT

            clang -framework Foundation -framework CoreGraphics \
              -o "$HELPER_PATH" scripts/create-virtual-display.m

            printf '%s\n' "{\"helperBinaryPath\":\"$HELPER_PATH\"}" > "$MANIFEST_PATH"
          fi

          # Start recording right before the test (after build/resolve).
          # The display churn regression creates its own virtual display above,
          # so recording must start after that harness is ready.
          if [ "$RECORD_VIDEO" = "true" ]; then
            DEVLIST=$( ffmpeg -f avfoundation -list_devices true -i "" 2>&1 || true )
            echo "Available devices:"
            echo "$DEVLIST" | grep -E "AVFoundation|Capture screen"

            SCREEN_INDEX=$( echo "$DEVLIST" | grep "Capture screen" | head -1 \
              | sed 's/.*\[\([0-9]*\)\].*/\1/' )
            SCREEN_INDEX="${SCREEN_INDEX:-0}"
            echo "Using screen device index: $SCREEN_INDEX"

            ffmpeg -f avfoundation -framerate 10 -capture_cursor 1 \
              -i "${SCREEN_INDEX}:none" \
              -c:v libx264 -preset ultrafast -pix_fmt yuv420p \
              /tmp/test-recording-raw.mp4 </dev/null >/tmp/ffmpeg.log 2>&1 &
            RECORD_PID=$!
            echo "RECORD_PID=$RECORD_PID" >> "$GITHUB_ENV"
            sleep 2

            if kill -0 "$RECORD_PID" 2>/dev/null; then
              echo "Recording started (PID $RECORD_PID)"
            else
              echo "::warning::ffmpeg screen recording failed to start"
              cat /tmp/ffmpeg.log
            fi
          fi

          XCODEBUILD_CMD=(
            xcodebuild -project GhosttyTabs.xcodeproj -scheme cmux -configuration Debug
            -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR"
            -disableAutomaticPackageResolution
            -destination "platform=macOS"
            -maximum-test-execution-time-allowance "$TEST_TIMEOUT"
            $ONLY_TESTING
            test
          )

          set +e
          if [ "${#DISPLAY_ENV_PREFIX[@]}" -gt 0 ]; then
            OUTPUT=$(env "${DISPLAY_ENV_PREFIX[@]}" "${XCODEBUILD_CMD[@]}" 2>&1)
          else
            OUTPUT=$("${XCODEBUILD_CMD[@]}" 2>&1)
          fi
          EXIT_CODE=$?
          set -e

          echo "$OUTPUT"

          # Save summary for the issue
          SUMMARY=$(echo "$OUTPUT" | grep -E "(Test Suite|Executed|FAIL|PASS)" | tail -20)
          {
            echo "test_summary<<EOFSUM"
            echo "$SUMMARY"
            echo "EOFSUM"
          } >> "$GITHUB_OUTPUT"

          if [ "$EXIT_CODE" -eq 0 ]; then
            echo "test_result=passed" >> "$GITHUB_OUTPUT"
          else
            echo "test_result=failed" >> "$GITHUB_OUTPUT"
            # Save full output for the issue body
            {
              echo "test_output<<EOFOUT"
              echo "$OUTPUT" | tail -200
              echo "EOFOUT"
            } >> "$GITHUB_OUTPUT"
            exit 1
          fi

      - name: Stop recording and trim
        if: ${{ always() && inputs.record_video && env.RECORD_PID != '' }}
        run: |
          # Stop ffmpeg cleanly
          kill -INT "$RECORD_PID" 2>/dev/null || true
          for i in $(seq 1 15); do
            if ! kill -0 "$RECORD_PID" 2>/dev/null; then
              echo "Recording stopped after ${i}s"
              break
            fi
            sleep 1
          done
          kill -9 "$RECORD_PID" 2>/dev/null || true

          echo "=== raw recording ==="
          ls -lh /tmp/test-recording-raw.mp4 2>/dev/null || { echo "No recording file"; exit 0; }

          # Trim: detect first non-black frame and cut from there
          BLACK_END=$(ffmpeg -i /tmp/test-recording-raw.mp4 \
            -vf "blackdetect=d=0.3:pic_th=0.95:pix_th=0.1" \
            -an -f null - 2>&1 \
            | grep "black_end" | tail -1 \
            | sed 's/.*black_end:\([0-9.]*\).*/\1/' || true)

          if [ -n "$BLACK_END" ] && [ "$BLACK_END" != "0" ]; then
            echo "Trimming ${BLACK_END}s of black frames from start"
            ffmpeg -y -i /tmp/test-recording-raw.mp4 -ss "$BLACK_END" \
              -c:v libx264 -preset ultrafast -pix_fmt yuv420p \
              /tmp/test-recording.mp4 2>/dev/null
          else
            echo "No black frames detected, using raw recording"
            mv /tmp/test-recording-raw.mp4 /tmp/test-recording.mp4
          fi

          echo "=== final recording ==="
          ls -lh /tmp/test-recording.mp4
          # Print duration
          ffprobe -v error -show_entries format=duration \
            -of default=noprint_wrappers=1:nokey=1 /tmp/test-recording.mp4 2>/dev/null \
            | xargs -I{} echo "Duration: {}s"

      - name: Upload recording artifact
        if: ${{ always() && inputs.record_video }}
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: test-recording
          path: /tmp/test-recording.mp4
          if-no-files-found: warn

      - name: Post results to cmux-dev-artifacts
        if: always()
        env:
          GH_TOKEN: ${{ secrets.DEV_ARTIFACTS_TOKEN }}
          TEST_RESULT: ${{ steps.tests.outputs.test_result || 'failed' }}
          TEST_SUMMARY: ${{ steps.tests.outputs.test_summary }}
          TEST_OUTPUT: ${{ steps.tests.outputs.test_output }}
          TEST_FILTER: ${{ inputs.test_filter }}
          COMMIT_SHA: ${{ steps.sha.outputs.sha }}
          RUN_ID: ${{ github.run_id }}
          RECORD_VIDEO: ${{ inputs.record_video }}
        run: |
          set -euo pipefail

          LABEL="$TEST_RESULT"
          if [ "$TEST_RESULT" = "passed" ]; then
            STATUS_EMOJI="PASSED"
          else
            STATUS_EMOJI="FAILED"
          fi

          REF_DISPLAY="${{ inputs.ref || github.ref_name }}"
          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/$RUN_ID"
          ARTIFACT_URL="$RUN_URL#artifacts"

          BODY="**Status:** $STATUS_EMOJI
          **Ref:** \`$REF_DISPLAY\`
          **SHA:** [\`${COMMIT_SHA:0:12}\`](https://github.com/${{ github.repository }}/commit/$COMMIT_SHA)
          **Test:** \`$TEST_FILTER\`
          **Workflow run:** $RUN_URL"

          if [ "$RECORD_VIDEO" = "true" ]; then
            BODY="$BODY
          **Recording:** [Download from artifacts]($ARTIFACT_URL)"
          fi

          if [ -n "$TEST_OUTPUT" ]; then
            BODY="$BODY

          <details><summary>Test output (last 200 lines)</summary>

          \`\`\`
          $TEST_OUTPUT
          \`\`\`

          </details>"
          fi

          if [ -n "$TEST_SUMMARY" ]; then
            BODY="$BODY

          \`\`\`
          $TEST_SUMMARY
          \`\`\`"
          fi

          ISSUE_URL=$(gh issue create \
            --repo manaflow-ai/cmux-dev-artifacts \
            --title "[$STATUS_EMOJI] $TEST_FILTER @ ${COMMIT_SHA:0:7} ($REF_DISPLAY)" \
            --body "$BODY" \
            --label "$LABEL")

          echo "Issue posted: $ISSUE_URL"
          echo "::notice title=Test Result Issue::$ISSUE_URL"