tests.yml

name: tests

on:
  push:
    branches:
      - main
    paths-ignore:
      - '**/*.md'
  pull_request:
    branches:
      - main
    paths-ignore:
      - '**/*.md'
  workflow_dispatch:

permissions:
  contents: read
  actions: write

jobs:
  blockguard-tests:
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Build and install blockguard
        run: pip install livekit-plugins/livekit-blockguard/

      - name: Install pytest
        run: pip install pytest

      - name: Run blockguard tests
        working-directory: livekit-plugins/livekit-blockguard
        run: python -m pytest test_blockguard.py -v

  unit-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install the project
        run: uv sync --all-extras --dev

      - name: Download NLTK data
        run: uv run python -c "import nltk; nltk.download('punkt_tab')"

      - name: Run tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: make unit-tests

  evaluation:
    # don't run tests for PRs on forks
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install the project
        run: uv sync --all-extras --dev

      - name: Run evals
        env:
          LIVEKIT_API_KEY: ${{ secrets.LIVEKIT_API_KEY }}
          LIVEKIT_API_SECRET: ${{ secrets.LIVEKIT_API_SECRET }}

        run: uv run pytest tests/test_workflows.py tests/test_evals.py

  tests:
    # don't run tests for PRs on forks
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false
    strategy:
      fail-fast: false
      matrix:
        plugin:
          [
            cartesia,
            deepgram,
            elevenlabs,
            groq,
            openai,
            inworld,
          ]

    runs-on: ubuntu-latest
    name: livekit-plugins-${{ matrix.plugin }}
    steps:
      - uses: actions/checkout@v6

      - name: Setup Google credentials
        if: matrix.plugin == 'google'
        shell: bash
        run: |
          printf '%s' '${{ secrets.GOOGLE_CREDENTIALS_JSON }}' > tests/google.json

      - name: Run tests
        shell: bash
        env:
          PLUGIN: ${{ matrix.plugin }}
          LIVEKIT_URL: ${{ secrets.LIVEKIT_URL }}
          LIVEKIT_API_KEY: ${{ secrets.LIVEKIT_API_KEY }}
          LIVEKIT_API_SECRET: ${{ secrets.LIVEKIT_API_SECRET }}
          DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
          ELEVEN_API_KEY: ${{ secrets.ELEVEN_API_KEY }}
          CARTESIA_API_KEY: ${{ secrets.CARTESIA_API_KEY }}
          AZURE_SPEECH_KEY: ${{ secrets.AZURE_SPEECH_KEY }}
          AZURE_SPEECH_REGION: ${{ secrets.AZURE_SPEECH_REGION }} # nit: doesn't have to be secret
          GOOGLE_CREDENTIALS_JSON: ${{ secrets.GOOGLE_CREDENTIALS_JSON }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
          ASSEMBLYAI_API_KEY: ${{ secrets.ASSEMBLYAI_API_KEY }}
          FAL_KEY: ${{ secrets.FAL_KEY }}
          PLAYHT_API_KEY: ${{ secrets.PLAYHT_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          PLAYHT_USER_ID: ${{ secrets.PLAYHT_USER_ID }}
          RIME_API_KEY: ${{ secrets.RIME_API_KEY }}
          SPEECHMATICS_API_KEY: ${{ secrets.SPEECHMATICS_API_KEY }}
          GOOGLE_APPLICATION_CREDENTIALS: tests/google.json
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          NEUPHONIC_API_KEY: ${{ secrets.NEUPHONIC_API_KEY }}
          RESEMBLE_API_KEY: ${{ secrets.RESEMBLE_API_KEY }}
          SPEECHIFY_API_KEY: ${{ secrets.SPEECHIFY_API_KEY }}
          HUME_API_KEY: ${{ secrets.HUME_API_KEY }}
          SPITCH_API_KEY: ${{ secrets.SPITCH_API_KEY }}
          LMNT_API_KEY: ${{ secrets.LMNT_API_KEY }}
          INWORLD_API_KEY: ${{ secrets.INWORLD_API_KEY }}
          SONIOX_API_KEY: ${{ secrets.SONIOX_API_KEY }}
        working-directory: tests
        run: make test

      - name: Upload LiveKit dump
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: lk-dump-${{ matrix.plugin }}
          path: lk_dump/**
          if-no-files-found: ignore
          retention-days: 1

  aggregate-dumps:
    if: always()
    needs: tests
    runs-on: ubuntu-latest

    steps:
      - name: download all dump shards
        uses: actions/download-artifact@v4
        with:
          pattern: lk-dump-*
          path: all_lk_dump
          merge-multiple: true

      - name: create tarball
        id: tar
        run: |
          if [ -d "all_lk_dump" ] && [ "$(ls -A all_lk_dump)" ]; then
            tar -czf lk-dump-all.tar.gz -C all_lk_dump .
            echo "created=true" >> "$GITHUB_OUTPUT"
          else
            echo "No dump shards found, skipping tarball creation."
            echo "created=false" >> "$GITHUB_OUTPUT"
          fi

      - name: upload merged dump
        if: steps.tar.outputs.created == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: lk-dump-all
          path: lk-dump-all.tar.gz
          retention-days: 5