ci.yml

name: CI

on:
  push:
    branches: [master, main]
  pull_request:
  workflow_dispatch:

concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

# Read-only by default. A job that needs more asks for it itself, so a
# compromised dependency in one step cannot push to the repository.
permissions:
  contents: read

env:
  PYTHONUNBUFFERED: "1"
  PIP_DISABLE_PIP_VERSION_CHECK: "1"
  FORCE_COLOR: "1"

jobs:
  lint:
    name: Lint and types
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
          cache: pip
          cache-dependency-path: pyproject.toml
      - run: pip install -e ".[dev]"
      - name: ruff check
        run: ruff check mlango tests
      - name: ruff format --check
        run: ruff format --check mlango tests
      # Blocking, not advisory: the framework hands users declared classes and
      # they type-check their own code against it, so our annotations have to
      # be true. Anything genuinely undecidable belongs in a narrow ignore with
      # a reason, not in a green build that ignores every error.
      - name: mypy
        run: mypy mlango

  test:
    name: Tests (${{ matrix.os }}, Python ${{ matrix.python }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            python: "3.10"
          - os: ubuntu-latest
            python: "3.11"
          - os: ubuntu-latest
            python: "3.12"
          - os: ubuntu-latest
            python: "3.13"
          - os: macos-latest
            python: "3.12"
          - os: windows-latest
            python: "3.12"
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: ${{ matrix.python }}
          cache: pip
          cache-dependency-path: pyproject.toml
      - name: Install with the scikit-learn extra
        run: pip install -e ".[sklearn,dev]"
      # No coverage gate here: the matrix exists to prove the suite passes on
      # every supported interpreter and OS, and a platform-specific branch
      # would make the threshold flap. The `coverage` job below is the gate.
      - name: Run the test suite
        run: pytest -q

  coverage:
    name: Coverage gate
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
          cache: pip
          cache-dependency-path: pyproject.toml
      - run: pip install -e ".[sklearn,dev]"
      # fail_under lives in pyproject.toml so `pytest --cov` enforces the same
      # threshold on a laptop as it does here.
      - name: Measure and enforce
        run: pytest -q --cov --cov-report=xml --cov-report=term-missing
      - uses: actions/upload-artifact@v7
        if: always()
        with:
          name: coverage-xml
          path: coverage.xml
          retention-days: 14

  audit:
    name: Dependency audit
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
          cache: pip
          cache-dependency-path: pyproject.toml
      # Upgrade pip first: a base image can ship an old one, and pip has its own
      # advisories. They are real, but they are the image's, not this package's —
      # auditing a stale build tool tells a user nothing about what they install.
      - run: pip install --upgrade pip
      - run: pip install -e ".[sklearn,dev]" pip-audit
      # Audits what is actually installed, which is what a user gets — not just
      # what the lockfile claims. Advisories appear against transitive
      # dependencies far more often than direct ones.
      #
      # --skip-editable leaves out mlango itself, which is installed from this
      # checkout and so has no release to compare against. --strict cannot be
      # combined with it: strict treats any dependency it could not audit as a
      # failure, including the one we deliberately skipped. The trade is that a
      # third-party package missing from PyPI would be passed over silently
      # rather than failing the build; a real advisory still fails it, which is
      # what this job is for.
      - name: pip-audit
        run: pip-audit --desc --skip-editable

  quickstart:
    name: Scaffolded project works end to end
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
          cache: pip
          cache-dependency-path: pyproject.toml
      # Built and installed, not `pip install -e`. An editable install puts the
      # checkout on the path, so a module missing from the wheel still imports
      # and this job stays green for someone who typed `pip install mlango` and
      # got a broken package.
      #
      # And no `dev` extra. Installing it here made `manage.py test` below pass
      # for a reason no user has: the scaffold's own requirements did not bring
      # a test runner, so following the printed steps exactly ended at "pytest
      # is not installed" while CI said the quickstart worked.
      - name: Build the wheel and install that
        run: |
          pip install build
          python -m build --wheel --outdir dist
          pip install "$(ls dist/*.whl)[sklearn]"

      # The README promises four commands from nothing to a working admin.
      # This job is what keeps that promise honest.
      - name: startproject
        run: mlango startproject demoproject

      # Exactly what the scaffold tells the user to run. It must be enough.
      - name: Install what the scaffold asks for
        working-directory: demoproject
        run: pip install -r requirements-dev.txt

      - name: The framework under test is the installed one
        working-directory: demoproject
        run: |
          python - <<'PY'
          import pathlib

          import mlango

          here = pathlib.Path(mlango.__file__).resolve()
          assert "site-packages" in here.parts, f"testing the checkout, not the wheel: {here}"
          print(f"mlango {mlango.__version__} from {here.parent}")
          PY

      - name: check
        working-directory: demoproject
        run: python manage.py check

      - name: migrate
        working-directory: demoproject
        run: |
          python manage.py migrate
          python manage.py makemigrations
          python manage.py migrate

      - name: train, evaluate, sweep
        working-directory: demoproject
        run: |
          python manage.py train demo.Sentiment
          python manage.py evaluate demo.SentimentAccuracy --min-pass-rate 0.9
          python manage.py sweep demo.Sentiment -p C=0.5,2.0 --metric accuracy --mode max

      - name: agent, runs, traces, dataset
        working-directory: demoproject
        run: |
          python manage.py agent demo.Helper 'hello there'
          python manage.py runs list
          python manage.py traces list
          python manage.py dataset head demo.Reviews -n 3
          python manage.py dataset materialize demo.Reviews

      - name: the project's own test suite
        working-directory: demoproject
        run: python manage.py test

      - name: serve, then probe the admin and the API
        working-directory: demoproject
        run: |
          python manage.py runserver 8099 &
          for _ in $(seq 1 30); do
            curl -sf http://127.0.0.1:8099/api/health >/dev/null && break
            sleep 1
          done
          curl -sf http://127.0.0.1:8099/api/health
          echo
          curl -sf -X POST http://127.0.0.1:8099/api/predict/ \
            -H 'Content-Type: application/json' \
            -d '{"input": "great movie, loved every minute"}'
          echo
          curl -sf -X POST http://127.0.0.1:8099/api/chat/ \
            -H 'Content-Type: application/json' \
            -d '{"message": "hello"}'
          echo
          # A stream route must not take the schema down with it.
          curl -sf http://127.0.0.1:8099/api/openapi.json | python -c 'import json,sys; print(sorted(json.load(sys.stdin)["paths"]))'
          test "$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8099/admin/)" = "200"
          test "$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8099/admin/o/nope.Nope)" = "404"

  transformers:
    name: Transformers trainer fine-tunes a real checkpoint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
          cache: pip
          cache-dependency-path: pyproject.toml
      - name: Install with the transformers extra
        run: pip install -e ".[transformers,dev]"
      - uses: actions/cache@v6
        with:
          path: ~/.cache/huggingface
          key: hf-${{ runner.os }}-tiny-bert

      # The trainer needs real pretrained weights to be exercised at all, so it
      # gets its own job with a deliberately tiny checkpoint.
      - name: Fine-tune a tiny checkpoint end to end
        run: |
          python - <<'PY'
          import tempfile

          from mlango.conf import settings

          settings.configure(
              BASE_DIR=tempfile.mkdtemp(),
              METASTORE={"URL": "sqlite:///ci.db"},
              DEFAULT_CALLBACKS=[],
              SEED=0,
          )

          from mlango.core import fields
          from mlango.data import Dataset, InMemorySource
          from mlango.training import TextClassifier

          ROWS = [
              {
                  "id": i,
                  "text": "this is wonderful" if i % 2 else "this is dreadful",
                  "label": "pos" if i % 2 else "neg",
              }
              for i in range(24)
          ]

          class Reviews(Dataset):
              id = fields.IntegerField()
              text = fields.TextField()
              label = fields.LabelField(["neg", "pos"])

              class Meta:
                  source = InMemorySource(ROWS)
                  primary_key = "id"

          class Sentiment(TextClassifier):
              """Fine-tuned in CI on a tiny checkpoint."""

              base_model = fields.CharField(
                  default="hf-internal-testing/tiny-random-DistilBertForSequenceClassification"
              )
              epochs = fields.IntegerField(default=1)
              batch_size = fields.IntegerField(default=4)
              max_length = fields.IntegerField(default=32)

              class Meta:
                  dataset = Reviews
                  features = ["text"]

          model = Sentiment()
          run = model.train()
          record = run.refresh()
          assert record.status == "finished", record.error
          assert "val_loss" in record.summary or record.summary, record.summary
          print("trained:", {k: v for k, v in record.summary.items() if isinstance(v, (int, float))})

          prediction = model.predict("this is wonderful")
          assert prediction in {"neg", "pos"}, prediction
          proba = model.predict_proba("this is wonderful")
          assert set(proba) == {"neg", "pos"}, proba
          print("predicted:", prediction, proba)

          reloaded = Sentiment.load()
          assert reloaded.predict("this is dreadful") in {"neg", "pos"}
          print("reloaded from the saved checkpoint successfully")
          PY

  build:
    name: Package builds and installs cleanly
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
      - run: pip install build twine
      - name: Build the wheel and sdist
        run: python -m build
      - name: Check the metadata
        run: twine check dist/*
      - name: Install the wheel into a clean environment
        run: |
          python -m venv /tmp/fresh
          /tmp/fresh/bin/pip install dist/*.whl
          /tmp/fresh/bin/mlango --version
          /tmp/fresh/bin/python -c "import mlango; print(mlango.get_version())"
          # py.typed has to ship, or the annotations are invisible downstream.
          /tmp/fresh/bin/python -c "import importlib.resources as r, mlango; assert (r.files(mlango) / 'py.typed').is_file()"
      - uses: actions/upload-artifact@v7
        with:
          name: dist
          path: dist/
          retention-days: 14

  docs:
    name: Documentation builds
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
          cache: pip
          cache-dependency-path: pyproject.toml
      - run: pip install mkdocs-material mkdocs-static-i18n
      # --strict turns a broken link or a missing translation into a failure.
      - name: Build in strict mode
        run: mkdocs build --strict

  ci:
    name: CI
    runs-on: ubuntu-latest
    if: always()
    needs: [lint, test, coverage, audit, quickstart, transformers, build, docs]
    steps:
      # One aggregate check to require in branch protection. Without it, adding
      # a job to the matrix silently leaves it unrequired, and a red job stops
      # blocking merges.
      - name: Fail if any job did not succeed
        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
        run: |
          echo "One or more jobs failed:"
          echo '${{ toJSON(needs) }}'
          exit 1
      - run: echo "All checks passed."