Makefile

SHELL := /bin/bash

# Default configuration
# The scheme for the CLI package is typically "osaurus-cli" (the package name)
SCHEME_CLI := osaurus-cli
SCHEME_APP := osaurus
CONFIG := Release
PROJECT := App/osaurus.xcodeproj
DERIVED := build/DerivedData

.PHONY: help cli app install-cli serve status test ci-test clean bench-setup bench-ingest bench-ingest-chunks bench-run bench evals evals-verbose evals-report evals-all evals-all-verbose evals-all-report

help:
	@echo "Targets:"
	@echo "  cli            Build CLI ($(SCHEME_CLI)) into $(DERIVED)"
	@echo "  app            Build app ($(SCHEME_APP)) and embed CLI"
	@echo "  install-cli    Install/update /usr/local/bin/osaurus symlink"
	@echo "  serve          Build CLI and start server (use PORT=XXXX, EXPOSE=1)"
	@echo "  status         Check if server is running"
	@echo "  bench-setup         Clone EasyLocomo + apply patches + install deps"
	@echo "  bench-ingest        Full LOCOMO ingestion (LLM extraction + chunks)"
	@echo "  bench-ingest-chunks Fast chunk-only backfill (no LLM, ~minutes)"
	@echo "  bench-run           Run LOCOMO benchmark only (skip ingestion)"
	@echo "  bench               Full ingest + run LOCOMO benchmark"
	@echo "  evals               Run one OsaurusEvals suite (MODEL=, FILTER=, EVALS_SUITE=)"
	@echo "  evals-verbose       Same as 'evals' plus per-case raw LLM response (debugging prompt iter)"
	@echo "  evals-report        Same as 'evals' but also writes JSON to EVALS_OUT (build/evals.json)"
	@echo "  evals-all           Run every suite under Packages/OsaurusEvals/Suites/* (MODEL=, FILTER=)"
	@echo "  evals-all-verbose   Same as 'evals-all' plus per-case raw LLM response"
	@echo "  evals-all-report    Same as 'evals-all' but writes per-suite JSON to EVALS_OUT_DIR (build/evals/)"
	@echo "  test           Run OsaurusCore package tests via 'swift test'"
	@echo "  ci-test        Reproduce the CI test-core job locally (xcodebuild + xcbeautify)"
	@echo "  clean          Remove DerivedData build output"

cli:
	@echo "Building CLI ($(SCHEME_CLI))…"
	xcodebuild -project $(PROJECT) -scheme $(SCHEME_CLI) -configuration $(CONFIG) -derivedDataPath $(DERIVED) build -quiet

app: cli
	@echo "Building app ($(SCHEME_APP))…"
	xcodebuild -project $(PROJECT) -scheme $(SCHEME_APP) -configuration $(CONFIG) -derivedDataPath $(DERIVED) build -quiet
	@echo "Embedding CLI into App Bundle (Helpers)…"
	# Copy osaurus-cli to osaurus.app/Contents/Helpers/osaurus
	mkdir -p "$(DERIVED)/Build/Products/$(CONFIG)/osaurus.app/Contents/Helpers"
	cp "$(DERIVED)/Build/Products/$(CONFIG)/osaurus-cli" "$(DERIVED)/Build/Products/$(CONFIG)/osaurus.app/Contents/Helpers/osaurus"
	chmod +x "$(DERIVED)/Build/Products/$(CONFIG)/osaurus.app/Contents/Helpers/osaurus"

install-cli: cli
	@echo "Installing CLI symlink…"
	./scripts/release/install_cli_symlink.sh --dev

serve: install-cli
	@echo "Starting Osaurus server…"
	@if [[ -n "$(PORT)" ]]; then \
		ARGS="$$ARGS --port $(PORT)"; \
	fi; \
	if [[ "$(EXPOSE)" == "1" ]]; then \
		ARGS="$$ARGS --expose"; \
	fi; \
	osaurus serve $$ARGS

status:
	osaurus status

test:
	@echo "Running OsaurusCore tests…"
	swift test --package-path Packages/OsaurusCore

# Mirrors the CI `test-core` job: same xcodebuild flags, same xcbeautify
# pipe, same xcresult bundle. Run this locally to repro a failed CI run.
# After it finishes (pass or fail) you can `open build/Tests.xcresult` to
# get the same Test Navigator UI as Xcode.
ci-test:
	@command -v xcbeautify >/dev/null 2>&1 || { \
		echo "xcbeautify not found. Install with: brew install xcbeautify"; \
		exit 1; \
	}
	@mkdir -p build
	@rm -rf build/Tests.xcresult
	@set -o pipefail; xcodebuild test \
		-workspace osaurus.xcworkspace \
		-scheme OsaurusCoreTests \
		-resultBundlePath build/Tests.xcresult \
		-quiet \
		-skipPackagePluginValidation \
		-skipMacroValidation \
		-enableCodeCoverage NO \
		-test-timeouts-enabled YES \
		-default-test-execution-time-allowance 60 \
		-maximum-test-execution-time-allowance 120 \
		COMPILER_INDEX_STORE_ENABLE=NO \
		SWIFT_COMPILATION_MODE=incremental \
		| xcbeautify --renderer terminal
	@echo ""
	@echo "Done. Inspect failures with: open build/Tests.xcresult"

## ── LOCOMO Benchmark ──────────────────────────────────────────────

BENCH_MODEL ?= openrouter/google/gemini-2.5-flash
BENCH_BASE_URL ?= http://localhost:1337
BENCH_BATCH ?= 20
EASYLOCOMO_REPO ?= https://github.com/playeriv65/EasyLocomo.git
EASYLOCOMO_DIR := benchmarks/EasyLocomo
BENCH_PYTHON := $(EASYLOCOMO_DIR)/.venv/bin/python

bench-setup:
	@echo "Setting up EasyLocomo benchmark…"
	@if [ ! -d "$(EASYLOCOMO_DIR)/.git" ]; then \
		mkdir -p benchmarks && \
		git clone $(EASYLOCOMO_REPO) $(EASYLOCOMO_DIR); \
	else \
		echo "EasyLocomo already cloned."; \
	fi
	@echo "Applying Osaurus patches…"
	cd $(EASYLOCOMO_DIR) && git checkout -- . && git apply ../../scripts/benchmark/easylocomo.patch
	@echo "Installing Python dependencies…"
	cd $(EASYLOCOMO_DIR) && python -m venv .venv && .venv/bin/pip install -q -r requirements.txt
	@echo "Done. Run 'make bench-ingest' then 'make bench-run'."

bench-ingest:
	@echo "Ingesting LOCOMO conversations into Osaurus memory…"
	$(BENCH_PYTHON) scripts/benchmark/ingest_locomo.py --base-url $(BENCH_BASE_URL)

bench-ingest-chunks:
	@echo "Backfilling LOCOMO conversation chunks (no LLM, fast)…"
	$(BENCH_PYTHON) scripts/benchmark/ingest_locomo.py --base-url $(BENCH_BASE_URL) --chunks-only --delay 0

bench-run:
	@echo "Running LOCOMO benchmark (model=$(BENCH_MODEL), no-context, batch=$(BENCH_BATCH))…"
	cd $(EASYLOCOMO_DIR) && .venv/bin/python run_evaluation.py \
		--model $(BENCH_MODEL) \
		--no-context \
		--overwrite \
		--batch-size $(BENCH_BATCH)

bench: bench-ingest bench-run

## ── OsaurusEvals (off-CI behaviour evals) ────────────────────────
# Override on the command line, e.g.
#   make evals MODEL=foundation
#   make evals MODEL=openai/gpt-4o-mini FILTER=browser
#   make evals-report EVALS_OUT=reports/today.json
# Default model is `auto` (whatever ChatConfigurationStore is set to);
# see Packages/OsaurusEvals/README.md for the full --model grammar.

EVALS_ROOT := Packages/OsaurusEvals/Suites
EVALS_SUITE ?= $(EVALS_ROOT)/Preflight
EVALS_OUT ?= build/evals.json
EVALS_OUT_DIR ?= build/evals
# Auto-discovered list of every subdirectory under Suites/. Adding a new
# `Suites/MyDomain/` automatically picks it up here — no Makefile edit
# required when a new suite lands.
EVALS_ALL_SUITES := $(sort $(dir $(wildcard $(EVALS_ROOT)/*/)))

evals:
	@echo "Running OsaurusEvals against $(EVALS_SUITE)…"
	swift run --package-path Packages/OsaurusEvals osaurus-evals run \
		--suite $(EVALS_SUITE) \
		$(if $(MODEL),--model $(MODEL),) \
		$(if $(FILTER),--filter $(FILTER),)

evals-verbose:
	@echo "Running OsaurusEvals (verbose) against $(EVALS_SUITE)…"
	swift run --package-path Packages/OsaurusEvals osaurus-evals run \
		--suite $(EVALS_SUITE) \
		--verbose \
		$(if $(MODEL),--model $(MODEL),) \
		$(if $(FILTER),--filter $(FILTER),)

evals-report:
	@mkdir -p $(dir $(EVALS_OUT))
	swift run --package-path Packages/OsaurusEvals osaurus-evals run \
		--suite $(EVALS_SUITE) \
		$(if $(MODEL),--model $(MODEL),) \
		$(if $(FILTER),--filter $(FILTER),) \
		--out $(EVALS_OUT)
	@echo "Wrote $(EVALS_OUT)"

# Run every suite directory under $(EVALS_ROOT). The CLI exits 1 on any
# failed/errored case, so we run each suite independently (don't `set -e`)
# and aggregate exit codes so a single failure doesn't mask later suites.
# Final exit is non-zero if ANY suite failed.
evals-all:
	@echo "Discovered suites: $(notdir $(patsubst %/,%,$(EVALS_ALL_SUITES)))"
	@rc=0; for suite in $(EVALS_ALL_SUITES); do \
		echo ""; \
		echo "── $$suite ──"; \
		swift run --package-path Packages/OsaurusEvals osaurus-evals run \
			--suite $$suite \
			$(if $(MODEL),--model $(MODEL),) \
			$(if $(FILTER),--filter $(FILTER),) \
			|| rc=$$?; \
	done; \
	exit $$rc

evals-all-verbose:
	@echo "Discovered suites: $(notdir $(patsubst %/,%,$(EVALS_ALL_SUITES)))"
	@rc=0; for suite in $(EVALS_ALL_SUITES); do \
		echo ""; \
		echo "── $$suite ──"; \
		swift run --package-path Packages/OsaurusEvals osaurus-evals run \
			--suite $$suite \
			--verbose \
			$(if $(MODEL),--model $(MODEL),) \
			$(if $(FILTER),--filter $(FILTER),) \
			|| rc=$$?; \
	done; \
	exit $$rc

# Writes one JSON report per suite under $(EVALS_OUT_DIR), named after
# the suite directory. Useful for CI dashboards / cross-run diffing.
evals-all-report:
	@mkdir -p $(EVALS_OUT_DIR)
	@rc=0; for suite in $(EVALS_ALL_SUITES); do \
		name=$$(basename $$suite); \
		out="$(EVALS_OUT_DIR)/$$name.json"; \
		echo ""; \
		echo "── $$suite → $$out ──"; \
		swift run --package-path Packages/OsaurusEvals osaurus-evals run \
			--suite $$suite \
			$(if $(MODEL),--model $(MODEL),) \
			$(if $(FILTER),--filter $(FILTER),) \
			--out $$out \
			|| rc=$$?; \
	done; \
	echo ""; \
	echo "Wrote per-suite reports to $(EVALS_OUT_DIR)/"; \
	exit $$rc

## ── Housekeeping ─────────────────────────────────────────────────

clean:
	rm -rf $(DERIVED)
	@echo "Cleaned $(DERIVED)"