diff --git a/.changeset/0234-auto-dotenv-support.md b/.changeset/0234-auto-dotenv-support.md new file mode 100644 index 00000000..97589b1b --- /dev/null +++ b/.changeset/0234-auto-dotenv-support.md @@ -0,0 +1,5 @@ +--- +"evalite": minor +--- + +Support .env files by default via dotenv/config. Environment variables from .env files are now automatically loaded without any configuration needed. Users no longer need to manually add `setupFiles: ["dotenv/config"]` to their evalite.config.ts. diff --git a/apps/evalite-docs/src/content/docs/guides/environment-variables.mdx b/apps/evalite-docs/src/content/docs/guides/environment-variables.mdx index 24e15b49..a39c3946 100644 --- a/apps/evalite-docs/src/content/docs/guides/environment-variables.mdx +++ b/apps/evalite-docs/src/content/docs/guides/environment-variables.mdx @@ -8,6 +8,8 @@ To call your LLM from a third-party service, you'll likely need some environment ## Setting Up Env Variables +Evalite automatically loads environment variables from `.env` files in your project root. No configuration needed! + 1. Create a `.env` file in the root of your project: @@ -24,24 +26,17 @@ To call your LLM from a third-party service, you'll likely need some environment .env ``` -3. Install `dotenv`: - - ```bash - pnpm add -D dotenv - ``` - -4. Add an `evalite.config.ts` file: + - ```ts - // evalite.config.ts +Now, your environment variables will be available in your evals. - import { defineConfig } from "evalite/config"; +## How It Works - export default defineConfig({ - setupFiles: ["dotenv/config"], - }); - ``` +Evalite uses [dotenv](https://www.npmjs.com/package/dotenv) under the hood to automatically load environment variables from `.env` files. The following files are supported (in order of precedence): - +- `.env.local` +- `.env.[mode].local` +- `.env.[mode]` +- `.env` -Now, your environment variables will be available in your evals. +This happens automatically—no setup required! diff --git a/packages/evalite-tests/tests/config.test.ts b/packages/evalite-tests/tests/config.test.ts index 8e1f1a07..4a2d5774 100644 --- a/packages/evalite-tests/tests/config.test.ts +++ b/packages/evalite-tests/tests/config.test.ts @@ -50,3 +50,35 @@ it("setupFiles in evalite.config.ts should load environment variables", async () "test_value_from_env" ); }); + +it("setupFiles in vitest.config.ts should be supported", async () => { + await using fixture = await loadFixture("config-setupfiles-vitest"); + + await fixture.run({ + mode: "run-once-and-exit", + }); + + const evals = await getEvalsAsRecordViaStorage(fixture.storage); + + // Should complete successfully with env var loaded from vitest.config.ts + expect(evals["Vitest Setup Test"]).toHaveLength(1); + expect(evals["Vitest Setup Test"]?.[0]?.status).toBe("success"); + expect(evals["Vitest Setup Test"]?.[0]?.results[0]?.output).toBe( + "from_vitest_config" + ); +}); + +it("setupFiles in evalite.config.ts should take precedence over vitest.config.ts", async () => { + await using fixture = await loadFixture("config-setupfiles-precedence"); + + await fixture.run({ + mode: "run-once-and-exit", + }); + + const evals = await getEvalsAsRecordViaStorage(fixture.storage); + + // Should complete successfully with env var from evalite setup (which runs after vitest) + expect(evals["Precedence Test"]).toHaveLength(1); + expect(evals["Precedence Test"]?.[0]?.status).toBe("success"); + expect(evals["Precedence Test"]?.[0]?.results[0]?.output).toBe("evalite"); +}); diff --git a/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/evalite-setup.ts b/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/evalite-setup.ts new file mode 100644 index 00000000..0d1a71e6 --- /dev/null +++ b/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/evalite-setup.ts @@ -0,0 +1,2 @@ +// Setup file from evalite.config.ts - should override vitest +process.env.SETUP_ORDER = "evalite"; diff --git a/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/evalite.config.ts b/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/evalite.config.ts new file mode 100644 index 00000000..d5dde4aa --- /dev/null +++ b/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/evalite.config.ts @@ -0,0 +1,5 @@ +import { defineConfig } from "evalite/config"; + +export default defineConfig({ + setupFiles: ["./evalite-setup.ts"], +}); diff --git a/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/test.eval.ts b/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/test.eval.ts new file mode 100644 index 00000000..0f4abcb3 --- /dev/null +++ b/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/test.eval.ts @@ -0,0 +1,15 @@ +import { evalite } from "evalite"; +import { Levenshtein } from "autoevals"; + +evalite("Precedence Test", { + data: () => [ + { + input: "test", + expected: "evalite", + }, + ], + task: async (input) => { + return process.env.SETUP_ORDER as string; + }, + scorers: [Levenshtein], +}); diff --git a/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/vitest-setup.ts b/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/vitest-setup.ts new file mode 100644 index 00000000..cd8b50a1 --- /dev/null +++ b/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/vitest-setup.ts @@ -0,0 +1,2 @@ +// Setup file from vitest.config.ts +process.env.SETUP_ORDER = "vitest"; diff --git a/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/vitest.config.ts b/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/vitest.config.ts new file mode 100644 index 00000000..998a1d93 --- /dev/null +++ b/packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + setupFiles: ["./vitest-setup.ts"], + }, +}); diff --git a/packages/evalite-tests/tests/fixtures/config-setupfiles-vitest/test.eval.ts b/packages/evalite-tests/tests/fixtures/config-setupfiles-vitest/test.eval.ts new file mode 100644 index 00000000..e0720a35 --- /dev/null +++ b/packages/evalite-tests/tests/fixtures/config-setupfiles-vitest/test.eval.ts @@ -0,0 +1,15 @@ +import { evalite } from "evalite"; +import { Levenshtein } from "autoevals"; + +evalite("Vitest Setup Test", { + data: () => [ + { + input: "test", + expected: process.env.VITEST_SETUP_VAR, + }, + ], + task: async (input) => { + return process.env.VITEST_SETUP_VAR as string; + }, + scorers: [Levenshtein], +}); diff --git a/packages/evalite-tests/tests/fixtures/config-setupfiles-vitest/vitest-setup.ts b/packages/evalite-tests/tests/fixtures/config-setupfiles-vitest/vitest-setup.ts new file mode 100644 index 00000000..6df377b4 --- /dev/null +++ b/packages/evalite-tests/tests/fixtures/config-setupfiles-vitest/vitest-setup.ts @@ -0,0 +1,2 @@ +// Setup file from vitest.config.ts +process.env.VITEST_SETUP_VAR = "from_vitest_config"; diff --git a/packages/evalite-tests/tests/fixtures/config-setupfiles-vitest/vitest.config.ts b/packages/evalite-tests/tests/fixtures/config-setupfiles-vitest/vitest.config.ts new file mode 100644 index 00000000..998a1d93 --- /dev/null +++ b/packages/evalite-tests/tests/fixtures/config-setupfiles-vitest/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + setupFiles: ["./vitest-setup.ts"], + }, +}); diff --git a/packages/evalite/package.json b/packages/evalite/package.json index 0066ea11..4cdd2e5f 100644 --- a/packages/evalite/package.json +++ b/packages/evalite/package.json @@ -45,7 +45,8 @@ "./runner": "./dist/run-evalite.js", "./export-static": "./dist/export-static.js", "./sqlite-storage": "./dist/storage/sqlite.js", - "./in-memory-storage": "./dist/storage/in-memory.js" + "./in-memory-storage": "./dist/storage/in-memory.js", + "./env-setup-file": "./dist/env-setup-file.js" }, "dependencies": { "@ai-sdk/provider": "^2.0.0", @@ -55,6 +56,7 @@ "@stricli/core": "^1.2.0", "@vitest/runner": "^3.2.4", "better-sqlite3": "^11.6.0", + "dotenv": "^16.4.7", "fastify": "^5.6.1", "file-type": "^19.6.0", "jiti": "^2.6.1", diff --git a/packages/evalite/src/config.ts b/packages/evalite/src/config.ts index 0765235c..e96b03f9 100644 --- a/packages/evalite/src/config.ts +++ b/packages/evalite/src/config.ts @@ -31,6 +31,13 @@ const CONFIG_FILE_NAMES = [ "evalite.config.mjs", ]; +const VITEST_CONFIG_FILE_NAMES = [ + "vitest.config.ts", + "vitest.config.mts", + "vitest.config.js", + "vitest.config.mjs", +]; + /** * Load Evalite configuration file from the specified directory. * Looks for evalite.config.{ts,mts,js,mjs} files. @@ -74,3 +81,49 @@ export async function loadEvaliteConfig( return undefined; } + +/** + * Load Vitest configuration file from the specified directory. + * Looks for vitest.config.{ts,mts,js,mjs} files and extracts setupFiles. + * + * @param cwd - Current working directory to search for config file + * @returns Array of setupFiles from vitest config, or empty array if none found + */ +export async function loadVitestSetupFiles( + cwd: string +): Promise { + const jiti = createJiti(import.meta.url, { + interopDefault: true, + requireCache: false, + }); + + for (const fileName of VITEST_CONFIG_FILE_NAMES) { + const configPath = path.join(cwd, fileName); + + try { + const loaded = (await jiti.import(configPath)) as any; + const config = loaded.default || loaded; + + if (config && typeof config === "object" && config.test?.setupFiles) { + const setupFiles = config.test.setupFiles; + // setupFiles can be a string or array of strings + return Array.isArray(setupFiles) ? setupFiles : [setupFiles]; + } + } catch (error: any) { + // File not found is expected, ignore it + if ( + error.code === "ERR_MODULE_NOT_FOUND" || + error.code === "ENOENT" || + error.message?.includes("Cannot find module") + ) { + continue; + } + // Other errors (syntax errors, etc) should be thrown + throw new Error( + `Failed to load Vitest config from ${configPath}: ${error.message}` + ); + } + } + + return []; +} diff --git a/packages/evalite/src/env-setup-file.ts b/packages/evalite/src/env-setup-file.ts new file mode 100644 index 00000000..6c181930 --- /dev/null +++ b/packages/evalite/src/env-setup-file.ts @@ -0,0 +1,3 @@ +// This file is automatically loaded by Evalite to support .env files +// It loads environment variables from .env files using dotenv +import "dotenv/config"; diff --git a/packages/evalite/src/run-evalite.ts b/packages/evalite/src/run-evalite.ts index c95df608..d95d9008 100644 --- a/packages/evalite/src/run-evalite.ts +++ b/packages/evalite/src/run-evalite.ts @@ -10,7 +10,7 @@ import EvaliteReporter from "./reporter.js"; import { createServer } from "./server.js"; import type { Evalite } from "./types.js"; import { createSqliteStorage } from "./storage/sqlite.js"; -import { loadEvaliteConfig } from "./config.js"; +import { loadEvaliteConfig, loadVitestSetupFiles } from "./config.js"; declare module "vitest" { export interface ProvidedContext { @@ -208,6 +208,9 @@ export const runEvalite = async (opts: { // Load config file if present const config = await loadEvaliteConfig(cwd); + // Load setupFiles from vitest.config.ts + const vitestSetupFiles = await loadVitestSetupFiles(cwd); + // Merge options: opts (highest priority) > config > defaults let storage = opts.storage; @@ -226,7 +229,16 @@ export const runEvalite = async (opts: { const serverPort = config?.server?.port ?? DEFAULT_SERVER_PORT; const testTimeout = config?.testTimeout; const maxConcurrency = config?.maxConcurrency; - const setupFiles = config?.setupFiles; + + // Merge setupFiles: + // 1. Always include env-setup-file first to load .env files + // 2. Add setupFiles from vitest.config.ts + // 3. Add setupFiles from evalite.config.ts (takes precedence) + const setupFiles = [ + "evalite/env-setup-file", + ...vitestSetupFiles, + ...(config?.setupFiles || []), + ]; const filters = opts.path ? [opts.path] : undefined; process.env.EVALITE_REPORT_TRACES = "true"; diff --git a/packages/evalite/src/types.ts b/packages/evalite/src/types.ts index 035ddb78..bac27611 100644 --- a/packages/evalite/src/types.ts +++ b/packages/evalite/src/types.ts @@ -85,11 +85,12 @@ export declare namespace Evalite { trialCount?: number; /** - * Setup files to run before tests (e.g., for loading environment variables) + * Setup files to run before tests. + * Note: .env files are loaded automatically via dotenv - no need to configure. * @example * ```ts * export default defineConfig({ - * setupFiles: ["dotenv/config"] + * setupFiles: ["./custom-setup.ts"] * }) * ``` */ diff --git a/packages/example/evalite.config.ts b/packages/example/evalite.config.ts index e89e95c5..27fce45d 100644 --- a/packages/example/evalite.config.ts +++ b/packages/example/evalite.config.ts @@ -1,5 +1,5 @@ import { defineConfig } from "evalite/config"; export default defineConfig({ - setupFiles: ["dotenv/config"], + // .env files are now loaded automatically! });