diff --git a/packages/evalite/src/generation/graph.ts b/packages/evalite/src/generation/graph.ts new file mode 100644 index 00000000..1373f99c --- /dev/null +++ b/packages/evalite/src/generation/graph.ts @@ -0,0 +1,149 @@ +export type NoData = {}; +export type GraphNodeData = G extends Graph ? N : never; +export type GraphEdgeMap = G extends Graph ? E : never; + +export type AddEdgeTypes< + TBase extends Record, + TNew extends Record, +> = { + [K in keyof TBase | keyof TNew]: K extends keyof TNew + ? TNew[K] + : K extends keyof TBase + ? TBase[K] + : never; +}; + +export type Edge< + TNodeData, + TEdgeTypeDataMap extends Record = {}, +> = { + [K in keyof TEdgeTypeDataMap]: { + type: K; + data: TEdgeTypeDataMap[K]; + from: Node; + to: Node; + }; +}[keyof TEdgeTypeDataMap]; + +export class Graph< + TNodeData, + TEdgeTypeDataMap extends Record = {}, +> { + private nodes: Map> = new Map(); + + constructor(nodes?: Node[]) { + if (nodes) { + nodes.forEach((node) => this.addNode(node)); + } + } + + addNode(node: Node) { + this.nodes.set(node.id, node); + return node; + } + + getNode(id: string) { + return this.nodes.get(id); + } + + getNodes() { + return this.nodes; + } + + addEdge( + node1: string, + node2: string, + type: K, + data: TEdgeTypeDataMap[K] + ): void { + const node1Node = this.nodes.get(node1); + const node2Node = this.nodes.get(node2); + if (!node1Node || !node2Node) { + throw new Error("One or more nodes not found"); + } + const edge = { + from: node1Node, + to: node2Node, + type, + data, + } as Edge; + node1Node.addEdge(edge); + } + + clone< + TNewNodeData extends TNodeData = TNodeData, + TNewEdgeTypeDataMap extends Record = TEdgeTypeDataMap, + >(): Graph { + const newNodes = new Map>(); + + for (const [id, node] of this.nodes) { + const clonedNode = new Node( + node.id, + node.type, + structuredClone(node.data) as unknown as TNewNodeData + ); + newNodes.set(id, clonedNode); + } + + for (const [id, node] of this.nodes) { + const clonedNode = newNodes.get(id)!; + for (const edge of node.getEdges()) { + const clonedFromNode = newNodes.get(edge.from.id)!; + const clonedToNode = newNodes.get(edge.to.id)!; + const clonedEdge = { + from: clonedFromNode, + to: clonedToNode, + type: edge.type, + data: structuredClone(edge.data), + } as unknown as Edge; + clonedNode.addEdge(clonedEdge); + } + } + + return new Graph(Array.from(newNodes.values())); + } +} + +export class Node< + TNodeData, + TEdgeTypeDataMap extends Record = {}, +> { + data: TNodeData; + readonly type: "document" | "chunk"; + private edges: Map> = new Map(); + + constructor( + readonly id: string, + type: "document" | "chunk", + data: TNodeData + ) { + this.type = type; + this.data = data; + } + + addEdge(edge: Edge) { + this.edges.set(edge.to.id, edge); + } + + getEdges() { + return Array.from(this.edges.values()); + } +} + +export function graph< + TNodeData, + TEdgeTypeDataMap extends Record = {}, +>(nodes?: Node[]) { + return new Graph(nodes); +} + +export function node< + TNodeData, + TEdgeTypeDataMap extends Record = {}, +>(type: "document" | "chunk", data: TNodeData, id?: string) { + return new Node( + id ?? crypto.randomUUID(), + type, + data + ); +} diff --git a/packages/evalite/src/generation/persona.ts b/packages/evalite/src/generation/persona.ts new file mode 100644 index 00000000..0f7b15af --- /dev/null +++ b/packages/evalite/src/generation/persona.ts @@ -0,0 +1,155 @@ +import { generateObject, jsonSchema, type LanguageModel } from "ai"; +import type { Graph, Node } from "./graph.js"; +import { promptBuilder } from "../scorers/prompt-builder.js"; + +export interface Persona { + description: string; + knowledgeLevel: "novice" | "intermediate" | "expert"; +} + +const PersonaSchema = jsonSchema<{ + description: string; +}>({ + type: "object", + properties: { + description: { + type: "string", + description: + "A detailed description of the fictional persona who would consume this content, including their background, motivations, and how they would interact with the material", + }, + }, + required: ["description"], +}); + +const generatePersonaPrompt = promptBuilder({ + prompt: + "Generate a fictional persona who would be interested in consuming the following content. The persona should represent a realistic reader/user with the specified knowledge level ({knowledgeLevel}). Provide a detailed description of who they are, their motivations for engaging with this content, and their background. Output JSON following the required schema.", + examples: [ + { + input: { + summary: + "A comprehensive guide to machine learning algorithms, covering supervised and unsupervised learning techniques with practical Python examples.", + knowledgeLevel: "intermediate", + }, + output: { + description: + "Sarah is a 32-year-old software developer at a mid-sized tech company. She has 5 years of experience in backend development and recently became interested in adding ML capabilities to her team's products. She's comfortable with Python but has limited exposure to data science concepts beyond basic statistics. She wants to understand the fundamentals well enough to have meaningful conversations with data scientists and potentially prototype simple ML features.", + }, + }, + { + input: { + summary: + "Introduction to gardening for beginners, covering basic soil preparation, plant selection, and watering techniques.", + knowledgeLevel: "novice", + }, + output: { + description: + "Emily is a 45-year-old office manager who just bought her first home with a backyard. She grew up in apartments and has never had outdoor space before. She's excited to start a vegetable garden but feels overwhelmed by all the options and doesn't know where to begin. She has no prior gardening experience and needs step-by-step guidance.", + }, + }, + { + input: { + summary: + "Advanced distributed systems architecture patterns for high-availability microservices deployments.", + knowledgeLevel: "expert", + }, + output: { + description: + "David is a 40-year-old principal engineer at a large fintech company. He has 15+ years of experience building distributed systems and has led several large-scale migrations. He's looking to stay current with the latest patterns and validate his architectural decisions against industry best practices. He often mentors junior engineers and needs authoritative references to share with his team.", + }, + }, + ], + task: ["summary", "knowledgeLevel"], +}); + +export async function generatePersona< + TNodeData extends { content: string; summary?: string }, + TEdgeMap extends Record = Record, +>( + graph: Graph, + { + model, + amount, + filter = (node) => node.type === "document", + }: { + model: LanguageModel; + amount?: number; + filter?: (node: Node) => boolean; + } +): Promise { + const allNodes = Array.from(graph.getNodes().values()); + const filteredNodes = allNodes.filter(filter); + + if (filteredNodes.length === 0) { + return []; + } + + const nodesWithSummaries = filteredNodes.filter( + (node) => node.data.summary !== undefined && node.data.summary.trim() !== "" + ); + + if (nodesWithSummaries.length === 0) { + return []; + } + + const totalPersonas = amount ?? nodesWithSummaries.length; + + if (totalPersonas === 0) { + return []; + } + + const distribution = calculatePersonasPerNode( + totalPersonas, + nodesWithSummaries.length + ); + + const generationPromises: Promise[] = []; + + for (let i = 0; i < nodesWithSummaries.length; i++) { + const node = nodesWithSummaries[i]!; + const personaCount = distribution[i] ?? 0; + + for (let j = 0; j < personaCount; j++) { + const knowledgeLevel = getRandomKnowledgeLevel(); + + const promise = generateObject({ + model, + schema: PersonaSchema, + prompt: generatePersonaPrompt({ + summary: node.data.summary!, + knowledgeLevel, + }), + }).then((result) => ({ + description: result.object.description, + knowledgeLevel, + })); + + generationPromises.push(promise); + } + } + + return Promise.all(generationPromises); +} + +function getRandomKnowledgeLevel(): "novice" | "intermediate" | "expert" { + const levels: readonly ["novice", "intermediate", "expert"] = [ + "novice", + "intermediate", + "expert", + ]; + const index = Math.floor(Math.random() * levels.length); + return levels[index] ?? "intermediate"; +} + +function calculatePersonasPerNode( + totalAmount: number, + nodeCount: number +): number[] { + if (nodeCount === 0) return []; + const baseCount = Math.floor(totalAmount / nodeCount); + const remainder = totalAmount % nodeCount; + return Array.from( + { length: nodeCount }, + (_, i) => baseCount + (i < remainder ? 1 : 0) + ); +} diff --git a/packages/evalite/src/generation/test.ts b/packages/evalite/src/generation/test.ts new file mode 100644 index 00000000..47b6fb5e --- /dev/null +++ b/packages/evalite/src/generation/test.ts @@ -0,0 +1,45 @@ +import { jaccardSimilarity } from "./transformers/jaccard-similarity.js"; +import { topicExtractor } from "./transformers/topic-extractor.js"; +import { summaryExtractor } from "./transformers/summary-extractor.js"; +import { graph, node } from "./graph.js"; +import { transform } from "./transformers/transformer.js"; +import { openai } from "@ai-sdk/openai"; +import { embedExtractor } from "./transformers/embed-extractor.js"; +import { embeddingSimilarity } from "./transformers/embedding-similarity.js"; +import { chunkExtractor } from "./transformers/chunk-extractor.js"; +import { generatePersona } from "./persona.js"; + +const g = await transform(graph([node("document", { content: "Hello world" })])) + .pipe(chunkExtractor({ chunker: (content) => content.split(" ") })) + .pipe(summaryExtractor({ model: openai("gpt-4.1") })) + .pipe(topicExtractor({ model: openai("gpt-4.1") })) + .pipe(jaccardSimilarity({ property: "topics" })) + .pipe( + embedExtractor({ + model: openai.embedding("text-embedding-3-small"), + property: "summary", + }) + ) + .pipe(embeddingSimilarity({ property: "summaryEmbedding" })) + .pipe(embeddingSimilarity({ property: "content" })) + .build(); + +g.getNodes().forEach((node) => { + node.getEdges().forEach((edge) => { + if (edge.type === "jaccardSimilarity") { + console.log( + ` Jaccard score: ${edge.data.score} (property: ${edge.data.property})` + ); + } else if (edge.type === "embeddingSimilarity") { + console.log( + ` Embedding score: ${edge.data.score} (property: ${edge.data.property})` + ); + } else if (edge.type === "chunk" || edge.type === "parent") { + console.log(` Chunk relationship (no score data)`); + } + }); +}); + +generatePersona(g, { model: openai("gpt-4.1") }).then((personas) => { + console.log(personas); +}); diff --git a/packages/evalite/src/generation/transformers/chunk-extractor.ts b/packages/evalite/src/generation/transformers/chunk-extractor.ts new file mode 100644 index 00000000..0d5e1aa5 --- /dev/null +++ b/packages/evalite/src/generation/transformers/chunk-extractor.ts @@ -0,0 +1,51 @@ +import { + node, + type AddEdgeTypes, + type Graph, + type Node, + type NoData, +} from "../graph.js"; +import type { Transformer } from "./transformer.js"; + +export type ChunkerFn = (content: string) => string[]; + +export function chunkExtractor< + TInput extends { content: string }, + TEdges extends Record = {}, +>(options: { + chunker: ChunkerFn; + filter?: (node: Node) => boolean; +}): Transformer< + Graph, + Graph> +> { + return async (graph) => { + const originalNodes = Array.from(graph.getNodes().values()); + const filteredIds = new Set( + (options.filter + ? originalNodes.filter(options.filter) + : originalNodes + ).map((n) => n.id) + ); + + const cloned = graph.clone< + TInput, + AddEdgeTypes + >(); + + for (const n of cloned.getNodes().values()) { + if (!filteredIds.has(n.id)) continue; + const chunks = options.chunker(n.data.content); + + for (const chunk of chunks) { + const newNode = cloned.addNode( + node("chunk", { content: chunk } as TInput) + ); + cloned.addEdge(n.id, newNode.id, "chunk", {}); + cloned.addEdge(newNode.id, n.id, "parent", {}); + } + } + + return cloned; + }; +} diff --git a/packages/evalite/src/generation/transformers/embed-extractor.ts b/packages/evalite/src/generation/transformers/embed-extractor.ts new file mode 100644 index 00000000..117d6741 --- /dev/null +++ b/packages/evalite/src/generation/transformers/embed-extractor.ts @@ -0,0 +1,43 @@ +import { embed, type EmbeddingModel } from "ai"; +import { type Graph, type Node } from "../graph.js"; +import type { Transformer } from "./transformer.js"; + +export function embedExtractor< + TInput extends Record, + TEdges extends Record = {}, + TProperty extends keyof TInput & string = keyof TInput & string, +>(options: { + model: EmbeddingModel; + property: TProperty; + filter?: (node: Node) => boolean; +}): Transformer< + Graph, + Graph +> { + return async (graph) => { + const cloned = graph.clone< + TInput & { [K in `${TProperty}Embedding`]: number[] }, + TEdges + >(); + const nodes = Array.from(cloned.getNodes().values()); + const filtered = options.filter ? nodes.filter(options.filter) : nodes; + + const embeddingKey = + `${options.property}Embedding` as `${TProperty}Embedding`; + + for (const node of filtered) { + if (node.data[options.property] == null) continue; + + const { embedding } = await embed({ + model: options.model, + value: String(node.data[options.property]), + }); + node.data = { + ...node.data, + [embeddingKey]: embedding, + }; + } + + return cloned; + }; +} diff --git a/packages/evalite/src/generation/transformers/embedding-similarity.ts b/packages/evalite/src/generation/transformers/embedding-similarity.ts new file mode 100644 index 00000000..f0c7221b --- /dev/null +++ b/packages/evalite/src/generation/transformers/embedding-similarity.ts @@ -0,0 +1,73 @@ +import { cosineSimilarity } from "ai"; +import { type AddEdgeTypes, type Graph, type Node } from "../graph.js"; +import type { Transformer } from "./transformer.js"; + +export function embeddingSimilarity< + TInput extends Record, + TEdges extends Record = {}, +>(options: { + property: keyof TInput & string; + threshold?: number; + filter?: (node: Node) => boolean; +}): Transformer< + Graph, + Graph< + TInput, + AddEdgeTypes< + TEdges, + { + embeddingSimilarity: { score: number; property: keyof TInput & string }; + } + > + > +> { + return async (graph) => { + const originalNodes = Array.from(graph.getNodes().values()); + const filteredIds = new Set( + (options.filter + ? originalNodes.filter(options.filter) + : originalNodes + ).map((n) => n.id) + ); + + const cloned = graph.clone< + TInput, + AddEdgeTypes< + TEdges, + { + embeddingSimilarity: { + score: number; + property: keyof TInput & string; + }; + } + > + >(); + const filtered = Array.from(cloned.getNodes().values()).filter((n) => + filteredIds.has(n.id) + ); + const threshold = options.threshold ?? 0.5; + + for (let i = 0; i < filtered.length; i++) { + for (let j = i + 1; j < filtered.length; j++) { + const nodeA = filtered[i]; + const nodeB = filtered[j]; + if (!nodeA || !nodeB) continue; + + const valueA = nodeA.data[options.property]; + const valueB = nodeB.data[options.property]; + if (!valueA || !valueB) continue; + if (!Array.isArray(valueA) || !Array.isArray(valueB)) continue; + + const similarity = cosineSimilarity(valueA, valueB); + if (similarity > threshold) { + cloned.addEdge(nodeA.id, nodeB.id, "embeddingSimilarity", { + score: similarity, + property: options.property, + }); + } + } + } + + return cloned; + }; +} diff --git a/packages/evalite/src/generation/transformers/entity-extractor.ts b/packages/evalite/src/generation/transformers/entity-extractor.ts new file mode 100644 index 00000000..30f4ae39 --- /dev/null +++ b/packages/evalite/src/generation/transformers/entity-extractor.ts @@ -0,0 +1,101 @@ +import { generateObject, jsonSchema, type LanguageModel } from "ai"; +import { type Graph, type Node } from "../graph.js"; +import type { Transformer } from "./transformer.js"; +import { promptBuilder } from "../../scorers/prompt-builder.js"; + +const EntitiesSchema = jsonSchema<{ + entities: Array<{ type: string; value: string; description?: string }>; +}>({ + type: "object", + properties: { + entities: { + type: "array", + items: { + type: "object", + properties: { + type: { + type: "string", + description: + "The type or category of the entity (e.g., PERSON, ORGANIZATION, LOCATION, DATE, etc.)", + }, + value: { + type: "string", + description: "The actual entity value extracted from the content", + }, + description: { + type: "string", + description: + "Optional additional context or description about the entity", + }, + }, + required: ["type", "value"], + }, + }, + }, + required: ["entities"], +}); + +const extractEntitiesPrompt = promptBuilder({ + prompt: + "Extract all named entities from the provided content. Identify entities such as people, organizations, locations, dates, products, or any other relevant entities. For each entity, provide its type, value, and optionally a brief description. Output JSON following the required schema.", + examples: [ + { + input: { + content: + "Apple Inc. announced that Tim Cook will speak at the conference in San Francisco on March 15, 2024.", + }, + output: { + entities: [ + { + type: "ORGANIZATION", + value: "Apple Inc.", + description: "Technology company", + }, + { type: "PERSON", value: "Tim Cook", description: "CEO of Apple" }, + { + type: "LOCATION", + value: "San Francisco", + description: "City location of the conference", + }, + { + type: "DATE", + value: "March 15, 2024", + description: "Conference date", + }, + ], + }, + }, + ], + task: ["content"], +}); + +type Entity = { type: string; value: string; description?: string }; + +export function entityExtractor< + TInput extends { content: string }, + TEdges extends Record = {}, +>(options: { + model: LanguageModel; + filter?: (node: Node) => boolean; +}): Transformer< + Graph, + Graph +> { + return async (graph) => { + const cloned = graph.clone(); + const nodes = Array.from(cloned.getNodes().values()); + const filtered = options.filter ? nodes.filter(options.filter) : nodes; + + for (const node of filtered) { + const result = await generateObject({ + model: options.model, + schema: EntitiesSchema, + prompt: extractEntitiesPrompt({ content: node.data.content }), + }); + + node.data = { ...node.data, entities: result.object.entities }; + } + + return cloned; + }; +} diff --git a/packages/evalite/src/generation/transformers/jaccard-similarity.ts b/packages/evalite/src/generation/transformers/jaccard-similarity.ts new file mode 100644 index 00000000..d6ab7454 --- /dev/null +++ b/packages/evalite/src/generation/transformers/jaccard-similarity.ts @@ -0,0 +1,83 @@ +import { type AddEdgeTypes, type Graph, type Node } from "../graph.js"; +import type { Transformer } from "./transformer.js"; + +export function jaccardSimilarity< + TInput extends Record, + TEdges extends Record = {}, +>(options: { + property: keyof TInput & string; + threshold?: number; + filter?: (node: Node) => boolean; +}): Transformer< + Graph, + Graph< + TInput, + AddEdgeTypes< + TEdges, + { + jaccardSimilarity: { score: number; property: keyof TInput & string }; + } + > + > +> { + return async (graph) => { + const originalNodes = Array.from(graph.getNodes().values()); + const filteredIds = new Set( + (options.filter + ? originalNodes.filter(options.filter) + : originalNodes + ).map((n) => n.id) + ); + + const cloned = graph.clone< + TInput, + AddEdgeTypes< + TEdges, + { + jaccardSimilarity: { score: number; property: keyof TInput & string }; + } + > + >(); + const filtered = Array.from(cloned.getNodes().values()).filter((n) => + filteredIds.has(n.id) + ); + const threshold = options.threshold ?? 0.5; + + for (let i = 0; i < filtered.length; i++) { + for (let j = i + 1; j < filtered.length; j++) { + const nodeA = filtered[i]; + const nodeB = filtered[j]; + if (!nodeA || !nodeB) continue; + + const valueA = nodeA.data[options.property]; + const valueB = nodeB.data[options.property]; + if (!valueA || !valueB) continue; + + const setA = new Set( + Array.isArray(valueA) + ? valueA + : String(valueA).toLowerCase().split(/\s+/) + ); + const setB = new Set( + Array.isArray(valueB) + ? valueB + : String(valueB).toLowerCase().split(/\s+/) + ); + + const intersection = new Set([...setA].filter((x) => setB.has(x))); + const union = new Set([...setA, ...setB]); + const similarity = + union.size === 0 ? 0 : intersection.size / union.size; + + if (similarity > threshold) { + cloned.addEdge(nodeA.id, nodeB.id, "jaccardSimilarity", { + score: similarity, + property: options.property, + }); + } + } + } + + return cloned; + }; +} diff --git a/packages/evalite/src/generation/transformers/summary-extractor.ts b/packages/evalite/src/generation/transformers/summary-extractor.ts new file mode 100644 index 00000000..61a8c59a --- /dev/null +++ b/packages/evalite/src/generation/transformers/summary-extractor.ts @@ -0,0 +1,65 @@ +import { generateObject, jsonSchema, type LanguageModel } from "ai"; +import { type Graph, type Node } from "../graph.js"; +import type { Transformer } from "./transformer.js"; +import { promptBuilder } from "../../scorers/prompt-builder.js"; + +const SummarySchema = jsonSchema<{ + summary: string; +}>({ + type: "object", + properties: { + summary: { + type: "string", + description: + "A concise summary of the content, capturing the main points and key information", + }, + }, + required: ["summary"], +}); + +const extractSummaryPrompt = promptBuilder({ + prompt: + "Generate a concise summary of the provided content. Capture the main points, key information, and essential details in a clear and coherent way. The summary should be comprehensive yet brief. Output JSON following the required schema.", + examples: [ + { + input: { + content: + "Apple Inc. announced that Tim Cook will speak at the conference in San Francisco on March 15, 2024. The conference will focus on the future of technology and innovation. Cook is expected to discuss Apple's latest developments in artificial intelligence and their vision for integrating AI into consumer products. Industry experts anticipate major announcements regarding new product lines.", + }, + output: { + summary: + "Apple's CEO Tim Cook will speak at a technology conference in San Francisco on March 15, 2024, focusing on AI developments and Apple's vision for AI-integrated consumer products, with anticipated announcements of new product lines.", + }, + }, + ], + task: ["content"], +}); + +export function summaryExtractor< + TInput extends { content: string }, + TEdges extends Record = {}, +>(options: { + model: LanguageModel; + filter?: (node: Node) => boolean; +}): Transformer< + Graph, + Graph +> { + return async (graph) => { + const cloned = graph.clone(); + const nodes = Array.from(cloned.getNodes().values()); + const filtered = options.filter ? nodes.filter(options.filter) : nodes; + + for (const node of filtered) { + const result = await generateObject({ + model: options.model, + schema: SummarySchema, + prompt: extractSummaryPrompt({ content: node.data.content }), + }); + + node.data = { ...node.data, summary: result.object.summary }; + } + + return cloned; + }; +} diff --git a/packages/evalite/src/generation/transformers/topic-extractor.ts b/packages/evalite/src/generation/transformers/topic-extractor.ts new file mode 100644 index 00000000..2db7d326 --- /dev/null +++ b/packages/evalite/src/generation/transformers/topic-extractor.ts @@ -0,0 +1,74 @@ +import { generateObject, jsonSchema, type LanguageModel } from "ai"; +import { type Graph, type Node } from "../graph.js"; +import type { Transformer } from "./transformer.js"; +import { promptBuilder } from "../../scorers/prompt-builder.js"; + +const TopicSchema = jsonSchema<{ + topics: string[]; +}>({ + type: "object", + properties: { + topics: { + type: "array", + items: { + type: "string", + description: "A key topic or keyword extracted from the content", + }, + }, + }, + required: ["topics"], +}); + +const extractTopicPrompt = promptBuilder({ + prompt: + "Extract a list of key topics or keywords from the provided content. These should represent the main themes or subjects discussed. Output JSON following the required schema.", + examples: [ + { + input: { + content: + "Machine learning is a field of inquiry devoted to understanding and building methods that 'learn', that is, methods that leverage data to improve performance on some set of tasks.", + }, + output: { + topics: [ + "Machine Learning", + "Artificial Intelligence", + "Data Science", + "Algorithms", + ], + }, + }, + ], + task: ["content"], +}); + +export function topicExtractor< + TInput extends { content: string }, + TEdges extends Record = {}, +>(options: { + model: LanguageModel; + filter?: (node: Node) => boolean; +}): Transformer< + Graph, + Graph +> { + return async (graph) => { + const cloned = graph.clone(); + const nodes = Array.from(cloned.getNodes().values()); + const filtered = options.filter ? nodes.filter(options.filter) : nodes; + + for (const node of filtered) { + const result = await generateObject({ + model: options.model, + schema: TopicSchema, + prompt: extractTopicPrompt({ content: node.data.content }), + }); + + node.data = { + ...node.data, + topics: result.object.topics.map((t) => t.trim().toLowerCase()), + }; + } + + return cloned; + }; +} diff --git a/packages/evalite/src/generation/transformers/transformer.ts b/packages/evalite/src/generation/transformers/transformer.ts new file mode 100644 index 00000000..d33a0643 --- /dev/null +++ b/packages/evalite/src/generation/transformers/transformer.ts @@ -0,0 +1,35 @@ +import type { Graph } from "../graph.js"; + +export type Transformer< + TInput extends Graph = Graph<{}, {}>, + TOutput extends Graph = Graph<{}, {}>, +> = (graph: TInput) => PromiseLike; + +export type TransformerPipeline> = { + pipe>( + transformer: Transformer + ): TransformerPipeline; + build(): Promise; +}; + +export function transform>( + graph: TGraph +): TransformerPipeline { + const createPipeline = >( + currentGraph: PromiseLike + ): TransformerPipeline => ({ + pipe>( + transformer: Transformer + ) { + const nextGraph = Promise.resolve(currentGraph).then((resolvedGraph) => + transformer(resolvedGraph) + ); + return createPipeline(nextGraph); + }, + build() { + return Promise.resolve(currentGraph); + }, + }); + + return createPipeline(Promise.resolve(graph)); +}