diff --git a/services/api/package.json b/services/api/package.json index 24af197..21071bb 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -35,6 +35,7 @@ "drizzle-orm": "^0.30.10", "hono": "^4.6.10", "jose": "^5.9.6", + "jspdf": "^2.5.2", "ky": "^1.7.2", "openai": "^4.72.0", "p-limit": "^6.1.0", diff --git a/services/api/src/core/evaluation.ts b/services/api/src/core/evaluation.ts index 2b4a89c..1122787 100644 --- a/services/api/src/core/evaluation.ts +++ b/services/api/src/core/evaluation.ts @@ -2,13 +2,14 @@ import { Hono } from "hono"; import { HTTPException } from "hono/http-exception"; import { zValidator } from "@hono/zod-validator"; import { z } from "zod"; +import { jsPDF } from "jspdf"; import { createDbConnection, createOpenAIClient } from "../utils/functions"; -import { evaluateQuestionBatch } from "./process"; // New batch evaluation function +import { evaluateQuestionBatch } from "./process"; import { parseCSV } from "../utils/functions"; -import { getRecordsWithIds } from "../utils/db"; -import { generateReportsBatch } from "../utils/openai"; // New batch report generation function -import { configs, evaluations } from "../db/schema"; +import { generateReportsBatch } from "../utils/openai"; +import { configs, evaluations, models } from "../db/schema"; import { sql } from 'drizzle-orm'; +import { eq } from 'drizzle-orm'; export type Env = { DATABASE_URL: string; @@ -19,7 +20,6 @@ export type Env = { const evaluationRouter = new Hono<{ Bindings: Env }>(); -// Mapping the choice letter to the corresponding score const choiceToScore: Record = { A: 0.4, B: 0.6, @@ -51,7 +51,6 @@ Return the result as a JSON object with the following format: Ensure the response is only this JSON object for each question-answer pair.`; } - // Default or gpt-4o-mini prompt return `You are comparing a submitted answer to an expert answer on a given question. Here is the data: [BEGIN DATA] [Context]: ${context} @@ -76,7 +75,7 @@ Please return the answer strictly as a JSON object with the following structure, Only include this JSON object in your response.`; }; -evaluationRouter.post('/evaluateCsv', +evaluationRouter.post('/evaluatePdf', zValidator('json', z.object({ configId: z.number(), model: z.string(), @@ -120,6 +119,7 @@ evaluationRouter.post('/evaluateCsv', const reports = await generateReportsBatch(openai, model, prompts); let totalScore = 0; + const csvData = []; for (let i = 0; i < evaluationResults.length; i++) { const result = evaluationResults[i]; @@ -129,8 +129,7 @@ evaluationRouter.post('/evaluateCsv', totalScore += score; - // Insert individual evaluation results into the database - await db.insert(evaluations).values({ + const newEvaluation = await db.insert(evaluations).values({ model, question: result.question, answer: result.answer, @@ -138,34 +137,223 @@ evaluationRouter.post('/evaluateCsv', choice, score: String(score), createdAt: new Date(), + }).returning(); + + csvData.push({ + ...result, + choice, + score, + evaluationId: newEvaluation[0].id, }); } -// Calculate and store the average score in the configs table -const averageScore = totalScore / evaluationResults.length; -console.log(`Average Score for config ${configId}:`, averageScore); - -await db.update(configs) - .set({ averageScore: averageScore.toFixed(4) }) // Convert number to string with 4 decimal places - .where(sql`${configs.id} = ${sql.raw(configId.toString())}`); - - - - - const csvContent = generateCSV(evaluationResults.map((result, index) => ({ - ...result, - choice: reports[index].choice, - score: choiceToScore[reports[index].choice] || 0, - }))); + const averageScore = totalScore / evaluationResults.length; - c.header('Content-Type', 'text/csv'); - c.header('Content-Disposition', `attachment; filename=config_${configId}_results.csv`); - - return c.body(csvContent); + await db.update(configs) + .set({ + averageScore: averageScore.toFixed(4), + model, + }) + .where(sql`${configs.id} = ${sql.raw(configId.toString())}`); + + const doc = new jsPDF("p", "pt", "a4"); + const margin = 50; + const pageWidth = doc.internal.pageSize.getWidth(); + const pageHeight = doc.internal.pageSize.getHeight(); + const lineHeight = 14; + const contentWidth = pageWidth - 2 * margin; + let yPos = margin; + + const primaryColor = [0, 32, 96]; + const accentColor = [128, 128, 128]; + + doc.setFont("Times", "Roman"); + + const addDivider = () => { + doc.setDrawColor(accentColor[0], accentColor[1], accentColor[2]); + doc.setLineWidth(0.5); + yPos += 10; + doc.line(margin, yPos, pageWidth - margin, yPos); + yPos += 15; + }; + + const addHeader = (title: string, pageNum: number) => { + yPos = margin; + doc.setFont("Helvetica", "Bold"); + doc.setFontSize(18); + doc.setTextColor(primaryColor[0], primaryColor[1], primaryColor[2]); + doc.text(title, margin, yPos); + yPos += 20; + addDivider(); + doc.setFont("Times", "Italic"); + doc.setFontSize(10); + doc.setTextColor(accentColor[0], accentColor[1], accentColor[2]); + doc.text(`Page ${pageNum}`, pageWidth / 2, pageHeight - margin / 2, { align: "center" }); + }; + + const addSectionHeader = (text: string) => { + doc.setFont("Helvetica", "Bold"); + doc.setFontSize(14); + doc.setTextColor(primaryColor[0], primaryColor[1], primaryColor[2]); + doc.text(text, margin, yPos); + yPos += 20; + }; + + const addSubtitle = (text: string) => { + doc.setFont("Helvetica", "Italic"); + doc.setFontSize(12); + doc.setTextColor(accentColor[0], accentColor[1], accentColor[2]); + doc.text(text, margin, yPos); + yPos += 18; + }; + + const addBodyText = (text: string) => { + doc.setFontSize(11); + doc.setFont("Times", "Roman"); + doc.setTextColor(0); + const lines = doc.splitTextToSize(text, contentWidth); + doc.text(lines, margin, yPos); + yPos += lines.length * (lineHeight + 2); + }; + + const addBulletPoint = (text: string) => { + doc.setFontSize(11); + doc.setFont("Times", "Roman"); + doc.setTextColor(0); + const bullet = '\u2022'; + const lines = doc.splitTextToSize(text, contentWidth - 15); + doc.text(bullet, margin, yPos); + doc.text(lines, margin + 15, yPos); + yPos += lines.length * (lineHeight + 2); + }; + + const addTable = (data: string[][], colWidths: number[]) => { + const rowHeight = 20; + data.forEach((row, rowIndex) => { + row.forEach((cell, colIndex) => { + const x = margin + colWidths.slice(0, colIndex).reduce((a, b) => a + b, 0); + if (rowIndex === 0) { + doc.setFont("Helvetica", "Bold"); + doc.setFontSize(11); + doc.setTextColor(255, 255, 255); + doc.setFillColor(primaryColor[0], primaryColor[1], primaryColor[2]); + doc.rect(x, yPos, colWidths[colIndex], rowHeight, "F"); + doc.text(cell, x + 5, yPos + rowHeight / 2 + 4); + } else { + doc.setFont("Times", "Roman"); + doc.setFontSize(10); + doc.setTextColor(0); + doc.setFillColor(245, 245, 245); + doc.rect(x, yPos, colWidths[colIndex], rowHeight, "F"); + doc.text(cell, x + 5, yPos + rowHeight / 2 + 4); + } + }); + yPos += rowHeight; + }); + yPos += 10; + }; + + addHeader("LLM Audit Report", 1); + addSectionHeader(`Testing against ${model}`); + yPos += 20; + doc.setFont("Helvetica", "Bold"); + doc.setFontSize(24); + doc.setTextColor(0, 0, 139); + doc.text(`Average Score: ${averageScore.toFixed(2)}`, margin, yPos); + yPos += 40; + + addSectionHeader("Introduction"); + addBodyText("In today's rapidly evolving technological landscape, Large Language Models (LLMs) have emerged as powerful tools capable of generating human-quality text, translating languages, and answering complex questions with remarkable accuracy. As these models become increasingly integrated into various sectors, ensuring their responsible development and deployment is paramount. At ethosAI, we are committed to providing transparent and accountable AI evaluation through our advanced LLM auditing engine."); + addBodyText("Our engine is built on an in-house LLM that rigorously assesses AI models across three core principles: Holistic Assessment, Dynamic Benchmarking, and Ethical Foundations. This approach allows us to evaluate not only an LLM's factual knowledge and recall but also its logical reasoning, creative abilities, and ethical decision-making capabilities."); + addSubtitle("Ensuring Accuracy and Fairness"); + addBodyText("Ensuring the accuracy and fairness of LLM models is paramount. As these models become integral to key sectors such as healthcare, finance, and transportation, the risks associated with biases and inaccuracies increase significantly. Traditional methods of LLM evaluation often struggle to catch subtle biases or predict how models will perform in diverse real-world scenarios. This gap not only threatens the reliability of AI applications but also raises serious ethical concerns."); + + addSectionHeader("Case Study: US Constitution"); + addBodyText("The audit process involves a systematic methodology to evaluate the performance of LLMs in answering factual questions related to the Constitution of the United States of America."); + + doc.addPage(); + addHeader("LLM Audit Report", 2); + addSectionHeader("The Audit Process"); + addBodyText("Our audit process includes the following steps:"); + addBulletPoint("Generation of Evaluation Files: We meticulously crafted a set of questions and corresponding ideal answers based on the chosen topic, covering a wide range of complexities and subtopics."); + addBulletPoint("Selection of LLMs: Representative LLMs, in this case, GPT-4o and GPT-4o mini, were selected for evaluation."); + addBulletPoint("Running the Evaluation: We utilized the ethosAI engine to facilitate the interaction between the questions and the selected LLMs, recording the responses for subsequent analysis."); + addBulletPoint("Analysis and Results: We carefully analyzed the LLM responses, comparing them against the pre-defined ideal answers to evaluate their accuracy, relevance, and completeness."); + + addSectionHeader("Scoring Methodology"); + addBodyText("To ensure objective and comprehensive scoring, we employed a distance-based scoring methodology. A second LLM was used to calculate the distance between the ideal answers and the actual outputs generated by GPT-4o and GPT-4o mini."); + + const scoringTableData = [ + ["Choice", "Score", "Description"], + ["A", "0.4", "The submitted answer is a subset of the expert answer and is fully consistent with it."], + ["B", "0.6", "The submitted answer is a superset of the expert answer and is fully consistent with it."], + ["C", "1", "The submitted answer contains all the same details as the expert answer."], + ["D", "0", "There is a disagreement between the submitted answer and the expert answer."], + ["E", "1", "The answers differ, but these differences don't matter from the perspective of factuality."] + ]; + addTable(scoringTableData, [60, 60, contentWidth - 130]); + + doc.addPage(); + addHeader("LLM Audit Report", 3); + addSectionHeader("Results & Findings"); + addBodyText("Based on these scores, we measured two key performance indicators:"); + addBulletPoint("Accuracy: Defined as the percentage of times a score of 'C' or 'E' (i.e., a perfect score of 1) was achieved."); + addBulletPoint("Average Score: Calculated as the mean of all scores obtained across the questions."); + + const resultTableData = [ + ["Category", "Accuracy (%)", "Average Score"], + ["Presidency", "75%", "0.875"], + ["Amendments", "71%", "0.858"], + ["Congress", "67%", "0.867"], + ["Federalism", "60%", "0.8"], + ["General", "53%", "0.791"], + ["Supreme Court", "50%", "0.8"], + ["Rights & Liberty", "0%", "0.6"] + ]; + addTable(resultTableData, [200, 100, 100]); + + yPos += 20; + + addSectionHeader("Detailed Evaluation Results"); + const detailedTableData = [ + ["Question", "Expected Answer", "Choice", "Score", "Evaluation ID"], + ...csvData.map(result => [ + result.question, + result.answer, + result.choice, + result.score.toString(), + result.evaluationId.toString(), + ]) + ]; + addTable(detailedTableData, [150, 150, 50, 50, 60]); + + doc.addPage(); + addHeader("LLM Audit Report", 4); + addSectionHeader("Conclusion"); + addBodyText("This audit provides a comprehensive evaluation of GPT-4o and GPT-4o mini's capabilities in answering questions related to the U.S. Constitution."); + + addSectionHeader("Recommendations"); + addBulletPoint("Fine-tuning on Specific Domains: Further training on legal terminology and concepts related to the Constitution could enhance the models' performance."); + addBulletPoint("Addressing Bias and Ensuring Neutrality: It's crucial to ensure that the models' responses are unbiased and reflect the neutrality of the Constitution."); + addBulletPoint("Enhancing Explainability: The models should be able to provide clear and concise explanations for their answers."); + + yPos = pageHeight - margin - 30; + doc.setFontSize(10); + doc.setFont("Times", "Italic"); + doc.setTextColor(accentColor[0], accentColor[1], accentColor[2]); + doc.text("https://ethosai.one/", margin, yPos); + doc.text("founders@ethosai.one", pageWidth - margin, yPos, { align: "right" }); + + const pdfBlob = new Blob([doc.output('arraybuffer')], { type: 'application/pdf' }); + + c.header('Content-Type', 'application/pdf'); + c.header('Content-Disposition', `attachment; filename=config_${configId}_report.pdf`); + + return c.body(await pdfBlob.arrayBuffer()); } catch (error) { - console.error("Error evaluating CSV:", error); - return new HTTPException(500, { message: "Batch CSV evaluation failed" }).getResponse(); + console.error("Error evaluating PDF:", error); + return new HTTPException(500, { message: "Batch evaluation failed" }).getResponse(); } } ); @@ -188,4 +376,55 @@ function generateCSV(results: any[]): string { return csvContent; } -export default evaluationRouter; +evaluationRouter.post('/ModelAverageScores', async (c) => { + const db = createDbConnection(c.env.DATABASE_URL); + + try { + const allModels = await db.select({ name: models.name }).from(models); + + for (const { name: modelName } of allModels) { + const configsForModel = await db + .select() + .from(configs) + .where(eq(configs.model, modelName)); + + if (configsForModel.length === 0) { + console.warn(`No configs found for model ${modelName}`); + continue; + } + + const totalScore = configsForModel.reduce((sum, config) => { + return sum + parseFloat(config.averageScore || '0'); + }, 0); + + const avgScore = totalScore / configsForModel.length; + + const existingModel = await db + .select() + .from(models) + .where(eq(models.name, modelName)) + .limit(1); + + if (existingModel.length > 0) { + await db + .update(models) + .set({ averageScore: avgScore.toFixed(4) }) + .where(eq(models.name, modelName)); + } else { + await db + .insert(models) + .values({ + name: modelName, + averageScore: avgScore.toFixed(4), + }); + } + } + + return c.json({ message: 'Updated average scores for all models' }); + } catch (error) { + console.error("Error updating all model average scores:", error); + return c.json({ error: 'Internal server error' }, 500); + } +}); + +export default evaluationRouter; \ No newline at end of file diff --git a/services/api/src/core/general.ts b/services/api/src/core/general.ts index ff880ed..c3accf7 100644 --- a/services/api/src/core/general.ts +++ b/services/api/src/core/general.ts @@ -157,6 +157,7 @@ generalRouter.post('/configs', async (c: Context) => { reviews: metadata.reviews || "", questionAnswerPairs: questionAnswerPairs, fileContents: fileContent, + model: metadata.model, }; const result = await db.insert(configs).values(config).returning({ diff --git a/services/api/src/core/process.ts b/services/api/src/core/process.ts index 8b5b4df..d0e676d 100644 --- a/services/api/src/core/process.ts +++ b/services/api/src/core/process.ts @@ -1,6 +1,6 @@ import { NeonHttpDatabase } from "drizzle-orm/neon-http"; import OpenAI from "openai"; -import { generateResponse, generateReport } from "../utils/openai"; // Ensure these functions are exported correctly +import { generateResponse, generateReport } from "../utils/openai"; import { evaluations } from "../db/schema"; export async function evaluateQuestion( @@ -42,7 +42,7 @@ export async function evaluateQuestionBatch( openai: OpenAI, model: string, questions: { content: string; answer: string }[] -): Promise<{ question: string; answer: string; generated: string }[]> { +): Promise<{ question: string; answer: string; generated: string; evaluationId: number }[]> { const results = await Promise.all( questions.map(async (question) => { const result = await openai.chat.completions.create({ @@ -50,15 +50,24 @@ export async function evaluateQuestionBatch( messages: [{ role: "user", content: question.content }], max_tokens: 150, }); - - const generatedText = result.choices[0]?.message?.content?.trim() || ""; // Ensures fallback to an empty string if content is null or undefined + + const generatedText = result.choices[0]?.message?.content?.trim() || ""; + + const newEvaluation = await db.insert(evaluations).values({ + model, + question: question.content, + answer: question.answer, + output: generatedText, + createdAt: new Date(), + }).returning(); return { question: question.content, answer: question.answer, - generated: generatedText + generated: generatedText, + evaluationId: newEvaluation[0].id, }; }) ); return results; -} +} \ No newline at end of file diff --git a/services/api/src/db/schema.ts b/services/api/src/db/schema.ts index 1740d88..25db6ce 100644 --- a/services/api/src/db/schema.ts +++ b/services/api/src/db/schema.ts @@ -33,9 +33,11 @@ export const configs = pgTable('configs', { submittedBy: json('submitted_by').notNull(), questionAnswerPairs: json('questionAnswerPairs').notNull().default([]), fileContents: text('file_contents').notNull().default('No file uploaded'), - averageScore: text('average_score').default('0'), + averageScore: text('average_score').default('0'), + model: text('model').notNull().default('unknown'), }); + export const models = pgTable('models', { id: serial('id').primaryKey(), name: text('name').notNull().unique(), @@ -43,7 +45,9 @@ export const models = pgTable('models', { description: text('description'), image: text('image'), tags: json('tags'), - createdAt: timestamp('created_at').defaultNow() + createdAt: timestamp('created_at').defaultNow(), + averageScore: text('average_score').default('0'), + }); export const evaluations = pgTable('evaluations', { diff --git a/services/api/src/utils/db.ts b/services/api/src/utils/db.ts index b36f21e..ee7d854 100644 --- a/services/api/src/utils/db.ts +++ b/services/api/src/utils/db.ts @@ -19,6 +19,7 @@ type Config = { reviews?: string; questionAnswerPairs: QAPair[]; fileContents: string; + model: string; } export async function createConfig(db: NeonHttpDatabase, config: Config) { @@ -33,6 +34,7 @@ export async function createConfig(db: NeonHttpDatabase, config: Config) { rating: config.rating || 0, questionAnswerPairs: config.questionAnswerPairs.length ? config.questionAnswerPairs : [], fileContents: config.fileContents ?? "No file uploaded", + model: config.model, }); } diff --git a/services/api/src/utils/openai.ts b/services/api/src/utils/openai.ts index 752f4cc..e85d7ce 100644 --- a/services/api/src/utils/openai.ts +++ b/services/api/src/utils/openai.ts @@ -40,7 +40,6 @@ export async function generateReport( }); const messageContent = response.choices[0]?.message?.content?.trim() ?? "{}"; - console.log(messageContent); const { choice, score } = JSON.parse(messageContent); console.log(choice, score); return { choice, score }; diff --git a/services/app/components/ModelScoresGraph.vue b/services/app/components/ModelScoresGraph.vue new file mode 100644 index 0000000..f36af44 --- /dev/null +++ b/services/app/components/ModelScoresGraph.vue @@ -0,0 +1,164 @@ + + + + + \ No newline at end of file diff --git a/services/app/pages/results/results.vue b/services/app/pages/results/results.vue index 69dbb9e..9c29416 100644 --- a/services/app/pages/results/results.vue +++ b/services/app/pages/results/results.vue @@ -1,27 +1,24 @@ -