From da207d3e0d5b3c1660298f481405b3fdc794b897 Mon Sep 17 00:00:00 2001 From: Priyanshi1908 <115202982+Priyanshi1908@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:52:58 +0530 Subject: [PATCH 1/6] Added model column and functionality to configs table --- services/api/src/core/evaluation.ts | 8 +++++--- services/api/src/core/general.ts | 1 + services/api/src/db/schema.ts | 4 +++- services/api/src/utils/db.ts | 2 ++ services/app/pages/reviewandrun/ReviewAndRun.vue | 4 ++-- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/services/api/src/core/evaluation.ts b/services/api/src/core/evaluation.ts index 2b4a89c..fd5afd9 100644 --- a/services/api/src/core/evaluation.ts +++ b/services/api/src/core/evaluation.ts @@ -129,7 +129,6 @@ evaluationRouter.post('/evaluateCsv', totalScore += score; - // Insert individual evaluation results into the database await db.insert(evaluations).values({ model, question: result.question, @@ -141,17 +140,20 @@ evaluationRouter.post('/evaluateCsv', }); } -// Calculate and store the average score in the configs table const averageScore = totalScore / evaluationResults.length; console.log(`Average Score for config ${configId}:`, averageScore); await db.update(configs) - .set({ averageScore: averageScore.toFixed(4) }) // Convert number to string with 4 decimal places + .set({ + averageScore: averageScore.toFixed(4), + model, + }) .where(sql`${configs.id} = ${sql.raw(configId.toString())}`); + const csvContent = generateCSV(evaluationResults.map((result, index) => ({ ...result, choice: reports[index].choice, diff --git a/services/api/src/core/general.ts b/services/api/src/core/general.ts index ff880ed..c3accf7 100644 --- a/services/api/src/core/general.ts +++ b/services/api/src/core/general.ts @@ -157,6 +157,7 @@ generalRouter.post('/configs', async (c: Context) => { reviews: metadata.reviews || "", questionAnswerPairs: questionAnswerPairs, fileContents: fileContent, + model: metadata.model, }; const result = await db.insert(configs).values(config).returning({ diff --git a/services/api/src/db/schema.ts b/services/api/src/db/schema.ts index 1740d88..140b4f1 100644 --- a/services/api/src/db/schema.ts +++ b/services/api/src/db/schema.ts @@ -33,9 +33,11 @@ export const configs = pgTable('configs', { submittedBy: json('submitted_by').notNull(), questionAnswerPairs: json('questionAnswerPairs').notNull().default([]), fileContents: text('file_contents').notNull().default('No file uploaded'), - averageScore: text('average_score').default('0'), + averageScore: text('average_score').default('0'), + model: text('model').notNull().default('unknown'), }); + export const models = pgTable('models', { id: serial('id').primaryKey(), name: text('name').notNull().unique(), diff --git a/services/api/src/utils/db.ts b/services/api/src/utils/db.ts index b36f21e..ee7d854 100644 --- a/services/api/src/utils/db.ts +++ b/services/api/src/utils/db.ts @@ -19,6 +19,7 @@ type Config = { reviews?: string; questionAnswerPairs: QAPair[]; fileContents: string; + model: string; } export async function createConfig(db: NeonHttpDatabase, config: Config) { @@ -33,6 +34,7 @@ export async function createConfig(db: NeonHttpDatabase, config: Config) { rating: config.rating || 0, questionAnswerPairs: config.questionAnswerPairs.length ? config.questionAnswerPairs : [], fileContents: config.fileContents ?? "No file uploaded", + model: config.model, }); } diff --git a/services/app/pages/reviewandrun/ReviewAndRun.vue b/services/app/pages/reviewandrun/ReviewAndRun.vue index 360abc4..b4dd105 100644 --- a/services/app/pages/reviewandrun/ReviewAndRun.vue +++ b/services/app/pages/reviewandrun/ReviewAndRun.vue @@ -197,11 +197,11 @@ export default { return; } - const model = selectedLlm.value === 'gpt-4o' ? 'gpt-4o' : 'gpt-4o-mini'; // Choose the model dynamically + const model = selectedLlm.value?.name === 'GPT-4o' ? 'gpt-4o' : 'gpt-4o-mini'; const payload = { configId: Number(configId.value), - model: model, // Pass the selected model dynamically + model: model, }; loading.value = true; From 73f8fcc396fc0c5ae170b301070a2de0ea591247 Mon Sep 17 00:00:00 2001 From: Priyanshi1908 <115202982+Priyanshi1908@users.noreply.github.com> Date: Fri, 15 Nov 2024 16:27:45 +0530 Subject: [PATCH 2/6] Added avg score for model --- services/api/src/core/evaluation.ts | 60 ++++++++++++++++++- services/api/src/db/schema.ts | 4 +- .../app/pages/reviewandrun/ReviewAndRun.vue | 29 ++++++++- 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/services/api/src/core/evaluation.ts b/services/api/src/core/evaluation.ts index fd5afd9..368dda9 100644 --- a/services/api/src/core/evaluation.ts +++ b/services/api/src/core/evaluation.ts @@ -7,8 +7,10 @@ import { evaluateQuestionBatch } from "./process"; // New batch evaluation funct import { parseCSV } from "../utils/functions"; import { getRecordsWithIds } from "../utils/db"; import { generateReportsBatch } from "../utils/openai"; // New batch report generation function -import { configs, evaluations } from "../db/schema"; +import { configs, evaluations,models } from "../db/schema"; import { sql } from 'drizzle-orm'; +import { eq } from 'drizzle-orm'; + export type Env = { DATABASE_URL: string; @@ -190,4 +192,60 @@ function generateCSV(results: any[]): string { return csvContent; } + +evaluationRouter.post('/ModelAverageScores', async (c) => { + const db = createDbConnection(c.env.DATABASE_URL); + + try { + const allModels = await db.select({ name: models.name }).from(models); + + for (const { name: modelName } of allModels) { + const configsForModel = await db + .select() + .from(configs) + .where(eq(configs.model, modelName)); + + if (configsForModel.length === 0) { + console.warn(`No configs found for model ${modelName}`); + continue; + } + + const totalScore = configsForModel.reduce((sum, config) => { + return sum + parseFloat(config.averageScore || '0'); // Convert score to float + }, 0); + + const avgScore = totalScore / configsForModel.length; + + console.log(`Average score for model ${modelName}:`, avgScore); + + const existingModel = await db + .select() + .from(models) + .where(eq(models.name, modelName)) + .limit(1); + + if (existingModel.length > 0) { + await db + .update(models) + .set({ averageScore: avgScore.toFixed(4) }) + .where(eq(models.name, modelName)); + console.log(`Updated average score for model ${modelName}`); + } else { + await db + .insert(models) + .values({ + name: modelName, + averageScore: avgScore.toFixed(4), + }); + console.log(`Inserted new model ${modelName} with average score ${avgScore.toFixed(4)}`); + } + } + + return c.json({ message: 'Updated average scores for all models' }); + } catch (error) { + console.error("Error updating all model average scores:", error); + return c.json({ error: 'Internal server error' }, 500); + } +}); + export default evaluationRouter; diff --git a/services/api/src/db/schema.ts b/services/api/src/db/schema.ts index 140b4f1..25db6ce 100644 --- a/services/api/src/db/schema.ts +++ b/services/api/src/db/schema.ts @@ -45,7 +45,9 @@ export const models = pgTable('models', { description: text('description'), image: text('image'), tags: json('tags'), - createdAt: timestamp('created_at').defaultNow() + createdAt: timestamp('created_at').defaultNow(), + averageScore: text('average_score').default('0'), + }); export const evaluations = pgTable('evaluations', { diff --git a/services/app/pages/reviewandrun/ReviewAndRun.vue b/services/app/pages/reviewandrun/ReviewAndRun.vue index b4dd105..13ad1a3 100644 --- a/services/app/pages/reviewandrun/ReviewAndRun.vue +++ b/services/app/pages/reviewandrun/ReviewAndRun.vue @@ -206,6 +206,7 @@ export default { loading.value = true; try { + // Step 1: Evaluate the configuration const response = await fetch('http://localhost:8787/eval/evaluateCsv', { method: 'POST', headers: { @@ -215,6 +216,7 @@ export default { }); if (response.ok) { + // Step 2: Download the CSV file after evaluation const contentDisposition = response.headers.get('Content-Disposition'); console.log("Content-Disposition Header:", contentDisposition); @@ -235,6 +237,11 @@ export default { document.body.removeChild(link); console.log('CSV file downloaded successfully'); + + // Step 3: Call the API to update the model's average score + await updateModelAverageScore(); + + // Step 4: Redirect after the evaluation is complete and model score is updated router.push({ name: 'auditqueue' }); } else { console.error('Failed to run evaluation'); @@ -242,7 +249,27 @@ export default { } catch (error) { console.error('Error in evaluation:', error); } finally { - loading.value = false; + loading.value = false; // Hide loading spinner + } +}; + +// Function to call the API to update the model's average score +const updateModelAverageScore = async () => { + try { + const response = await fetch('http://localhost:8787/eval/ModelAverageScores', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }); + + if (response.ok) { + console.log("Model average score updated successfully."); + } else { + console.error("Failed to update model average score."); + } + } catch (error) { + console.error("Error updating model average score:", error); } }; From 96bfc9d7cd64a3bea20347943c1b437d935a762f Mon Sep 17 00:00:00 2001 From: Priyanshi1908 <115202982+Priyanshi1908@users.noreply.github.com> Date: Sat, 16 Nov 2024 07:16:32 +0530 Subject: [PATCH 3/6] Added scores to graph in results page --- services/api/src/core/evaluation.ts | 4 - services/api/src/utils/openai.ts | 1 - services/app/components/ModelScoresGraph.vue | 164 +++++++++++++++++++ services/app/pages/results/results.vue | 144 ++++------------ 4 files changed, 195 insertions(+), 118 deletions(-) create mode 100644 services/app/components/ModelScoresGraph.vue diff --git a/services/api/src/core/evaluation.ts b/services/api/src/core/evaluation.ts index 368dda9..a0a8e87 100644 --- a/services/api/src/core/evaluation.ts +++ b/services/api/src/core/evaluation.ts @@ -143,7 +143,6 @@ evaluationRouter.post('/evaluateCsv', } const averageScore = totalScore / evaluationResults.length; -console.log(`Average Score for config ${configId}:`, averageScore); await db.update(configs) .set({ @@ -216,7 +215,6 @@ evaluationRouter.post('/ModelAverageScores', async (c) => { const avgScore = totalScore / configsForModel.length; - console.log(`Average score for model ${modelName}:`, avgScore); const existingModel = await db .select() @@ -229,7 +227,6 @@ evaluationRouter.post('/ModelAverageScores', async (c) => { .update(models) .set({ averageScore: avgScore.toFixed(4) }) .where(eq(models.name, modelName)); - console.log(`Updated average score for model ${modelName}`); } else { await db .insert(models) @@ -237,7 +234,6 @@ evaluationRouter.post('/ModelAverageScores', async (c) => { name: modelName, averageScore: avgScore.toFixed(4), }); - console.log(`Inserted new model ${modelName} with average score ${avgScore.toFixed(4)}`); } } diff --git a/services/api/src/utils/openai.ts b/services/api/src/utils/openai.ts index 752f4cc..e85d7ce 100644 --- a/services/api/src/utils/openai.ts +++ b/services/api/src/utils/openai.ts @@ -40,7 +40,6 @@ export async function generateReport( }); const messageContent = response.choices[0]?.message?.content?.trim() ?? "{}"; - console.log(messageContent); const { choice, score } = JSON.parse(messageContent); console.log(choice, score); return { choice, score }; diff --git a/services/app/components/ModelScoresGraph.vue b/services/app/components/ModelScoresGraph.vue new file mode 100644 index 0000000..f36af44 --- /dev/null +++ b/services/app/components/ModelScoresGraph.vue @@ -0,0 +1,164 @@ + + + + + \ No newline at end of file diff --git a/services/app/pages/results/results.vue b/services/app/pages/results/results.vue index 69dbb9e..9c29416 100644 --- a/services/app/pages/results/results.vue +++ b/services/app/pages/results/results.vue @@ -1,27 +1,24 @@ -