From da207d3e0d5b3c1660298f481405b3fdc794b897 Mon Sep 17 00:00:00 2001
From: Priyanshi1908 <115202982+Priyanshi1908@users.noreply.github.com>
Date: Fri, 15 Nov 2024 15:52:58 +0530
Subject: [PATCH 1/6] Added model column and functionality to configs table
---
services/api/src/core/evaluation.ts | 8 +++++---
services/api/src/core/general.ts | 1 +
services/api/src/db/schema.ts | 4 +++-
services/api/src/utils/db.ts | 2 ++
services/app/pages/reviewandrun/ReviewAndRun.vue | 4 ++--
5 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/services/api/src/core/evaluation.ts b/services/api/src/core/evaluation.ts
index 2b4a89c..fd5afd9 100644
--- a/services/api/src/core/evaluation.ts
+++ b/services/api/src/core/evaluation.ts
@@ -129,7 +129,6 @@ evaluationRouter.post('/evaluateCsv',
totalScore += score;
- // Insert individual evaluation results into the database
await db.insert(evaluations).values({
model,
question: result.question,
@@ -141,17 +140,20 @@ evaluationRouter.post('/evaluateCsv',
});
}
-// Calculate and store the average score in the configs table
const averageScore = totalScore / evaluationResults.length;
console.log(`Average Score for config ${configId}:`, averageScore);
await db.update(configs)
- .set({ averageScore: averageScore.toFixed(4) }) // Convert number to string with 4 decimal places
+ .set({
+ averageScore: averageScore.toFixed(4),
+ model,
+ })
.where(sql`${configs.id} = ${sql.raw(configId.toString())}`);
+
const csvContent = generateCSV(evaluationResults.map((result, index) => ({
...result,
choice: reports[index].choice,
diff --git a/services/api/src/core/general.ts b/services/api/src/core/general.ts
index ff880ed..c3accf7 100644
--- a/services/api/src/core/general.ts
+++ b/services/api/src/core/general.ts
@@ -157,6 +157,7 @@ generalRouter.post('/configs', async (c: Context) => {
reviews: metadata.reviews || "",
questionAnswerPairs: questionAnswerPairs,
fileContents: fileContent,
+ model: metadata.model,
};
const result = await db.insert(configs).values(config).returning({
diff --git a/services/api/src/db/schema.ts b/services/api/src/db/schema.ts
index 1740d88..140b4f1 100644
--- a/services/api/src/db/schema.ts
+++ b/services/api/src/db/schema.ts
@@ -33,9 +33,11 @@ export const configs = pgTable('configs', {
submittedBy: json('submitted_by').notNull(),
questionAnswerPairs: json('questionAnswerPairs').notNull().default([]),
fileContents: text('file_contents').notNull().default('No file uploaded'),
- averageScore: text('average_score').default('0'),
+ averageScore: text('average_score').default('0'),
+ model: text('model').notNull().default('unknown'),
});
+
export const models = pgTable('models', {
id: serial('id').primaryKey(),
name: text('name').notNull().unique(),
diff --git a/services/api/src/utils/db.ts b/services/api/src/utils/db.ts
index b36f21e..ee7d854 100644
--- a/services/api/src/utils/db.ts
+++ b/services/api/src/utils/db.ts
@@ -19,6 +19,7 @@ type Config = {
reviews?: string;
questionAnswerPairs: QAPair[];
fileContents: string;
+ model: string;
}
export async function createConfig(db: NeonHttpDatabase, config: Config) {
@@ -33,6 +34,7 @@ export async function createConfig(db: NeonHttpDatabase, config: Config) {
rating: config.rating || 0,
questionAnswerPairs: config.questionAnswerPairs.length ? config.questionAnswerPairs : [],
fileContents: config.fileContents ?? "No file uploaded",
+ model: config.model,
});
}
diff --git a/services/app/pages/reviewandrun/ReviewAndRun.vue b/services/app/pages/reviewandrun/ReviewAndRun.vue
index 360abc4..b4dd105 100644
--- a/services/app/pages/reviewandrun/ReviewAndRun.vue
+++ b/services/app/pages/reviewandrun/ReviewAndRun.vue
@@ -197,11 +197,11 @@ export default {
return;
}
- const model = selectedLlm.value === 'gpt-4o' ? 'gpt-4o' : 'gpt-4o-mini'; // Choose the model dynamically
+ const model = selectedLlm.value?.name === 'GPT-4o' ? 'gpt-4o' : 'gpt-4o-mini';
const payload = {
configId: Number(configId.value),
- model: model, // Pass the selected model dynamically
+ model: model,
};
loading.value = true;
From 73f8fcc396fc0c5ae170b301070a2de0ea591247 Mon Sep 17 00:00:00 2001
From: Priyanshi1908 <115202982+Priyanshi1908@users.noreply.github.com>
Date: Fri, 15 Nov 2024 16:27:45 +0530
Subject: [PATCH 2/6] Added avg score for model
---
services/api/src/core/evaluation.ts | 60 ++++++++++++++++++-
services/api/src/db/schema.ts | 4 +-
.../app/pages/reviewandrun/ReviewAndRun.vue | 29 ++++++++-
3 files changed, 90 insertions(+), 3 deletions(-)
diff --git a/services/api/src/core/evaluation.ts b/services/api/src/core/evaluation.ts
index fd5afd9..368dda9 100644
--- a/services/api/src/core/evaluation.ts
+++ b/services/api/src/core/evaluation.ts
@@ -7,8 +7,10 @@ import { evaluateQuestionBatch } from "./process"; // New batch evaluation funct
import { parseCSV } from "../utils/functions";
import { getRecordsWithIds } from "../utils/db";
import { generateReportsBatch } from "../utils/openai"; // New batch report generation function
-import { configs, evaluations } from "../db/schema";
+import { configs, evaluations,models } from "../db/schema";
import { sql } from 'drizzle-orm';
+import { eq } from 'drizzle-orm';
+
export type Env = {
DATABASE_URL: string;
@@ -190,4 +192,60 @@ function generateCSV(results: any[]): string {
return csvContent;
}
+
+evaluationRouter.post('/ModelAverageScores', async (c) => {
+ const db = createDbConnection(c.env.DATABASE_URL);
+
+ try {
+ const allModels = await db.select({ name: models.name }).from(models);
+
+ for (const { name: modelName } of allModels) {
+ const configsForModel = await db
+ .select()
+ .from(configs)
+ .where(eq(configs.model, modelName));
+
+ if (configsForModel.length === 0) {
+ console.warn(`No configs found for model ${modelName}`);
+ continue;
+ }
+
+ const totalScore = configsForModel.reduce((sum, config) => {
+ return sum + parseFloat(config.averageScore || '0'); // Convert score to float
+ }, 0);
+
+ const avgScore = totalScore / configsForModel.length;
+
+ console.log(`Average score for model ${modelName}:`, avgScore);
+
+ const existingModel = await db
+ .select()
+ .from(models)
+ .where(eq(models.name, modelName))
+ .limit(1);
+
+ if (existingModel.length > 0) {
+ await db
+ .update(models)
+ .set({ averageScore: avgScore.toFixed(4) })
+ .where(eq(models.name, modelName));
+ console.log(`Updated average score for model ${modelName}`);
+ } else {
+ await db
+ .insert(models)
+ .values({
+ name: modelName,
+ averageScore: avgScore.toFixed(4),
+ });
+ console.log(`Inserted new model ${modelName} with average score ${avgScore.toFixed(4)}`);
+ }
+ }
+
+ return c.json({ message: 'Updated average scores for all models' });
+ } catch (error) {
+ console.error("Error updating all model average scores:", error);
+ return c.json({ error: 'Internal server error' }, 500);
+ }
+});
+
export default evaluationRouter;
diff --git a/services/api/src/db/schema.ts b/services/api/src/db/schema.ts
index 140b4f1..25db6ce 100644
--- a/services/api/src/db/schema.ts
+++ b/services/api/src/db/schema.ts
@@ -45,7 +45,9 @@ export const models = pgTable('models', {
description: text('description'),
image: text('image'),
tags: json('tags'),
- createdAt: timestamp('created_at').defaultNow()
+ createdAt: timestamp('created_at').defaultNow(),
+ averageScore: text('average_score').default('0'),
+
});
export const evaluations = pgTable('evaluations', {
diff --git a/services/app/pages/reviewandrun/ReviewAndRun.vue b/services/app/pages/reviewandrun/ReviewAndRun.vue
index b4dd105..13ad1a3 100644
--- a/services/app/pages/reviewandrun/ReviewAndRun.vue
+++ b/services/app/pages/reviewandrun/ReviewAndRun.vue
@@ -206,6 +206,7 @@ export default {
loading.value = true;
try {
+ // Step 1: Evaluate the configuration
const response = await fetch('http://localhost:8787/eval/evaluateCsv', {
method: 'POST',
headers: {
@@ -215,6 +216,7 @@ export default {
});
if (response.ok) {
+ // Step 2: Download the CSV file after evaluation
const contentDisposition = response.headers.get('Content-Disposition');
console.log("Content-Disposition Header:", contentDisposition);
@@ -235,6 +237,11 @@ export default {
document.body.removeChild(link);
console.log('CSV file downloaded successfully');
+
+ // Step 3: Call the API to update the model's average score
+ await updateModelAverageScore();
+
+ // Step 4: Redirect after the evaluation is complete and model score is updated
router.push({ name: 'auditqueue' });
} else {
console.error('Failed to run evaluation');
@@ -242,7 +249,27 @@ export default {
} catch (error) {
console.error('Error in evaluation:', error);
} finally {
- loading.value = false;
+ loading.value = false; // Hide loading spinner
+ }
+};
+
+// Function to call the API to update the model's average score
+const updateModelAverageScore = async () => {
+ try {
+ const response = await fetch('http://localhost:8787/eval/ModelAverageScores', {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ });
+
+ if (response.ok) {
+ console.log("Model average score updated successfully.");
+ } else {
+ console.error("Failed to update model average score.");
+ }
+ } catch (error) {
+ console.error("Error updating model average score:", error);
}
};
From 96bfc9d7cd64a3bea20347943c1b437d935a762f Mon Sep 17 00:00:00 2001
From: Priyanshi1908 <115202982+Priyanshi1908@users.noreply.github.com>
Date: Sat, 16 Nov 2024 07:16:32 +0530
Subject: [PATCH 3/6] Added scores to graph in results page
---
services/api/src/core/evaluation.ts | 4 -
services/api/src/utils/openai.ts | 1 -
services/app/components/ModelScoresGraph.vue | 164 +++++++++++++++++++
services/app/pages/results/results.vue | 144 ++++------------
4 files changed, 195 insertions(+), 118 deletions(-)
create mode 100644 services/app/components/ModelScoresGraph.vue
diff --git a/services/api/src/core/evaluation.ts b/services/api/src/core/evaluation.ts
index 368dda9..a0a8e87 100644
--- a/services/api/src/core/evaluation.ts
+++ b/services/api/src/core/evaluation.ts
@@ -143,7 +143,6 @@ evaluationRouter.post('/evaluateCsv',
}
const averageScore = totalScore / evaluationResults.length;
-console.log(`Average Score for config ${configId}:`, averageScore);
await db.update(configs)
.set({
@@ -216,7 +215,6 @@ evaluationRouter.post('/ModelAverageScores', async (c) => {
const avgScore = totalScore / configsForModel.length;
- console.log(`Average score for model ${modelName}:`, avgScore);
const existingModel = await db
.select()
@@ -229,7 +227,6 @@ evaluationRouter.post('/ModelAverageScores', async (c) => {
.update(models)
.set({ averageScore: avgScore.toFixed(4) })
.where(eq(models.name, modelName));
- console.log(`Updated average score for model ${modelName}`);
} else {
await db
.insert(models)
@@ -237,7 +234,6 @@ evaluationRouter.post('/ModelAverageScores', async (c) => {
name: modelName,
averageScore: avgScore.toFixed(4),
});
- console.log(`Inserted new model ${modelName} with average score ${avgScore.toFixed(4)}`);
}
}
diff --git a/services/api/src/utils/openai.ts b/services/api/src/utils/openai.ts
index 752f4cc..e85d7ce 100644
--- a/services/api/src/utils/openai.ts
+++ b/services/api/src/utils/openai.ts
@@ -40,7 +40,6 @@ export async function generateReport(
});
const messageContent = response.choices[0]?.message?.content?.trim() ?? "{}";
- console.log(messageContent);
const { choice, score } = JSON.parse(messageContent);
console.log(choice, score);
return { choice, score };
diff --git a/services/app/components/ModelScoresGraph.vue b/services/app/components/ModelScoresGraph.vue
new file mode 100644
index 0000000..f36af44
--- /dev/null
+++ b/services/app/components/ModelScoresGraph.vue
@@ -0,0 +1,164 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/services/app/pages/results/results.vue b/services/app/pages/results/results.vue
index 69dbb9e..9c29416 100644
--- a/services/app/pages/results/results.vue
+++ b/services/app/pages/results/results.vue
@@ -1,27 +1,24 @@
-
@@ -158,59 +155,32 @@
-