From 719934c2ff438901ec6a30a8bc19fb46712aef9c Mon Sep 17 00:00:00 2001 From: zackKonfs Date: Thu, 26 Mar 2026 23:13:43 +0800 Subject: [PATCH 01/13] feat: add interview AI coaching with real OpenAI scoring --- app/api/interview/coaching/route.ts | 90 ++++++++++++++++++++++++ app/interview/page.tsx | 105 ++++++---------------------- package-lock.json | 13 ---- 3 files changed, 113 insertions(+), 95 deletions(-) create mode 100644 app/api/interview/coaching/route.ts diff --git a/app/api/interview/coaching/route.ts b/app/api/interview/coaching/route.ts new file mode 100644 index 0000000..b87e24b --- /dev/null +++ b/app/api/interview/coaching/route.ts @@ -0,0 +1,90 @@ +export const runtime = "nodejs"; + +import { NextRequest, NextResponse } from "next/server"; +import OpenAI from "openai"; +import Groq from "groq-sdk"; + +const OPENAI_MODEL = "gpt-4o-mini"; +const GROQ_FALLBACK_MODEL = "llama-3.1-8b-instant"; + +const SYSTEM_PROMPT = `You are an expert interview coach reviewing a mock interview conversation for a Singapore job seeker. + +Analyse the candidate's answers so far and return ONLY a valid JSON object with exactly these fields: +{ + "score": a number from 0 to 100 reflecting overall interview performance so far, + "strengths": an array of 2 specific things the candidate did well in their answers, + "improvements": an array of 2 specific coaching suggestions to improve their answers +} + +Be specific and practical. Reference actual things the candidate said. No generic advice.`; + +function extractJson(text: string): string { + const fenceMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/); + if (fenceMatch) return fenceMatch[1].trim(); + const braceStart = text.indexOf("{"); + const braceEnd = text.lastIndexOf("}"); + if (braceStart !== -1 && braceEnd > braceStart) { + return text.slice(braceStart, braceEnd + 1); + } + return text.trim(); +} + +export async function POST(req: NextRequest) { + try { + const { conversation } = await req.json(); + + if (!Array.isArray(conversation) || conversation.length === 0) { + return NextResponse.json({ error: "No conversation provided" }, { status: 400 }); + } + + const transcript = conversation + .filter((m: { speaker: string; text: string }) => m.speaker !== "system") + .map((m: { speaker: string; text: string }) => + `${m.speaker === "agent" ? "Interviewer" : "Candidate"}: ${m.text}` + ) + .join("\n"); + + let raw = ""; + + const openaiKey = process.env.OPENAI_API_KEY; + if (openaiKey) { + try { + const client = new OpenAI({ apiKey: openaiKey }); + const response = await client.chat.completions.create({ + model: OPENAI_MODEL, + messages: [ + { role: "system", content: SYSTEM_PROMPT }, + { role: "user", content: transcript }, + ], + temperature: 0, + }); + raw = response.choices[0]?.message?.content ?? ""; + } catch (err) { + console.warn("OpenAI coaching failed, falling back to Groq:", err); + } + } + + if (!raw) { + const groqKey = process.env.GROQ_API_KEY; + if (!groqKey) { + return NextResponse.json({ error: "No AI API key configured" }, { status: 500 }); + } + const client = new Groq({ apiKey: groqKey }); + const response = await client.chat.completions.create({ + model: GROQ_FALLBACK_MODEL, + messages: [ + { role: "system", content: SYSTEM_PROMPT }, + { role: "user", content: transcript }, + ], + temperature: 0, + }); + raw = response.choices[0]?.message?.content ?? ""; + } + + const parsed = JSON.parse(extractJson(raw)); + return NextResponse.json(parsed); + } catch (err) { + console.error("Coaching route error:", err); + return NextResponse.json({ error: "Failed to generate coaching" }, { status: 500 }); + } +} diff --git a/app/interview/page.tsx b/app/interview/page.tsx index b2c6619..6b0b88d 100644 --- a/app/interview/page.tsx +++ b/app/interview/page.tsx @@ -1,7 +1,7 @@ "use client"; import { useRouter } from "next/navigation"; -import { useMemo, useState } from "react"; +import { useState } from "react"; import { SiteNavbar } from "@/components/layout/SiteNavbar"; import { useLanguage } from "@/components/providers/language-provider"; import { createClient } from "@/lib/supabase/client"; @@ -60,81 +60,6 @@ function getOpeningLine(locale: string, role: InterviewRole, interviewType: Inte return `Hi, I’ll be your AI interviewer. We’re starting a ${interviewType} interview practice session for a ${roleLabel} role. Please begin by introducing yourself.`; } -function buildMockCoaching(locale: string, messages: ConversationMessage[]) { - const userTurns = messages.filter((m) => m.speaker === "user"); - const totalLength = userTurns.map((m) => m.text).join(" ").length; - const score = userTurns.length === 0 ? 0 : Math.max(58, Math.min(91, 58 + Math.floor(totalLength / 20))); - - if (locale === "zh") { - return { - score, - strengths: - userTurns.length === 0 - ? [] - : totalLength > 180 - ? ["回答更加完整", "表达更清晰", "结构逐渐改善"] - : ["愿意作答", "基础表达清楚"], - improvements: - userTurns.length === 0 - ? [] - : totalLength > 180 - ? ["加入量化成果", "强化岗位相关性", "结尾更有说服力"] - : ["加入真实例子", "说明行动与结果", "补充更多细节"], - }; - } - - if (locale === "ms") { - return { - score, - strengths: - userTurns.length === 0 - ? [] - : totalLength > 180 - ? ["Jawapan semakin lengkap", "Penyampaian lebih jelas", "Struktur semakin baik"] - : ["Sedia menjawab", "Asas jawapan boleh difahami"], - improvements: - userTurns.length === 0 - ? [] - : totalLength > 180 - ? ["Tambah hasil yang boleh diukur", "Kaitkan lebih rapat dengan jawatan", "Penutup boleh lebih kuat"] - : ["Tambah contoh sebenar", "Terangkan tindakan dan hasil", "Tambah lebih banyak perincian"], - }; - } - - if (locale === "ta") { - return { - score, - strengths: - userTurns.length === 0 - ? [] - : totalLength > 180 - ? ["பதில்கள் மேலும் முழுமையாக உள்ளன", "விளக்கம் தெளிவாக உள்ளது", "அமைப்பு மேம்படுகிறது"] - : ["பதிலளிக்கும் முனைப்பு உள்ளது", "அடிப்படை கருத்து புரிகிறது"], - improvements: - userTurns.length === 0 - ? [] - : totalLength > 180 - ? ["அளவிடக்கூடிய முடிவுகளைச் சேர்க்கவும்", "பதவியுடன் தொடர்பை வலுப்படுத்தவும்", "முடிவை வலுப்படுத்தவும்"] - : ["உண்மையான உதாரணத்தைச் சேர்க்கவும்", "நடவடிக்கை மற்றும் முடிவை விளக்கவும்", "மேலும் விவரம் சேர்க்கவும்"], - }; - } - - return { - score, - strengths: - userTurns.length === 0 - ? [] - : totalLength > 180 - ? ["Answers are becoming more complete", "Communication is clearer", "Structure is improving"] - : ["Willing to engage", "Basic ideas are understandable"], - improvements: - userTurns.length === 0 - ? [] - : totalLength > 180 - ? ["Add measurable outcomes", "Tie answers closer to the role", "End more strongly"] - : ["Use a real example", "Explain actions and results", "Add more detail"], - }; -} export default function InterviewPage() { const router = useRouter(); @@ -152,10 +77,11 @@ export default function InterviewPage() { const [textInput, setTextInput] = useState(""); const [sessionStarted, setSessionStarted] = useState(false); - const coaching = useMemo( - () => buildMockCoaching(safeLocale, conversation), - [safeLocale, conversation] - ); + const [coaching, setCoaching] = useState<{ + score: number; + strengths: string[]; + improvements: string[]; + }>({ score: 0, strengths: [], improvements: [] }); function startSession() { setSessionStarted(true); @@ -188,7 +114,7 @@ export default function InterviewPage() { ]); } - function sendTextReply() { + async function sendTextReply() { if (!textInput.trim()) return; const userMessage: ConversationMessage = { @@ -198,8 +124,23 @@ export default function InterviewPage() { timestamp: formatTime(), }; - setConversation((prev) => [...prev, userMessage]); + const updatedConversation = [...conversation, userMessage]; + setConversation(updatedConversation); setTextInput(""); + + try { + const res = await fetch("/api/interview/coaching", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ conversation: updatedConversation }), + }); + if (res.ok) { + const data = await res.json(); + setCoaching(data); + } + } catch { + // silently keep existing coaching if the request fails + } } return ( diff --git a/package-lock.json b/package-lock.json index cec3633..6c28944 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1325,7 +1325,6 @@ "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.98.0.tgz", "integrity": "sha512-Ohc97CtInLwZyiSASz7tT9/Abm/vqnIbO9REp+PivVUII8UZsuI3bngRQnYgJdFoOIwvaEII1fX1qy8x0CyNiw==", "license": "MIT", - "peer": true, "dependencies": { "@supabase/auth-js": "2.98.0", "@supabase/functions-js": "2.98.0", @@ -1407,7 +1406,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.28.tgz", "integrity": "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw==", "license": "MIT", - "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -1490,7 +1488,6 @@ "integrity": "sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.1", "@typescript-eslint/types": "8.56.1", @@ -2073,7 +2070,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3322,7 +3318,6 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -3492,7 +3487,6 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -5054,7 +5048,6 @@ "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", "dev": true, "license": "MIT", - "peer": true, "bin": { "jiti": "bin/jiti.js" } @@ -6085,7 +6078,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -6352,7 +6344,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -6365,7 +6356,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -7450,7 +7440,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -7522,7 +7511,6 @@ "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "~0.27.0", "get-tsconfig": "^4.7.5" @@ -7660,7 +7648,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" From c32fbf3d813344d8d97c526ea0775e63dc8fa572 Mon Sep 17 00:00:00 2001 From: zackKonfs Date: Thu, 26 Mar 2026 23:51:04 +0800 Subject: [PATCH 02/13] feat: add voice to text for interview mic button --- app/api/speech/route.ts | 33 +++++++++++++ app/interview/page.tsx | 63 +++++++++++++++++++++++-- app/resume/page.tsx | 100 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 186 insertions(+), 10 deletions(-) create mode 100644 app/api/speech/route.ts diff --git a/app/api/speech/route.ts b/app/api/speech/route.ts new file mode 100644 index 0000000..434ca50 --- /dev/null +++ b/app/api/speech/route.ts @@ -0,0 +1,33 @@ +export const runtime = "nodejs"; + +import { NextResponse } from "next/server"; + +export async function GET() { + const key = process.env.AZURE_SPEECH_KEY; + const region = process.env.AZURE_SPEECH_REGION; + + if (!key || !region) { + return NextResponse.json( + { error: "Azure Speech credentials are not configured" }, + { status: 500 }, + ); + } + + const tokenRes = await fetch( + `https://${region}.api.cognitive.microsoft.com/sts/v1.0/issueToken`, + { + method: "POST", + headers: { "Ocp-Apim-Subscription-Key": key }, + }, + ); + + if (!tokenRes.ok) { + return NextResponse.json( + { error: "Failed to fetch Azure Speech token" }, + { status: 502 }, + ); + } + + const token = await tokenRes.text(); + return NextResponse.json({ token, region }); +} diff --git a/app/interview/page.tsx b/app/interview/page.tsx index 6b0b88d..f49f4d6 100644 --- a/app/interview/page.tsx +++ b/app/interview/page.tsx @@ -1,7 +1,7 @@ "use client"; import { useRouter } from "next/navigation"; -import { useState } from "react"; +import { useRef, useState } from "react"; import { SiteNavbar } from "@/components/layout/SiteNavbar"; import { useLanguage } from "@/components/providers/language-provider"; import { createClient } from "@/lib/supabase/client"; @@ -83,6 +83,53 @@ export default function InterviewPage() { improvements: string[]; }>({ score: 0, strengths: [], improvements: [] }); + const [micState, setMicState] = useState<"idle" | "listening" | "processing">("idle"); + const recognizerRef = useRef<{ + stopContinuousRecognitionAsync: (cb?: () => void, err?: (e: string) => void) => void; + } | null>(null); + + async function handleMicClick() { + if (micState === "listening") { + recognizerRef.current?.stopContinuousRecognitionAsync( + () => setMicState("idle"), + () => setMicState("idle"), + ); + return; + } + + setMicState("processing"); + try { + const res = await fetch("/api/speech"); + if (!res.ok) throw new Error("Failed to get speech token"); + const { token, region } = await res.json(); + + const SpeechSDK = await import("microsoft-cognitiveservices-speech-sdk"); + const speechConfig = SpeechSDK.SpeechConfig.fromAuthorizationToken(token, region); + speechConfig.speechRecognitionLanguage = "en-SG"; + const audioConfig = SpeechSDK.AudioConfig.fromDefaultMicrophoneInput(); + const recognizer = new SpeechSDK.SpeechRecognizer(speechConfig, audioConfig); + + recognizerRef.current = recognizer; + + recognizer.recognized = (_: unknown, e: { result: { reason: number; text: string } }) => { + if (e.result.reason === SpeechSDK.ResultReason.RecognizedSpeech && e.result.text) { + setTextInput((prev) => (prev ? prev + " " + e.result.text : e.result.text)); + } + }; + + recognizer.startContinuousRecognitionAsync( + () => setMicState("listening"), + (err: string) => { + console.error("Speech recognition error:", err); + setMicState("idle"); + }, + ); + } catch (err) { + console.error("Mic setup failed:", err); + setMicState("idle"); + } + } + function startSession() { setSessionStarted(true); setConversation([ @@ -346,9 +393,19 @@ export default function InterviewPage() { diff --git a/app/resume/page.tsx b/app/resume/page.tsx index daeb4d6..db11346 100644 --- a/app/resume/page.tsx +++ b/app/resume/page.tsx @@ -24,6 +24,11 @@ export default function ResumeOnboardingPage() { const [successMessage, setSuccessMessage] = useState(null); const [lastResumeId, setLastResumeId] = useState(null); const [userResumes, setUserResumes] = useState([]); + const [micState, setMicState] = useState<"idle" | "listening" | "processing">("idle"); + const [voiceText, setVoiceText] = useState(""); + const recognizerRef = useRef<{ + stopContinuousRecognitionAsync: (cb?: () => void, err?: (e: string) => void) => void; + } | null>(null); useEffect(() => { const supabase = createClient(); @@ -165,6 +170,48 @@ export default function ResumeOnboardingPage() { } }, [router, selectedFile, refetchResumeStatus, t]); + async function handleMicClick() { + if (micState === "listening") { + recognizerRef.current?.stopContinuousRecognitionAsync( + () => setMicState("idle"), + () => setMicState("idle"), + ); + return; + } + + setMicState("processing"); + try { + const res = await fetch("/api/speech"); + if (!res.ok) throw new Error("Failed to get speech token"); + const { token, region } = await res.json(); + + const SpeechSDK = await import("microsoft-cognitiveservices-speech-sdk"); + const speechConfig = SpeechSDK.SpeechConfig.fromAuthorizationToken(token, region); + speechConfig.speechRecognitionLanguage = "en-SG"; + const audioConfig = SpeechSDK.AudioConfig.fromDefaultMicrophoneInput(); + const recognizer = new SpeechSDK.SpeechRecognizer(speechConfig, audioConfig); + + recognizerRef.current = recognizer; + + recognizer.recognized = (_: unknown, e: { result: { reason: number; text: string } }) => { + if (e.result.reason === SpeechSDK.ResultReason.RecognizedSpeech && e.result.text) { + setVoiceText((prev) => (prev ? prev + " " + e.result.text : e.result.text)); + } + }; + + recognizer.startContinuousRecognitionAsync( + () => setMicState("listening"), + (err: string) => { + console.error("Speech recognition error:", err); + setMicState("idle"); + }, + ); + } catch (err) { + console.error("Mic setup failed:", err); + setMicState("idle"); + } + } + const reviewHref = lastResumeId != null ? `/resume/review?resume_id=${encodeURIComponent(lastResumeId)}` @@ -223,13 +270,39 @@ export default function ResumeOnboardingPage() { void handleUpload(); }} > - +
+ + +
{selectedFileName ? (

@@ -255,6 +328,19 @@ export default function ResumeOnboardingPage() { +

+ +