diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..15ab664 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + ci: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - run: npm ci + + - name: Lint + run: npm run lint + + - name: Test + run: npm test + + - name: Build + run: npm run build diff --git a/app/pictogram/page.tsx b/app/pictogram/page.tsx index 309fb83..96429e8 100644 --- a/app/pictogram/page.tsx +++ b/app/pictogram/page.tsx @@ -5,6 +5,11 @@ import DesktopLayout from "@/components/DesktopLayout"; import { pictogramTree, PictogramNode } from "@/lib/pictogramTree"; import { useArasaacImage } from "@/lib/useArasaacImage"; import { useSpeech } from "@/hooks/useSpeech"; +import AudioRecorder from "@/components/AudioRecorder"; +import { + resolveArasaacImages, + WordPictogram, +} from "@/lib/resolveArasaacImages"; function IconChevronSmall() { return ( @@ -189,6 +194,320 @@ function TrailChips({ trail }: { trail: PictogramNode[] }) { ); } +// ─── Speech evaluation types ────────────────────────────────────────────────── + +interface EvaluationWord { + word: string; + accuracy_score: number; + is_omitted: boolean; +} + +interface EvaluationResult { + target: string; + overall_accuracy: number; + words: EvaluationWord[]; +} + +// ─── Speech helpers ─────────────────────────────────────────────────────────── + +// Each repetition gets slower and more emphatic +const SPEECH_PASSES = [ + { rate: 0.4, pitch: 1.2 }, + { rate: 0.2, pitch: 1.1 }, + { rate: 0.1, pitch: 1.0 }, +]; + +function speakAt(text: string, rate: number, pitch: number): Promise { + return new Promise((resolve) => { + if (typeof window === "undefined" || !window.speechSynthesis) { resolve(); return; } + window.speechSynthesis.cancel(); + const utt = new SpeechSynthesisUtterance(text); + utt.rate = rate; utt.pitch = pitch; utt.volume = 1; utt.lang = "en-US"; + utt.onend = () => resolve(); utt.onerror = () => resolve(); + window.speechSynthesis.speak(utt); + }); +} + +function msDelay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +// ─── Word-by-word speech practice ──────────────────────────────────────────── + +type WordPhase = "teaching" | "pronounce" | "result"; + +function SpeechPractice({ + words, + onBack, +}: { + words: WordPictogram[]; + onBack: () => void; +}) { + const [wordIndex, setWordIndex] = useState(0); + const [phase, setWordPhase] = useState("teaching"); + const [wordResult, setWordResult] = useState(null); + const [allDone, setAllDone] = useState(false); + const [error, setError] = useState(null); + const [isSpeaking, setIsSpeaking] = useState(false); + const [speakPass, setSpeakPass] = useState(0); + const cancelledRef = useRef(false); + + const currentWord = words[wordIndex]; + // Single-letter "I" gets a period so TTS doesn't spell it out + const ttsText = currentWord.label === "I" ? "I." : currentWord.label; + + const playWordSlowly = useCallback(async () => { + if (isSpeaking) return; + cancelledRef.current = false; + setIsSpeaking(true); + for (let i = 0; i < SPEECH_PASSES.length; i++) { + if (cancelledRef.current) break; + setSpeakPass(i); + const { rate, pitch } = SPEECH_PASSES[i]; + await speakAt(ttsText, rate, pitch); + if (i < SPEECH_PASSES.length - 1 && !cancelledRef.current) await msDelay(1800); + } + setIsSpeaking(false); + }, [ttsText, isSpeaking]); + + // Auto-play when entering teaching phase + useEffect(() => { + if (phase !== "teaching") return; + let isCancelled = false; + cancelledRef.current = false; + const run = async () => { + setIsSpeaking(true); + await msDelay(400); + for (let i = 0; i < SPEECH_PASSES.length; i++) { + if (isCancelled || cancelledRef.current) break; + setSpeakPass(i); + const { rate, pitch } = SPEECH_PASSES[i]; + await speakAt(ttsText, rate, pitch); + if (i < SPEECH_PASSES.length - 1 && !isCancelled && !cancelledRef.current) { + await msDelay(1800); + } + } + if (!isCancelled) setIsSpeaking(false); + }; + run(); + return () => { + isCancelled = true; + cancelledRef.current = true; + window.speechSynthesis?.cancel(); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [wordIndex, phase]); + + const handleResult = useCallback((result: EvaluationResult) => { + setWordResult(result.words[0] ?? null); + setError(null); + setWordPhase("result"); + }, []); + + const handleNext = useCallback(() => { + cancelledRef.current = true; + window.speechSynthesis?.cancel(); + if (wordIndex < words.length - 1) { + setWordIndex((i) => i + 1); + setWordResult(null); + setError(null); + setWordPhase("teaching"); + } else { + setAllDone(true); + } + }, [wordIndex, words.length]); + + const handleRetry = useCallback(() => { + setWordResult(null); + setError(null); + setWordPhase("teaching"); + }, []); + + // ── All done ───────────────────────────────────────────────────────────── + + if (allDone) { + return ( +
+
🎉
+
Great job!
+
You said all the words!
+ +
+ ); + } + + // ── Main practice screen ────────────────────────────────────────────────── + + return ( +
+ {/* Progress dots */} +
+ {words.map((_, i) => ( +
+ ))} +
+ +
+ {/* Pictogram */} +
+ {currentWord.imageUrl ? ( + // eslint-disable-next-line @next/next/no-img-element + {currentWord.label} + ) : ( + {currentWord.label} + )} +
+ + {/* Word */} +
+ {currentWord.label} +
+ + {/* Teaching phase */} + {phase === "teaching" && ( +
+
+
+ 🔊 + {speakPass === 0 ? "Listen…" : speakPass === 1 ? "Slower…" : "Very slow…"} +
+
+ {SPEECH_PASSES.map((_, i) => ( +
+ ))} +
+
+ + + + +
+ )} + + {/* Pronounce phase */} + {phase === "pronounce" && ( +
+
+ Say: “{currentWord.label}” +
+ + setError(e)} + /> + + +
+ )} + + {/* Result phase */} + {phase === "result" && wordResult && ( +
+ {wordResult.accuracy_score >= 70 ? ( + <> +
✅
+
Well done!
+
{wordResult.accuracy_score}% correct
+ + + ) : ( + <> +
🔄
+
Try again!
+
+ {wordResult.is_omitted ? "Not heard" : `${wordResult.accuracy_score}% correct`} +
+ + + + )} +
+ )} + + {/* Error display */} + {error && ( +
+ {error.includes("AZURE_SPEECH_KEY") ? ( + <> +
Speech check not set up yet
+
Azure credentials are missing. You can still practice!
+ + + ) : ( +
{error}
+ )} +
+ )} +
+ +
+ +
+
+ ); +} + // ─── Output screen ──────────────────────────────────────────────────────────── function OutputScreen({ @@ -200,29 +519,32 @@ function OutputScreen({ }) { const [sentence, setSentence] = useState(null); const [loading, setLoading] = useState(true); + const [showPractice, setShowPractice] = useState(false); + const [wordPictograms, setWordPictograms] = useState([]); const fetchSentence = useCallback(async () => { setLoading(true); setSentence(null); + setShowPractice(false); try { const res = await fetch("/api/generate", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ trail: trail.map((n) => n.label), - context: trail - .map((n) => n.llmContext) - .filter(Boolean), + context: trail.map((n) => n.llmContext).filter(Boolean), }), }); const data = await res.json(); - if (data.sentence) { - setSentence(data.sentence); - } else { - setSentence("I need help."); - } + const text = data.sentence || "I need help."; + setSentence(text); + // Pre-resolve pictogram images for each sentence word + const words = text.trim().split(/\s+/); + resolveArasaacImages(words).then(setWordPictograms); } catch { - setSentence("I need help."); + const fallback = "I need help."; + setSentence(fallback); + resolveArasaacImages(["I", "need", "help"]).then(setWordPictograms); } finally { setLoading(false); } @@ -230,7 +552,6 @@ function OutputScreen({ const fetchedRef = useRef(false); - // Fetch on mount useEffect(() => { if (fetchedRef.current) return; fetchedRef.current = true; @@ -241,10 +562,14 @@ function OutputScreen({ // Auto-speak when sentence arrives useEffect(() => { - if (sentence) { + if (sentence && !showPractice) { speak(sentence, { rate: 0.85, pitch: 1 }); } - }, [sentence, speak]); + }, [sentence, speak, showPractice]); + + if (showPractice && wordPictograms.length > 0) { + return setShowPractice(false)} />; + } return (
@@ -293,11 +618,12 @@ function OutputScreen({ Grid
@@ -388,4 +714,4 @@ export default function PictogramPage() {
); -} +} \ No newline at end of file diff --git a/components/PronunciationFeedback.tsx b/components/PronunciationFeedback.tsx index 7e79c98..93c7c81 100644 --- a/components/PronunciationFeedback.tsx +++ b/components/PronunciationFeedback.tsx @@ -20,7 +20,7 @@ import React, { useState, useEffect, useRef, useCallback } from "react"; interface Pictogram { id: string; label: string; - imageUrl: string; + imageUrl: string | null; } interface WordScore { @@ -34,7 +34,7 @@ type WordStatus = "pending" | "active" | "correct" | "incorrect"; interface WordState { id: string; label: string; - imageUrl: string; + imageUrl: string | null; status: WordStatus; score: number; isOmitted: boolean; @@ -280,13 +280,19 @@ export default function PronunciationFeedback({ }} aria-label={entry.label} > - {/* Pictogram image */} - {/* eslint-disable-next-line @next/next/no-img-element */} - + {/* Pictogram image or word fallback */} + {entry.imageUrl ? ( + /* eslint-disable-next-line @next/next/no-img-element */ + + ) : ( + + {entry.label} + + )} {/* Score ring (circular progress) */} {(entry.status === "correct" || entry.status === "incorrect") && ( diff --git a/lib/resolveArasaacImages.ts b/lib/resolveArasaacImages.ts new file mode 100644 index 0000000..91100ee --- /dev/null +++ b/lib/resolveArasaacImages.ts @@ -0,0 +1,51 @@ +/** + * Resolves ARASAAC pictogram image URLs for a list of words. + * Uses the same API endpoint as useArasaacImage hook but as a plain async function + * suitable for non-hook contexts. + */ + +export interface WordPictogram { + id: string; + label: string; + imageUrl: string | null; +} + +const imageCache = new Map(); + +async function fetchImageUrl(keyword: string): Promise { + if (imageCache.has(keyword)) return imageCache.get(keyword)!; + + try { + const res = await fetch( + `https://api.arasaac.org/v1/pictograms/en/search/${encodeURIComponent(keyword)}` + ); + const data = await res.json(); + if (Array.isArray(data) && data.length > 0) { + const id = data[0]._id; + const url = `https://static.arasaac.org/pictograms/${id}/${id}_2500.png`; + imageCache.set(keyword, url); + return url; + } + } catch { + // ignore fetch errors + } + + imageCache.set(keyword, null); + return null; +} + +export async function resolveArasaacImages( + words: string[] +): Promise { + const results = await Promise.all( + words.map(async (word, i) => { + const imageUrl = await fetchImageUrl(word.toLowerCase()); + return { + id: `word-${i}-${word}`, + label: word, + imageUrl, + }; + }) + ); + return results; +}