From c76674c1df6e4813fea31944ff8a2c64935e3d8d Mon Sep 17 00:00:00 2001 From: T04435 Date: Fri, 23 Oct 2026 21:59:04 +1100 Subject: [PATCH 1/3] fix(research): research always ends, and Busk uses your sound-alikes v1.1.1. - Research no longer hangs on "Loading your songs": own songs are saved in one batched insert, the Spotify catalogue fetch has a 20 s deadline (partial results), each step times out at 45 s and a watchdog ends the run before the 120 s function limit. A stream that drops without an end shows Stopped with Try again; a reload waits for a run still running elsewhere. Stale sweep raised to 150 s. - /busks starts from the Repertoire's sound-alikes (the form remounts when the Repertoire changes; "plan again" merges instead of replacing). For an act Qloo doesn't know, suggestions come from the sound-alikes and the answered genres go to Qloo as signal.interests.tags. - Step feed: one-line summaries sit beside the hex cell; no empty rows. - Layout test re-queries the footer link instead of failing on a detach. - package.json 1.1.1. Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- src/app/api/playlist/research/route.ts | 4 +- src/app/busks/page.tsx | 10 +- src/components/landing/performer-form.tsx | 31 +++- src/components/plan/agent-steps.tsx | 22 ++- src/components/plan/tool-results.tsx | 9 +- src/components/playlist/research-view.tsx | 46 ++++- src/lib/repertoire-research-view.ts | 6 +- src/lib/repertoire-research.ts | 14 +- src/server/adapters/qloo/client.ts | 50 ++++-- src/server/adapters/qloo/index.ts | 15 +- src/server/adapters/repertoire/client.ts | 119 +++++++------ src/server/adapters/repertoire/pacer.ts | 25 ++- src/server/agent/context.ts | 34 ++-- src/server/playlist/config.ts | 13 +- src/server/playlist/own-artist.ts | 16 +- src/server/playlist/store.ts | 101 +++++++++++ src/server/request.ts | 12 ++ src/server/research/run.ts | 80 ++++++++- tests/e2e/layout-helpers.ts | 5 +- tests/e2e/repertoire-research.spec.ts | 13 ++ tests/unit/adapters/repertoire-pacer.test.ts | 47 +++++- tests/unit/playlist/store.test.ts | 49 ++++++ .../playlist/unknown-act-sounds-like.test.ts | 159 ++++++++++++++++++ tests/unit/research/run-timeouts.test.ts | 103 ++++++++++++ tests/unit/research/view.test.ts | 10 ++ 26 files changed, 863 insertions(+), 132 deletions(-) create mode 100644 tests/unit/playlist/unknown-act-sounds-like.test.ts create mode 100644 tests/unit/research/run-timeouts.test.ts diff --git a/package.json b/package.json index 2b636766..53daac8d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "busk-map", - "version": "1.1.0", + "version": "1.1.1", "private": true, "scripts": { "dev": "node scripts/copy-maplibre-worker.mjs && next dev", diff --git a/src/app/api/playlist/research/route.ts b/src/app/api/playlist/research/route.ts index fd3045d2..98dc3a7d 100644 --- a/src/app/api/playlist/research/route.ts +++ b/src/app/api/playlist/research/route.ts @@ -9,7 +9,7 @@ import { ensurePlaylist, guardMutation, isResponse, readSession } from "@/server import { LEARN_MAX_PENDING, LEARN_WINDOW_MINUTES } from "@/server/playlist/config"; import { executeResearchRun } from "@/server/research/run"; import { learnRunGate, refundLearnRun } from "@/server/rate-limit"; -import { adapterContextFor, clientIp, FAULTS_HEADER, learnLimitFaultRequested, qlooBudgetFaultRequested, researchStepDelayMs } from "@/server/request"; +import { adapterContextFor, clientIp, FAULTS_HEADER, learnLimitFaultRequested, qlooBudgetFaultRequested, researchCutRequested, researchStepDelayMs } from "@/server/request"; /** A run takes well under a minute; the summary call times out at 30 s, then the data's words are used. */ export const maxDuration = 120; @@ -86,7 +86,7 @@ export async function POST(request: Request) { registerLearnRun(run); const ip = clientIp(request.headers); const exempt = limitsExempt(); - void executeResearchRun({ learn, playlists: session.store, playlist, spotifyId, mode: body.mode, adapters: adapterContextFor(faults), record: started.run, run, stepDelayMs: researchStepDelayMs(faults), budgetFault: qlooBudgetFaultRequested(faults), ...(body.mode === "full" && body.qlooChoice ? { qlooChoice: body.qlooChoice } : {}), ...(body.mode === "full" && body.genre ? { genre: body.genre } : {}) }).then(() => { + void executeResearchRun({ learn, playlists: session.store, playlist, spotifyId, mode: body.mode, adapters: adapterContextFor(faults), record: started.run, run, stepDelayMs: researchStepDelayMs(faults), ...(researchCutRequested(faults) ? { cutAt: "songs" as const } : {}), budgetFault: qlooBudgetFaultRequested(faults), ...(body.mode === "full" && body.qlooChoice ? { qlooChoice: body.qlooChoice } : {}), ...(body.mode === "full" && body.genre ? { genre: body.genre } : {}) }).then(() => { // A run that ended with Qloo or Spotify down didn't count: its network hit is given back. const end = run.events.at(-1); if (!exempt && end?.type === "end") refundLearnRun(end.code, ip); diff --git a/src/app/busks/page.tsx b/src/app/busks/page.tsx index 90a90d2b..75aab38c 100644 --- a/src/app/busks/page.tsx +++ b/src/app/busks/page.tsx @@ -34,6 +34,12 @@ export default function BusksPage({ searchParams }: PageProps<"/busks">) { ); } +/** + * The form's key: a Repertoire saved since the form was drawn (research adding sound-alikes, say) starts it afresh. Cache + * Components keeps this page mounted while hidden (React Activity), and new props alone never reset the form's state. + */ +const formKey = (base: string, repertoire: { revision: number } | null) => `${base}:${repertoire?.revision ?? 0}`; + /** * `?again=` (from a plan whose slot has passed): the form pre-filled with that act, on the next same weekday (an * any-time plan opens on Any time, on the next date a window can still start). Without it, the form defaults to Any time @@ -64,7 +70,7 @@ async function OwnAct({ searchParams }: Pick, "searchParams" ))} - + ); } @@ -80,7 +86,7 @@ async function OwnAct({ searchParams }: Pick, "searchParams" Planning again for {performer.name}. Your act, artists, songs and permit come from your earlier busk: pick a new date and time. s.replace(/[|~]/g, " ").trim(); @@ -67,15 +67,32 @@ const prefillSongs = (prefill: FormPrefill | undefined): PickedSong[] => own: song.own, })); -/** The Repertoire as the form's fields (docs/playlist-architecture.md §3.1): name, sounds-like artists, songs. */ +/** The Repertoire as the form's fields (docs/playlist-architecture.md §3.1): name and songs (its sound-alikes: startingArtists). */ const repertoireFields = (view: PlaylistView) => ({ name: view.actName ?? "", - artists: view.soundsLike.map((a): PickedArtist => ({ name: a.name, spotifyId: a.spotifyId ?? null, qlooId: a.qlooId ?? null, image: a.image ?? null })), picked: view.songs.map( (song): PickedSong => ({ key: `rep-${song.id}`, title: song.title, artist: song.artist, durationMs: song.durationMs, own: song.own }), ), }); +/** + * The artists the form starts with: planning again, the plan's artists, then the Repertoire's sound-alikes it lacks (by name); + * otherwise the Repertoire's. Never fewer than the Repertoire has (up to the picker's maximum), so an act Qloo doesn't know + * still starts with the sound-alikes its research saved. + */ +export function startingArtists(prefill: FormPrefill | undefined, repertoire: Pick | null): PickedArtist[] { + const fromRep = repertoire ? repertoire.soundsLike.map((a): PickedArtist => ({ name: a.name, spotifyId: a.spotifyId ?? null, qlooId: a.qlooId ?? null, image: a.image ?? null })) : []; + const out = prefillArtists(prefill); + for (const a of fromRep) { + if (out.length >= MAX_PICKED_ARTISTS) break; + const at = out.findIndex((b) => b.name.toLowerCase() === a.name.toLowerCase()); + // The plan's copy keeps its place; the Repertoire fills in ids and the photo it lacks. + if (at >= 0) out[at] = { ...out[at], spotifyId: out[at].spotifyId ?? a.spotifyId, qlooId: out[at].qlooId ?? a.qlooId ?? null, image: out[at].image ?? a.image }; + else out.push(a); + } + return out; +} + /** The Repertoire's saved Spotify profile as ArtistProfile draws it (read-only here: no audience tags, no refresh). */ const profileOf = (profile: SpotifyProfile) => ({ name: profile.name, image: profilePhoto(profile.images), genres: profile.genres, audience: [], followers: profile.followers ?? null }); @@ -124,7 +141,8 @@ export function PerformerForm({ defaultDate, today, prefill, repertoire = null } const rep = repertoire; const repAct = rep && !prefill ? repertoireFields(rep) : null; const [name, setName] = useState(repAct?.name ?? prefill?.performer.name ?? ""); - const [artists, setArtists] = useState(() => repAct?.artists ?? prefillArtists(prefill)); + // The artists start from the plan (planning again) and the Repertoire's sound-alikes together, never one wiping the other. + const [artists, setArtists] = useState(() => startingArtists(prefill, rep)); const [artistQuery, setArtistQuery] = useState(""); // The songs the busk carries: the Repertoire's (or, planning again, the last busk's). const songs = useMemo(() => (prefill ? prefillSongs(prefill) : rep ? repertoireFields(rep).picked : []), [prefill, rep]); @@ -163,7 +181,8 @@ export function PerformerForm({ defaultDate, today, prefill, repertoire = null } /** What the form holds now, as the request will carry it. */ function currentFields() { const parsedSongs = songs.map((p) => ({ title: p.title, artist: p.artist, own: p.own, ...(p.durationMs ? { durationMs: p.durationMs } : {}) })); - const chosen = [...artists]; + // An empty picker never fails the busk while the Repertoire has sound-alikes: they are the act's answer to "who you sound like". + const chosen = artists.length > 0 ? [...artists] : startingArtists(undefined, rep); for (const typed of typedArtists(artistQuery)) { if (!chosen.some((a) => a.name.toLowerCase() === typed.toLowerCase())) chosen.push({ name: typed, spotifyId: null, image: null }); } @@ -274,7 +293,7 @@ export function PerformerForm({ defaultDate, today, prefill, repertoire = null } a.qlooId ?? "").join(",")}`} picked={currentFields().chosen.map((a) => a.name)} onAdd={(add) => setArtists((current) => withSuggested(current, add))} /> diff --git a/src/components/plan/agent-steps.tsx b/src/components/plan/agent-steps.tsx index 4d2d7e9b..9ec142a6 100644 --- a/src/components/plan/agent-steps.tsx +++ b/src/components/plan/agent-steps.tsx @@ -8,6 +8,7 @@ import { type CSSProperties, type ReactNode } from "react"; import { cn } from "cn"; import { HexCell } from "@/components/honeycomb/hex-cell"; import { Icon } from "@/components/honeycomb/icon"; +import { RichText } from "@/components/honeycomb/mention"; import { Spinner } from "@/components/ui/spinner"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Badge } from "@/components/ui/badge"; @@ -58,7 +59,9 @@ function ToolCard({ step, model, when, pins, fresh, renderWaiting }: { step: Age const say = step.notes.filter((n) => n.kind === "text").map((n) => n.text); const title = running ? `${ui?.doing(step) ?? step.label}…` : (ui?.title(step) ?? step.label); const out = step.output !== null && typeof step.output === "object" ? (step.output as Record) : {}; - const foot = (text: string) =>

{text}

; + const foot = (text: ReactNode) =>

{text}

; + // One-line results sit right under the title, beside the step cell (no gap under a one-line title); only wide content goes below. + let lead: ReactNode = null; let body: ReactNode = null; if (running) body = ( @@ -68,11 +71,17 @@ function ToolCard({ step, model, when, pins, fresh, renderWaiting }: { step: Age ); else if (rejected) body = ui?.render(out, step, { when, pins, model }); - else if (bad) body = foot("This check didn't come back, so the plan goes on without it."); - else if (cut) body = foot("Stopped before it finished."); - else if (waiting) body = renderWaiting ? renderWaiting(step) : step.summary ? foot(step.summary) : null; - else if (step.output !== undefined && ui) body = ui.render(out, step, { when, pins, model }); - else if (step.summary) body = foot(step.summary); + else if (bad) lead = foot("This check didn't come back, so the plan goes on without it."); + else if (cut) lead = foot("Stopped before it finished."); + else if (waiting) { + if (renderWaiting) body = renderWaiting(step); + else if (step.summary) lead = foot(step.summary); + } else if (step.output !== undefined && ui) { + const line = ui.line?.(out); + if (line !== undefined) { + if (line.length) lead = foot(); + } else body = ui.render(out, step, { when, pins, model }); + } else if (step.summary) lead = foot(step.summary); return (
  • ) : null} + {lead} {rejected ? "rejected" : waiting ? "waiting for you" : step.status} diff --git a/src/components/plan/tool-results.tsx b/src/components/plan/tool-results.tsx index a173440e..f4ef80db 100644 --- a/src/components/plan/tool-results.tsx +++ b/src/components/plan/tool-results.tsx @@ -176,7 +176,13 @@ const Section = ({ title, children }: { title: string; children: ReactNode }) => /** `pins`: the busk's ranked pitches, once the plan exists, so a card's mini map can show them. */ export type MiniPin = { id: string; name: string; lat: number; lon: number; rank: number }; type Ctx = { when?: string; pins?: MiniPin[]; model?: ExplanationsModel | null }; -type ToolUi = { doing: (s: AgentStep) => string; title: (s: AgentStep) => string; render: (out: Rec, s: AgentStep, ctx: Ctx) => ReactNode }; +type ToolUi = { + doing: (s: AgentStep) => string; + title: (s: AgentStep) => string; + render: (out: Rec, s: AgentStep, ctx: Ctx) => ReactNode; + /** A tool whose whole result is one line: the feed sets it right under the title, beside the step cell. */ + line?: (out: Rec) => RichTextData | string; +}; /** "Gillian Welch 0.31; Paul Kelly 0.2" → the because chips, each with its explained weight (0–100). */ const becauseArtists = (s: string) => @@ -193,6 +199,7 @@ const becauseArtists = (s: string) => const researchUi = (line: (o: Rec) => RichTextData | string): ToolUi => ({ doing: (s) => s.label, title: (s) => s.label, + line, render: (o) => ( diff --git a/src/components/playlist/research-view.tsx b/src/components/playlist/research-view.tsx index 3abb5132..4506274e 100644 --- a/src/components/playlist/research-view.tsx +++ b/src/components/playlist/research-view.tsx @@ -132,6 +132,8 @@ function ListScroll({ label, maxHeight = LIST_MAX_HEIGHT, children }: { label: s } const R = COPY.research; +/** How often the list is read while a joined run is going on another instance (409 running_elsewhere). */ +const ELSEWHERE_POLL_MS = 3000; /** shadcn's destructive description is destructive/90 (4.49:1): full strength passes AA. */ const DESTRUCTIVE = "*:data-[slot=alert-description]:text-destructive"; @@ -171,6 +173,8 @@ export function ResearchView() { const [events, setEvents] = useState(null); const [replay, setReplay] = useState(null); const [running, setRunning] = useState(false); + /** The last stream ended with no `end`: the run is shown stopped (the step cut short, Try again), never as still going. */ + const [cut, setCut] = useState(false); const [notice, setNotice] = useState<{ kind: "error" | "info"; text: string } | null>(null); const [info, setInfo] = useState(""); const [busy, setBusy] = useState(false); @@ -206,6 +210,7 @@ export function ResearchView() { const consume = useCallback( async (stream: AsyncGenerator) => { setRunning(true); + setCut(false); let ended = false; try { for await (const event of stream) { @@ -217,7 +222,8 @@ export function ResearchView() { // below } if (!mounted.current) return; - if (!ended) setNotice({ kind: "error", text: COPY.learn.errors.unavailable }); + // A stream that ends with no `end` (dropped, or the server stopped mid-step) is a stopped run: the feed's own alert says so. + if (!ended) setCut(true); setRunning(false); await reload(); }, @@ -237,12 +243,26 @@ export function ResearchView() { }; }, [apply]); - // A run already going (a reload, a second tab) is joined. + // A run already going (a reload, a second tab) is joined. One going on another server instance (or one whose function was + // stopped, until the stale sweep marks it failed) is a 409 running_elsewhere: the list is polled until the run leaves + // `running`, then it is replayed (its end, or the failed state the sweep gave it). const runningId = list?.running?.runId ?? null; useEffect(() => { if (!runningId || running || joined.current === runningId) return; joined.current = runningId; - void joinLearn(runningId).then((r) => r.ok && mounted.current && void consume(r.events)); + const still = () => mounted.current && joined.current === runningId; + void (async () => { + let r = await joinLearn(runningId); + while (!r.ok && r.code === "running_elsewhere" && still()) { + await new Promise((resolve) => setTimeout(resolve, ELSEWHERE_POLL_MS)); + if (!still()) return; + const l = await loadLearn(); + if (!still()) return; + if (l.ok && l.list.running?.runId === runningId) continue; + r = await joinLearn(runningId); + } + if (r.ok && still()) void consume(r.events); + })(); }, [runningId, running, consume]); // The folded research of a saved run: its stored steps, replayed once (no model, no Qloo). @@ -262,8 +282,8 @@ export function ResearchView() { }, [snapshotRun, events, running]); const view: RepertoireView = useMemo( - () => toRepertoireView({ snapshot: playlist?.research ?? null, events, playlist, suggestions: running ? null : (list?.suggestions ?? null) }), - [playlist, events, list, running], + () => toRepertoireView({ snapshot: playlist?.research ?? null, events, playlist, suggestions: running ? null : (list?.suggestions ?? null), cut: cut && !running }), + [playlist, events, list, running, cut], ); useEffect(() => { @@ -276,6 +296,7 @@ export function ResearchView() { async function start(body: { link?: string; refresh?: boolean; mode?: "full" | "more"; qlooChoice?: { id: string } | { none: true }; genre?: string }) { setNotice(null); setInfo(""); + setCut(false); setRunning(true); setEvents([]); const result = await startResearch(body, pageFaults()); @@ -348,7 +369,15 @@ export function ResearchView() { const research = playlist?.research ?? null; const hasContent = view.phase !== "empty" || (playlist?.songs.length ?? 0) > 0 || (list?.suggestions.length ?? 0) > 0; const showForm = !running && (!research || changingLink); - const live = running || (events !== null && !view.end); + const stopped = cut && !running && events !== null && !view.end; + const live = running || (events !== null && !view.end && !stopped); + /** Try again after a stopped run: the same kind of run, for the same link. */ + const retryCut = () => { + const kind = events?.find((e) => e.type === "run"); + const id = playlist?.spotifyArtistId ?? research?.spotifyArtistId ?? parseSpotifyArtistId(link.trim()); + if (kind?.type === "run" && kind.inputs.kind === "more") void start({ mode: "more" }); + else if (id) void start({ link: id, refresh: true }); + }; const feedEvents = events && events.length > 0 ? events : replay; const feed = feedEvents ? stepsOf(feedEvents) : []; const songs = playlist?.songs ?? []; @@ -414,7 +443,8 @@ export function ResearchView() { 0 ?
    {researchFeed()}
    : null; + const researchNode = live || stopped ? researchFeed() : research && feed.length > 0 ?
    {researchFeed()}
    : null; const showResults = (hasContent || running) && (running || !changingLink); const results = (
    diff --git a/src/lib/repertoire-research-view.ts b/src/lib/repertoire-research-view.ts index 0e672bc0..202cb21c 100644 --- a/src/lib/repertoire-research-view.ts +++ b/src/lib/repertoire-research-view.ts @@ -59,6 +59,8 @@ export type ViewInput = { playlist?: Pick & Partial> | null; /** The Learn list's suggestions (GET /api/playlist/learn); when given they are the To learn list. */ suggestions?: readonly LearnSuggestionView[] | null; + /** The live stream ended with no `end` (the connection dropped, or the server was stopped): the run is over, cut short. */ + cut?: boolean; }; type Outputs = { [K in ResearchStepKey]?: StepOutputs[K] | null }; @@ -208,7 +210,7 @@ export function toRepertoireView(input: ViewInput): RepertoireView { const paused = end?.code === "needs_sounds_like" || (!run && Boolean(snapshot?.waiting)); let st = status.get(key) ?? (snapshot && !run && !paused ? "skipped" : "pending"); // A live run that ended: a step it never started was skipped (or cut short, if it was still running). A paused run's are not started yet. - if (end && !paused && ran.includes(key) && (st === "pending" || st === "running")) st = st === "running" ? "failed" : "skipped"; + if ((end || input.cut) && !paused && ran.includes(key) && (st === "pending" || st === "running")) st = st === "running" ? "failed" : "skipped"; const f = found.get(key); const songsFrom = key === "songs" && st === "done" && outputs.songs && outputs.songs.source !== "none" ? outputs.songs.source : undefined; const source = via.get(key) ?? songsFrom; @@ -260,7 +262,7 @@ export function toRepertoireView(input: ViewInput): RepertoireView { learned: playlist ? playlist.songs.filter((s) => s.state === "learned").map((s) => songOf(s, media, ownPhoto)) : null, }; - const phase: RepertoirePhase = run ? (end ? "done" : "running") : snapshot ? "done" : "empty"; + const phase: RepertoirePhase = run ? (end || input.cut ? "done" : "running") : snapshot ? "done" : "empty"; const mode = liveMode ?? snapshot?.mode ?? null; return { phase, diff --git a/src/lib/repertoire-research.ts b/src/lib/repertoire-research.ts index 8233fd88..9a03c5db 100644 --- a/src/lib/repertoire-research.ts +++ b/src/lib/repertoire-research.ts @@ -181,10 +181,18 @@ export type ResearchSnapshot = { /** The busk local scene's optional input: the act's genre tags (step 8), at most this many. */ export const MAX_ACT_GENRES = 6; -/** The act's genre tags from a snapshot (step 8, else the Qloo entity's genres by name), for the busk form. */ +/** + * The act's genre tags from a snapshot, for the busk form: the genres the performer answered when Qloo doesn't know the act + * (step 3's genreTags) first, then step 8's, once each by id. + */ export function actGenresOf(snapshot: ResearchSnapshot | null | undefined): { id: string; name: string }[] { - const tags = snapshot?.steps.genres?.output?.tags ?? []; - return tags.filter((t) => t.id && t.name).slice(0, MAX_ACT_GENRES).map((t) => ({ id: t.id, name: t.name })); + const qloo = snapshot?.steps.qloo?.output; + const answered = qloo && !qloo.matched ? (qloo.genreTags ?? []) : []; + const tags = [...answered, ...(snapshot?.steps.genres?.output?.tags ?? [])]; + return tags + .filter((t, i) => t.id && t.name && tags.findIndex((u) => u.id === t.id) === i) + .slice(0, MAX_ACT_GENRES) + .map((t) => ({ id: t.id, name: t.name })); } // --- POST /api/playlist/research --- diff --git a/src/server/adapters/qloo/client.ts b/src/server/adapters/qloo/client.ts index 48fbfaef..79564953 100644 --- a/src/server/adapters/qloo/client.ts +++ b/src/server/adapters/qloo/client.ts @@ -151,6 +151,11 @@ function listEntities(path: string, params: Params, items: unknown[]) { } const ids = (list: readonly string[]) => list.join(","); +/** + * Genre tags sent beside the entities (`signal.interests.tags`): the act's genres when Qloo doesn't know the act. A lookup + * key gains them only when there are some, so a call without tags keeps the key (and the cached answer) it always had. + */ +const withTags = (tags: readonly string[] | undefined): [readonly string[]] | [] => (tags?.length ? [tags] : []); // ---- Tool calls (spec.md, Agent design) ---------------------------------- @@ -220,13 +225,16 @@ export type HeatmapBoundary = "city" | "neighborhood"; /** `urn:heatmap`: where in the area this taste concentrates. */ /** `boundary` (U40): Qloo's `output.heatmap.boundary` (documented values "city" and "neighborhood"); omitted, cells are geohash squares. */ -export async function heatmap(backend: QlooBackend, entityIds: string[], area: BBox, boundary?: HeatmapBoundary): Promise { - return countLookup("heatmap", [backend, entityIds, area, boundary ?? null], () => heatmapCached(backend, entityIds, area, boundary)); +export async function heatmap(backend: QlooBackend, entityIds: string[], area: BBox, boundary?: HeatmapBoundary, tagIds: readonly string[] = []): Promise { + const tags = sceneTags(tagIds); + return countLookup("heatmap", [backend, entityIds, area, boundary ?? null, ...withTags(tags)], () => + tags.length ? heatmapCached(backend, entityIds, area, boundary, tags) : heatmapCached(backend, entityIds, area, boundary), + ); } -async function heatmapCached(backend: QlooBackend, entityIds: string[], area: BBox, boundary?: HeatmapBoundary): Promise { +async function heatmapCached(backend: QlooBackend, entityIds: string[], area: BBox, boundary?: HeatmapBoundary, tagIds?: string[]): Promise { "use cache: remote"; - return lookupScope("heatmap", [backend, entityIds, area, boundary ?? null], async () => { + return lookupScope("heatmap", [backend, entityIds, area, boundary ?? null, ...withTags(tagIds)], async () => { // A world-sized area is the Repertoire's "Where your fans are": fresh for 24 h. City-sized areas (the planner) keep "days". if (area.east - area.west > 180 || area.north - area.south > 120) cacheLife({ stale: 3600, revalidate: 86400, expire: 86400 }); else cacheLife("days"); @@ -234,6 +242,7 @@ async function heatmapCached(backend: QlooBackend, entityIds: string[], area: BB const params = { "filter.type": "urn:heatmap", "signal.interests.entities": ids(entityIds), + "signal.interests.tags": tagIds?.length ? ids(tagIds) : undefined, "filter.location": wktPolygon(area), ...(boundary ? { "output.heatmap.boundary": boundary } : {}), }; @@ -261,8 +270,12 @@ export async function places( center: LatLon, radiusM: number, weekday: string | null, + tagIds: readonly string[] = [], ): Promise { - return countLookup("places", [backend, entityIds, center, radiusM, weekday], () => placesCached(backend, entityIds, center, radiusM, weekday)); + const tags = sceneTags(tagIds); + return countLookup("places", [backend, entityIds, center, radiusM, weekday, ...withTags(tags)], () => + tags.length ? placesCached(backend, entityIds, center, radiusM, weekday, tags) : placesCached(backend, entityIds, center, radiusM, weekday), + ); } async function placesCached( @@ -271,9 +284,10 @@ async function placesCached( center: LatLon, radiusM: number, weekday: string | null, + tagIds?: string[], ): Promise { "use cache: remote"; - return lookupScope("places", [backend, entityIds, center, radiusM, weekday], async () => { + return lookupScope("places", [backend, entityIds, center, radiusM, weekday, ...withTags(tagIds)], async () => { cacheLife("days"); cacheTag("qloo", "qloo:places"); const params = { @@ -282,6 +296,7 @@ async function placesCached( "filter.location.radius": radiusM, "filter.hours": weekday ?? undefined, "signal.interests.entities": ids(entityIds), + "signal.interests.tags": tagIds?.length ? ids(tagIds) : undefined, take: MAX_TAKE, }; const body = await qlooGet(backend, "/v2/insights", params, entitiesResponseSchema); @@ -296,19 +311,23 @@ async function placesCached( * explanations: per result, and the top-level aggregate (each input artist's * average effect on the top 3, 5, 10 and all results). */ -export async function localArtists(backend: QlooBackend, entityIds: string[], area: BBox, take: number): Promise { - return countLookup("localArtists", [backend, entityIds, area, take], () => localArtistsCached(backend, entityIds, area, take)); +export async function localArtists(backend: QlooBackend, entityIds: string[], area: BBox, take: number, tagIds: readonly string[] = []): Promise { + const tags = sceneTags(tagIds); + return countLookup("localArtists", [backend, entityIds, area, take, ...withTags(tags)], () => + tags.length ? localArtistsCached(backend, entityIds, area, take, tags) : localArtistsCached(backend, entityIds, area, take), + ); } -async function localArtistsCached(backend: QlooBackend, entityIds: string[], area: BBox, take: number): Promise { +async function localArtistsCached(backend: QlooBackend, entityIds: string[], area: BBox, take: number, tagIds?: string[]): Promise { "use cache: remote"; - return lookupScope("localArtists", [backend, entityIds, area, take], async () => { + return lookupScope("localArtists", [backend, entityIds, area, take, ...withTags(tagIds)], async () => { cacheLife("days"); cacheTag("qloo", "qloo:artists"); const params = { "filter.type": "urn:entity:artist", "signal.location": wktPolygon(area), "signal.interests.entities": ids(entityIds), + "signal.interests.tags": tagIds?.length ? ids(tagIds) : undefined, "feature.explainability": entityIds.length > 0 ? true : undefined, take: Math.min(take, MAX_TAKE), }; @@ -553,16 +572,17 @@ export async function trending(backend: QlooBackend, entityIds: string[], startD } /** `urn:demographics`: the age and gender lean of fans of these artists. */ -export async function demographics(backend: QlooBackend, entityIds: string[]): Promise { - return countLookup("demographics", [backend, entityIds], () => demographicsCached(backend, entityIds)); +export async function demographics(backend: QlooBackend, entityIds: string[], tagIds: readonly string[] = []): Promise { + const tags = sceneTags(tagIds); + return countLookup("demographics", [backend, entityIds, ...withTags(tags)], () => (tags.length ? demographicsCached(backend, entityIds, tags) : demographicsCached(backend, entityIds))); } -async function demographicsCached(backend: QlooBackend, entityIds: string[]): Promise { +async function demographicsCached(backend: QlooBackend, entityIds: string[], tagIds?: string[]): Promise { "use cache: remote"; - return lookupScope("demographics", [backend, entityIds], async () => { + return lookupScope("demographics", [backend, entityIds, ...withTags(tagIds)], async () => { cacheLife("days"); cacheTag("qloo", "qloo:demographics"); - const params = { "filter.type": "urn:demographics", "signal.interests.entities": ids(entityIds) }; + const params = { "filter.type": "urn:demographics", "signal.interests.entities": ids(entityIds), "signal.interests.tags": tagIds?.length ? ids(tagIds) : undefined }; const body = await qlooGet(backend, "/v2/insights", params, demographicsResponseSchema); logIfEmpty("/v2/insights", params, body.results.demographics.length); const rows = body.results.demographics; diff --git a/src/server/adapters/qloo/index.ts b/src/server/adapters/qloo/index.ts index 4ec5a053..313e0a77 100644 --- a/src/server/adapters/qloo/index.ts +++ b/src/server/adapters/qloo/index.ts @@ -90,8 +90,8 @@ export function resolveSpotifyArtists(spotifyIds: string[], context: AdapterCont } /** taste_map: heatmap cells for the performer's artists over the area. */ -export function tasteMap(entityIds: string[], area: BBox, context: AdapterContext, boundary?: client.HeatmapBoundary): Promise> { - return run(context, [], (backend) => client.heatmap(backend, entityIds, area, boundary)); +export function tasteMap(entityIds: string[], area: BBox, context: AdapterContext, boundary?: client.HeatmapBoundary, tagIds: readonly string[] = []): Promise> { + return run(context, [], (backend) => client.heatmap(backend, entityIds, area, boundary, tagIds)); } /** busy_places: places near a point, open on the slot's weekday (or any day, with null). */ @@ -101,13 +101,14 @@ export function busyPlaces( radiusM: number, weekday: string | null, context: AdapterContext, + tagIds: readonly string[] = [], ): Promise> { - return run(context, [], (backend) => client.places(backend, entityIds, center, radiusM, weekday)); + return run(context, [], (backend) => client.places(backend, entityIds, center, radiusM, weekday, tagIds)); } /** local_artists: what this crowd likes, with explainability (per result and aggregate). */ -export function localArtists(entityIds: string[], area: BBox, take: number, context: AdapterContext): Promise> { - return run(context, { artists: [], impact: [] }, (backend) => client.localArtists(backend, entityIds, area, take)); +export function localArtists(entityIds: string[], area: BBox, take: number, context: AdapterContext, tagIds: readonly string[] = []): Promise> { + return run(context, { artists: [], impact: [] }, (backend) => client.localArtists(backend, entityIds, area, take, tagIds)); } /** taste_artists: artists whose fans overlap these artists' fans, anywhere, with explainability (the learn run's related_artists). */ @@ -141,8 +142,8 @@ export function rising(entityIds: string[], startDate: string, endDate: string, } /** crowd_profile: age and gender lean of fans of these artists. */ -export function crowdProfile(entityIds: string[], context: AdapterContext): Promise> { - return run(context, null, (backend) => client.demographics(backend, entityIds)); +export function crowdProfile(entityIds: string[], context: AdapterContext, tagIds: readonly string[] = []): Promise> { + return run(context, null, (backend) => client.demographics(backend, entityIds, tagIds)); } /** crowd_tags: the genres that characterise this crowd around a pitch (urn:tag). */ diff --git a/src/server/adapters/repertoire/client.ts b/src/server/adapters/repertoire/client.ts index cc9d9f8d..ccba4fab 100644 --- a/src/server/adapters/repertoire/client.ts +++ b/src/server/adapters/repertoire/client.ts @@ -6,7 +6,7 @@ import { env } from "@/server/env"; import { log } from "@/server/log"; import { AdapterError, errorKindOf, fetchJson } from "../core"; import { cacheBody, cachedCall, safeParams } from "../call-log"; -import { BudgetExhausted, CallBudget, CATALOGUE_CALL_BUDGET, CATALOGUE_QUEUE_WAIT_MS, DEFAULT_RETRY_AFTER_S, MAX_QUEUE_WAIT_MS, SHORT_RETRY_AFTER_S, countSpotifyCall, limitSpotifyUntil, paceSpotify, spotifyLimitedUntil } from "./pacer"; +import { BudgetExhausted, CallBudget, CATALOGUE_CALL_BUDGET, CATALOGUE_DEADLINE_MS, CATALOGUE_QUEUE_WAIT_MS, DEFAULT_RETRY_AFTER_S, MAX_QUEUE_WAIT_MS, SHORT_RETRY_AFTER_S, countSpotifyCall, limitSpotifyUntil, paceSpotify, spotifyLimitedUntil } from "./pacer"; import { albumTracksSchema, albumsBatchSchema, @@ -84,7 +84,14 @@ async function spotify(path: string, schema: S, budget?: Ca // The message ends like fetchJson's, so errorKindOf still reads "rate_limited" after a "use cache" boundary. if (until) throw new AdapterError("repertoire", "rate_limited", `Spotify is limited until ${until.toISOString()} (returned HTTP 429)`, 429); budget?.take(); - await paceSpotify(maxQueueWaitMs); + // A budget with a deadline never queues past it: a slot after the deadline ends the fetch (the songs found so far stay). + const queueWait = budget ? budget.capWait(maxQueueWaitMs) : maxQueueWaitMs; + try { + await paceSpotify(queueWait); + } catch (error) { + if (budget && queueWait < maxQueueWaitMs) throw new BudgetExhausted("deadline"); + throw error; + } await countSpotifyCall(); try { return await fetchJson(`${API}${path}`, { @@ -105,6 +112,7 @@ async function spotify(path: string, schema: S, budget?: Ca await limitSpotifyUntil(new Date(Date.now() + 60_000)); throw error; } + if (budget && wait * 1000 > budget.remainingMs()) throw new BudgetExhausted("deadline"); await new Promise((r) => setTimeout(r, wait * 1000)); } } @@ -288,7 +296,7 @@ async function fetchReleaseCountCached(artistId: string, market: string): Promis /** * An artist's own songs: albums and singles, deduplicated by title. Spotify calls: the artist's album list (10 per page), * then Get Several Albums (20 per call, each album with its first 50 tracks; `tracks.next` is followed only for longer - * albums): about 39 calls for 256 releases, where one call per release took 282. The fetch has a call budget; when it is + * albums): about 39 calls for 256 releases, where one call per release took 282. The fetch has a call budget and a 20 s deadline (CATALOGUE_DEADLINE_MS); when either is * used up, or Spotify rate-limits part-way, the songs found so far are returned and cached briefly. Fails (so nothing is * cached and the caller falls back to Last.fm) when nothing was found. */ @@ -300,7 +308,7 @@ async function fetchArtistCatalogueCached(artistId: string, market: string): Pro "use cache: remote"; return cacheBody("spotify.fetchArtistCatalogue", [artistId, market], async () => { cacheTag("repertoire"); - const budget = new CallBudget(CATALOGUE_CALL_BUDGET); + const budget = new CallBudget(CATALOGUE_CALL_BUDGET, Date.now() + CATALOGUE_DEADLINE_MS); let partial: "budget" | "limited" | "error" | null = null; const stop = (error: unknown) => { if (error instanceof BudgetExhausted) return (partial ??= "budget"); @@ -309,23 +317,6 @@ async function fetchArtistCatalogueCached(artistId: string, market: string): Pro }; const albums: (RawAlbum & { id: string })[] = []; - let path: string | null = `/artists/${encodeURIComponent(artistId)}/albums?${new URLSearchParams({ - include_groups: "album,single", - limit: "10", - market, - })}`; - try { - for (let page = 0; path && page < ALBUM_PAGES_SAFETY; page++) { - const body: z.infer = await spotify(path, albumsSchema, budget, CATALOGUE_QUEUE_WAIT_MS); - albums.push(...body.items); - path = body.next ? body.next.replace(API, "") : null; - } - } catch (error) { - // The first page failing is a failed catalogue; a later page failing leaves the releases found so far. - if (albums.length === 0) throw error; - stop(error); - } - const trackLists: CatalogueTrack[][] = []; /** An album's first page of tracks, from either route, with its own next link. */ type AlbumTracks = { album: RawAlbum; items: z.infer["items"]; next: string | null }; @@ -347,41 +338,75 @@ async function fetchArtistCatalogueCached(artistId: string, market: string): Pro if (partial) break; } }; - try { - for (let from = 0; from < albums.length; from += ALBUMS_PER_BATCH) { - const group = albums.slice(from, from + ALBUMS_PER_BATCH); - let found: AlbumTracks[] | null = null; - if (Date.now() >= batchRefusedUntil) { + /** One group of up to ALBUMS_PER_BATCH releases: Get Several Albums, or one by one once that is refused. */ + const readGroup = async (group: (RawAlbum & { id: string })[]) => { + let found: AlbumTracks[] | null = null; + if (Date.now() >= batchRefusedUntil) { + try { + const params = new URLSearchParams({ ids: group.map((a) => a.id).join(","), market }); + const body: z.infer = await spotify(`/albums?${params}`, albumsBatchSchema, budget, CATALOGUE_QUEUE_WAIT_MS); + found = body.albums.flatMap((album) => (album ? [{ album, items: album.tracks.items, next: album.tracks.next ?? null }] : [])); + } catch (error) { + if (!batchRefused(error)) throw error; + // Spotify refused the batch call itself (an endpoint closed to this app, or a shape we don't read): say why once, and + // read the albums one by one from here on (within the call budget) instead of losing the whole catalogue. + batchRefusedUntil = Date.now() + BATCH_RETRY_AFTER_MS; + log.warn("spotify batch albums refused; reading albums one by one", { kind: (error as AdapterError).kind, status: (error as AdapterError).status ?? null }); + } + } + if (found === null) { + found = []; + for (const album of group) { try { - const params = new URLSearchParams({ ids: group.map((a) => a.id).join(","), market }); - const body: z.infer = await spotify(`/albums?${params}`, albumsBatchSchema, budget, CATALOGUE_QUEUE_WAIT_MS); - found = body.albums.flatMap((album) => (album ? [{ album, items: album.tracks.items, next: album.tracks.next ?? null }] : [])); + const body: z.infer = await spotify(`/albums/${encodeURIComponent(album.id)}/tracks?${new URLSearchParams({ limit: "50", market })}`, albumTracksSchema, budget, CATALOGUE_QUEUE_WAIT_MS); + found.push({ album, items: body.items, next: body.next ?? null }); } catch (error) { - if (!batchRefused(error)) throw error; - // Spotify refused the batch call itself (an endpoint closed to this app, or a shape we don't read): say why once, and - // read the albums one by one from here on (within the call budget) instead of losing the whole catalogue. - batchRefusedUntil = Date.now() + BATCH_RETRY_AFTER_MS; - log.warn("spotify batch albums refused; reading albums one by one", { kind: (error as AdapterError).kind, status: (error as AdapterError).status ?? null }); + stop(error); + break; } } - if (found === null) { - found = []; - for (const album of group) { - try { - const body: z.infer = await spotify(`/albums/${encodeURIComponent(album.id)}/tracks?${new URLSearchParams({ limit: "50", market })}`, albumTracksSchema, budget, CATALOGUE_QUEUE_WAIT_MS); - found.push({ album, items: body.items, next: body.next ?? null }); - } catch (error) { - stop(error); - break; - } - } + } + await take(found); + }; + + let path: string | null = `/artists/${encodeURIComponent(artistId)}/albums?${new URLSearchParams({ + include_groups: "album,single", + limit: "10", + market, + })}`; + // The first album page failing is a failed catalogue (thrown after the loop, past the partial handling). + let failed: unknown = null; + let read = 0; + try { + // The tracks are read as the releases come in (newest first), a group of ALBUMS_PER_BATCH at a time, so a fetch cut + // short by its deadline or budget still has the newest songs rather than a full release list and no tracks. + for (let page = 0; path && page < ALBUM_PAGES_SAFETY && !partial; page++) { + let body: z.infer; + try { + body = await spotify(path, albumsSchema, budget, CATALOGUE_QUEUE_WAIT_MS); + } catch (error) { + // A later page failing leaves the releases found so far. + if (albums.length === 0) failed = error; + else stop(error); + break; } - await take(found); - if (partial) break; + albums.push(...body.items); + path = body.next ? body.next.replace(API, "") : null; + while (!partial && albums.length - read >= ALBUMS_PER_BATCH) { + const group = albums.slice(read, read + ALBUMS_PER_BATCH); + read += group.length; + await readGroup(group); + } + } + while (!failed && !partial && read < albums.length) { + const group = albums.slice(read, read + ALBUMS_PER_BATCH); + read += group.length; + await readGroup(group); } } catch (error) { stop(error); } + if (failed) throw failed; const seen = new Set(); const result = trackLists.flat().filter((track) => { diff --git a/src/server/adapters/repertoire/pacer.ts b/src/server/adapters/repertoire/pacer.ts index 45ea028a..39eb37ec 100644 --- a/src/server/adapters/repertoire/pacer.ts +++ b/src/server/adapters/repertoire/pacer.ts @@ -32,6 +32,11 @@ export const SHORT_RETRY_AFTER_S = 30; export const DEFAULT_RETRY_AFTER_S = 2; /** Spotify calls one catalogue fetch may spend (album list pages + batches + track pages, retries included). */ export const CATALOGUE_CALL_BUDGET = 60; +/** + * Wall-clock time one catalogue fetch may take. At 20 s it stops and returns the songs found so far (a partial catalogue), + * so research step 2 stays well inside the function's 120 s (a big catalogue paced at 2 calls a second took minutes). + */ +export const CATALOGUE_DEADLINE_MS = 20_000; const ADAPTER = "spotify"; /** How often an instance re-reads the shared row while it believes Spotify is fine. */ const RECHECK_MS = 30_000; @@ -235,22 +240,34 @@ export async function countSpotifyCall(): Promise { if (s.dayCalls === s.dailyMax) await limitSpotifyUntil(midnightUtc(now)); } -/** A per-fetch allowance of Spotify calls. */ +/** A per-fetch allowance of Spotify calls, and optionally of wall-clock time (a deadline in epoch ms). */ export class CallBudget { private used = 0; - constructor(readonly max: number) {} + constructor( + readonly max: number, + readonly deadline = Number.POSITIVE_INFINITY, + ) {} take() { if (this.used >= this.max) throw new BudgetExhausted(); + if (Date.now() >= this.deadline) throw new BudgetExhausted("deadline"); this.used++; } get spent() { return this.used; } + /** Time left before the deadline (Infinity without one), never negative. */ + remainingMs() { + return Math.max(0, this.deadline - Date.now()); + } + /** A wait (for a pacer slot or a short Retry-After) capped to the time left. */ + capWait(ms: number) { + return Math.min(ms, this.remainingMs()); + } } export class BudgetExhausted extends Error { - constructor() { - super("Spotify call budget for this fetch is used up"); + constructor(readonly why: "calls" | "deadline" = "calls") { + super(why === "deadline" ? "The time for this Spotify fetch is used up" : "Spotify call budget for this fetch is used up"); this.name = "BudgetExhausted"; } } diff --git a/src/server/agent/context.ts b/src/server/agent/context.ts index 4477d018..708174a7 100644 --- a/src/server/agent/context.ts +++ b/src/server/agent/context.ts @@ -237,6 +237,16 @@ export class PlanContext { return own; } + /** + * The act's genres as Qloo tag ids (`signal.interests.tags`), only when Qloo doesn't know the act (no own artist): beside the + * artists you sound like in the taste reads, so a genres-only answer still gives a taste signal. Empty otherwise, and then + * every call is the one it always was (same params, same cache key). + */ + signalTags(): string[] { + if (this.ownArtist()) return []; + return qloo.sceneTags((this.request.performer.genreTags ?? []).map((g) => g.id)); + } + /** The signal as Qloo gets it: own artist first (see ownWeight), then the artists you sound like. */ private withOwn(soft: string[]): string[] { const own = this.ownArtist(); @@ -288,8 +298,9 @@ export class PlanContext { return this.once("tasteCells", async () => { const soft = await this.soundsLikeIds(); const ids = this.withOwn(soft); - if (ids.length === 0) return []; - let cells = await this.track("taste", () => qloo.tasteMap(ids, this.city.bbox, this.adapters)); + const tags = this.signalTags(); + if (ids.length === 0 && tags.length === 0) return []; + let cells = await this.track("taste", () => qloo.tasteMap(ids, this.city.bbox, this.adapters, undefined, tags)); if (cells.length === 0 && !this.qlooFailed() && ids.length > soft.length && soft.length > 0) { // Qloo found nothing usable with the own artist: fall back to the artists you sound like only (the extra read costs one call). const fallback = await this.track("taste:soft", () => qloo.tasteMap(soft, this.city.bbox, this.adapters)); @@ -301,7 +312,7 @@ export class PlanContext { } if (cells.length > 0 || this.qlooFailed()) return cells; const useIds = this.ownDropped ? soft : ids; - const wider = await this.track("taste:wide", () => qloo.tasteMap(useIds, widen(this.city.bbox, WIDEN_FACTOR), this.adapters)); + const wider = await this.track("taste:wide", () => qloo.tasteMap(useIds, widen(this.city.bbox, WIDEN_FACTOR), this.adapters, undefined, tags)); this.notes.push( wider.length > 0 ? "Qloo had no taste data for the city centre alone, so the taste map uses a wider area." @@ -315,11 +326,12 @@ export class PlanContext { places() { return this.once("placesWithRetry", async () => { const ids = await this.entityIds(); - if (ids.length === 0) return []; - const found = await this.track("places", () => qloo.busyPlaces(ids, this.city.center, PLACES_RADIUS_M, this.weekday, this.adapters)); + const tags = this.signalTags(); + if (ids.length === 0 && tags.length === 0) return []; + const found = await this.track("places", () => qloo.busyPlaces(ids, this.city.center, PLACES_RADIUS_M, this.weekday, this.adapters, tags)); if (found.length > 0 || this.qlooFailed()) return found; const wider = await this.track("places:wide", () => - qloo.busyPlaces(ids, this.city.center, PLACES_RADIUS_M * WIDEN_FACTOR, this.weekday, this.adapters), + qloo.busyPlaces(ids, this.city.center, PLACES_RADIUS_M * WIDEN_FACTOR, this.weekday, this.adapters, tags), ); this.notes.push( wider.length > 0 @@ -333,8 +345,9 @@ export class PlanContext { /** What this city's crowd likes given the performer's artists, with Qloo's explainability. */ private async localCrowd(): Promise { const ids = await this.entityIds(); - if (ids.length === 0) return { artists: [], impact: [] }; - return this.track("localArtists", () => qloo.localArtists(ids, this.city.bbox, LOCAL_ARTISTS_TAKE, this.adapters)); + const tags = this.signalTags(); + if (ids.length === 0 && tags.length === 0) return { artists: [], impact: [] }; + return this.track("localArtists", () => qloo.localArtists(ids, this.city.bbox, LOCAL_ARTISTS_TAKE, this.adapters, tags)); } async localArtists(): Promise { @@ -367,8 +380,9 @@ export class PlanContext { async crowdProfile() { const ids = await this.entityIds(); - if (ids.length === 0) return null; - return this.track("crowd", () => qloo.crowdProfile(ids, this.adapters)); + const tags = this.signalTags(); + if (ids.length === 0 && tags.length === 0) return null; + return this.track("crowd", () => qloo.crowdProfile(ids, this.adapters, tags)); } pitchData() { diff --git a/src/server/playlist/config.ts b/src/server/playlist/config.ts index f433eeff..cf02399f 100644 --- a/src/server/playlist/config.ts +++ b/src/server/playlist/config.ts @@ -83,8 +83,11 @@ export const LEARN_LIMITS = { timeoutMs: 75_000, /** `finish_suggestions` rejections before the run gives up on the model's words. */ maxRejections: 2, - /** A `running` row older than timeoutMs + this is marked failed by the next request for that Repertoire. */ - staleGraceMs: 30_000, + /** + * A `running` row older than timeoutMs + this (150 s) is marked failed by the next request for that Repertoire. It must be + * at least the routes' maxDuration (120 s): at 105 s a run still going on its instance was swept as failed. + */ + staleGraceMs: 75_000, } as const; /** Per-run call caps for the model's tools (a further call is a tool error the model sees). */ @@ -105,6 +108,12 @@ export const SUGGEST_RELATED_TAKE = 20; // --- Repertoire research (hackathons/qloo-2026/repertoire-research-spec.md) --- +/** + * Time limits of one research run, inside the routes' maxDuration of 120 s (research and research//input). A step + * that takes longer than `stepMs` fails ("took too long") and ends the run; `runMs` after the start a watchdog ends the run + * as failed (end code "timeout") so the stream always gets its `end` before the function is stopped. + */ +export const RESEARCH_TIMEOUTS = { stepMs: 45_000, runMs: 110_000 } as const; /** Sound-alikes (step 4) saved as your "Artists you sound like" when that list is empty. */ export const RESEARCH_SOUNDS_LIKE_SAVED = 6; /** Sound-alikes that join your own artist in step 6's signal. */ diff --git a/src/server/playlist/own-artist.ts b/src/server/playlist/own-artist.ts index 0c864f07..55941c9a 100644 --- a/src/server/playlist/own-artist.ts +++ b/src/server/playlist/own-artist.ts @@ -104,19 +104,27 @@ export async function findOwnOnQloo(spotifyId: string, name: string, context: Ad return { entity: null, qloo: "unknown", by: null, candidates: decision.kind === "ask" ? decision.candidates.map(candidateView) : [], weak: false, searches }; } +/** Saved sound-alikes (with a Qloo id) that stand in for an act Qloo doesn't know, at most. */ +export const SOUNDS_LIKE_SIGNAL = 3; + /** * Artists whose fans overlap this performer's: Qloo insights (`filter.type=urn:entity:artist`) signalled with the - * performer's own entity. Cached for days by the Qloo client; the own artist is left out. + * performer's own entity. When Qloo doesn't know the act, the saved "Artists you sound like" Qloo knows are the signal + * instead (unknown_artist only when there are none). Cached for days by the Qloo client; the own artist and the ones + * already saved are left out. */ export async function soundsLikeFor(playlist: PlaylistRecord, context: AdapterContext): Promise { const none = (status: SoundsLikeStatus): SoundsLikeSuggestions => ({ status, artists: [] }); if (!playlist.spotifyArtistId) return none("no_artist"); - if (!playlist.qlooArtistId) return none(playlist.spotifyProfile ? "unknown_artist" : "not_loaded"); - const result = await tasteArtists([playlist.qlooArtistId], SOUNDS_LIKE_TAKE + 2, context); + const alikes = playlist.qlooArtistId ? [] : playlist.soundsLike.flatMap((a) => (a.qlooId ? [a.qlooId] : [])).slice(0, SOUNDS_LIKE_SIGNAL); + if (!playlist.qlooArtistId && alikes.length === 0) return none(playlist.spotifyProfile ? "unknown_artist" : "not_loaded"); + const signal = playlist.qlooArtistId ? [playlist.qlooArtistId] : alikes; + const result = await tasteArtists(signal, SOUNDS_LIKE_TAKE + 2 + (playlist.qlooArtistId ? 0 : playlist.soundsLike.length), context); if (result.mode === "fallback") return none("unavailable"); const own = playlist.spotifyProfile ? nameKey(playlist.spotifyProfile.name) : null; + const saved = playlist.qlooArtistId ? new Set() : new Set(playlist.soundsLike.map((a) => nameKey(a.name))); const artists = result.data.artists - .filter((a) => a.id !== playlist.qlooArtistId && nameKey(a.name) !== own) + .filter((a) => !signal.includes(a.id) && nameKey(a.name) !== own && !saved.has(nameKey(a.name))) .slice(0, SOUNDS_LIKE_TAKE) .map((a) => ({ name: a.name, diff --git a/src/server/playlist/store.ts b/src/server/playlist/store.ts index 800a4f5a..ce5eb38f 100644 --- a/src/server/playlist/store.ts +++ b/src/server/playlist/store.ts @@ -48,6 +48,11 @@ export interface PlaylistStore { songs(playlistId: string): Promise; /** Adds a "can sing" song. Refuses a repeat (same song key). */ addSong(playlistId: string, song: AddSongInput): Promise; + /** + * Adds many "can sing" songs at once (research step 2): one transaction, one multi-row INSERT … ON CONFLICT. A result per + * input, in order, as addSong would give it; a song repeated within the batch is a duplicate of its first copy. + */ + addSongs(playlistId: string, songs: readonly AddSongInput[]): Promise; setSongState(playlistId: string, songId: string, state: SongState): Promise; removeSong(playlistId: string, songId: string): Promise; /** Act name, sounds like, usual pitches. A stale `revision` is refused. */ @@ -231,6 +236,12 @@ export class MemoryPlaylistStore implements PlaylistStore { return { ok: true, song: structuredClone(song) }; } + async addSongs(playlistId: string, songs: readonly AddSongInput[]): Promise { + const out: AddSongResult[] = []; + for (const song of songs) out.push(await this.addSong(playlistId, song)); + return out; + } + /** A song typed into Songs you can sing that matches a pending suggestion marks it learned, so the two lists agree (docs §1.2). */ private markSuggestionLearned(playlistId: string, songKey: string) { const pending = this.suggestionRows.get(playlistId)?.find((x) => x.songKey === songKey && x.status === "pending"); @@ -372,6 +383,8 @@ type SongRow = { }; const PLAYLIST_COLUMNS = "id, act_name, sounds_like, spotify_artist_id, spotify_profile, qloo_artist_id, research, usual_pitches, recent_busks, city, revision, created_at, last_seen_at"; +/** Rows per multi-row INSERT in addSongs. */ +const ADD_SONGS_CHUNK = 500; const SONG_COLUMNS = "id::text AS id, song_key, state, source, title, artist, own, spotify_track_id, year, duration_ms, image, artist_image, added_at"; const toRecord = (r: PlaylistRow): PlaylistRecord => ({ @@ -510,6 +523,94 @@ export class PostgresPlaylistStore implements PlaylistStore { return { ok: false, reason: existing.length ? "duplicate" : "missing" }; } + async addSongs(playlistId: string, songs: readonly AddSongInput[]): Promise { + // A batch of up to ADD_SONGS_CHUNK rows per transaction keeps each request small; a big catalogue is a few round trips. + const out: AddSongResult[] = []; + const seen = new Set(); + for (let from = 0; from < songs.length; from += ADD_SONGS_CHUNK) { + const chunk = songs.slice(from, from + ADD_SONGS_CHUNK); + // A key already written by an earlier chunk is a duplicate here too (its first copy wins). + const fresh = chunk.map((s) => !seen.has(playlistSongKey(s.artist, s.title))); + const results = await this.addSongsChunk(playlistId, chunk.filter((_, i) => fresh[i])); + let k = 0; + for (const [i, s] of chunk.entries()) { + seen.add(playlistSongKey(s.artist, s.title)); + out.push(fresh[i] ? results[k++] : { ok: false, reason: "duplicate" }); + } + if (results.some((r) => !r.ok && r.reason === "missing")) { + for (let i = out.length; i < songs.length; i++) out.push({ ok: false, reason: "missing" }); + break; + } + } + return out; + } + + private async addSongsChunk(playlistId: string, songs: readonly AddSongInput[]): Promise { + if (songs.length === 0) return []; + const keys = songs.map((s) => playlistSongKey(s.artist, s.title)); + // The first copy of a key is written; a later copy in the same batch is a duplicate of it. + const firstAt = new Map(); + keys.forEach((k, i) => firstAt.has(k) || firstAt.set(k, i)); + const batch = [...firstAt.values()].map((i) => ({ key: keys[i], s: songs[i] })); + const params = [ + playlistId, + batch.map((b) => b.key), + batch.map((b) => b.s.source), + batch.map((b) => b.s.title), + batch.map((b) => b.s.artist), + batch.map((b) => b.s.own), + batch.map((b) => b.s.spotifyTrackId ?? null), + batch.map((b) => b.s.year ?? null), + batch.map((b) => b.s.durationMs ?? null), + batch.map((b) => b.s.image ?? null), + batch.map((b) => b.s.artistImage ?? null), + ]; + const rows = "unnest($2::text[], $3::text[], $4::text[], $5::text[], $6::boolean[], $7::text[], $8::smallint[], $9::integer[], $10::text[], $11::text[]) WITH ORDINALITY AS v(song_key, source, title, artist, own, spotify_track_id, year, duration_ms, image, artist_image, ord)"; + // One transaction (one HTTP round trip): lock the Repertoire, fill in the repeats as addSong does (before the INSERT, so + // only rows already there), then insert the rest, mark matching pending suggestions learned and bump the revision once. + const [locked, , inserted] = (await this.sql.transaction((txn) => [ + txn.query("SELECT id FROM playlists WHERE id = $1::uuid FOR UPDATE", [playlistId]), + txn.query( + `UPDATE playlist_songs ps SET + image = COALESCE(ps.image, v.image), artist_image = COALESCE(ps.artist_image, v.artist_image), + title = CASE WHEN v.source = 'spotify' AND ps.source IN ('lastfm', 'typed') THEN v.title ELSE ps.title END, + spotify_track_id = CASE WHEN v.source = 'spotify' AND ps.source IN ('lastfm', 'typed') THEN COALESCE(v.spotify_track_id, ps.spotify_track_id) ELSE ps.spotify_track_id END, + year = COALESCE(ps.year, v.year), duration_ms = COALESCE(ps.duration_ms, v.duration_ms), + source = CASE WHEN v.source = 'spotify' AND ps.source IN ('lastfm', 'typed') THEN 'spotify' ELSE ps.source END + FROM ${rows} + WHERE ps.playlist_id = $1::uuid AND ps.song_key = v.song_key + RETURNING ps.song_key`, + params, + ), + txn.query( + `WITH ins AS ( + INSERT INTO playlist_songs (playlist_id, song_key, state, source, title, artist, own, spotify_track_id, year, duration_ms, image, artist_image) + SELECT $1::uuid, v.song_key, 'can_sing', v.source, v.title, v.artist, v.own, v.spotify_track_id, v.year, v.duration_ms, v.image, v.artist_image + FROM ${rows} + WHERE EXISTS (SELECT 1 FROM playlists WHERE id = $1::uuid) + ORDER BY v.ord + ON CONFLICT (playlist_id, song_key) DO NOTHING + RETURNING ${SONG_COLUMNS} + ), + learned AS ( + UPDATE learn_suggestions SET status = 'learned', learned_at = now() + WHERE playlist_id = $1::uuid AND status = 'pending' AND song_key IN (SELECT song_key FROM ins) + ), + bumped AS ( + UPDATE playlists SET revision = revision + 1, updated_at = now() WHERE id = $1::uuid AND EXISTS (SELECT 1 FROM ins) + ) + SELECT * FROM ins`, + params, + ), + ])) as [unknown[], unknown[], SongRow[]]; + if (locked.length === 0) return songs.map(() => ({ ok: false, reason: "missing" })); + const byKey = new Map(inserted.map((r) => [r.song_key, r])); + return keys.map((k, i): AddSongResult => { + const row = firstAt.get(k) === i ? byKey.get(k) : undefined; + return row ? { ok: true, song: toSong(row) } : { ok: false, reason: "duplicate" }; + }); + } + async setSongState(playlistId: string, songId: string, state: SongState) { if (!isSongId(songId)) return null; const rows = (await this.sql.query( diff --git a/src/server/request.ts b/src/server/request.ts index cdf5d89b..ed66d30f 100644 --- a/src/server/request.ts +++ b/src/server/request.ts @@ -73,6 +73,18 @@ export function researchStepDelayMs(faults: string | null | undefined): number { return faults.split(",").some((s) => s.trim() === RESEARCH_SLOW_FAULT) ? RESEARCH_SLOW_STEP_MS : 0; } +/** + * Test fault `research-cut`: a Repertoire research run stops right after step 2 ("Your songs") starts, as if the function were + * stopped mid-step: the stream ends with no `end` and the run row stays `running` (until the stale sweep). Lets a test look at + * the page's stopped state (the step cut, the Stopped tag, Try again). Same gate as the others. + */ +export const RESEARCH_CUT_FAULT = "research-cut"; + +export function researchCutRequested(faults: string | null | undefined): boolean { + if (!testSwitches().allowFaults || !faults) return false; + return faults.split(",").some((s) => s.trim() === RESEARCH_CUT_FAULT); +} + /** * The test faults a page passes on to its API calls: the provider outages plus `fresh-limit`. * Empty unless fault injection is allowed (never in production). diff --git a/src/server/research/run.ts b/src/server/research/run.ts index 265584fb..c077ca2f 100644 --- a/src/server/research/run.ts +++ b/src/server/research/run.ts @@ -60,6 +60,7 @@ import { RESEARCH_LEARN_ARTISTS, RESEARCH_LEARN_PER_ARTIST, RESEARCH_SIGNAL_SOUND_ALIKES, + RESEARCH_TIMEOUTS, RESEARCH_SOUNDS_LIKE_SAVED, } from "@/server/playlist/config"; import { fansMapFor } from "./fans-map"; @@ -98,6 +99,10 @@ export type ResearchRunArgs = { budgetFault?: boolean; /** Test fault `research-slow`: wait this long at the start of every step. */ stepDelayMs?: number; + /** Test fault `research-cut`: the run stops right after this step starts, with no `end` and the row left running (a stopped function). */ + cutAt?: ResearchStepKey; + /** Tests shorten the step timeout and the run watchdog (RESEARCH_TIMEOUTS). */ + timeouts?: { stepMs: number; runMs: number }; /** The performer's answer to "Is one of these you?": a candidate the last run offered, or none of them. */ qlooChoice?: { id: string } | { none: true }; /** A genre the performer picked ("Your genre"), used as the first Qloo genre tag when Qloo and Last.fm give the Hive nothing else. */ @@ -127,6 +132,15 @@ class QlooDown extends Error { super(message); } } +/** Test fault `research-cut`: the run is dropped as a stopped function would drop it. */ +class RunCut extends Error {} +/** A step ran past RESEARCH_TIMEOUTS.stepMs: the step fails and the run ends (end code "timeout"). */ +class StepTimeout extends Error { + constructor(label: string) { + super(`${label} took too long`); + } +} + /** Why Qloo is down for this run: today's budget is spent (the page then offers the examples) or Qloo is not answering. */ const qlooEndCode = (forceBudget = false) => (forceBudget || qlooBudgetSpent() ? "qloo_budget_spent" : "qloo_unavailable") as "qloo_budget_spent" | "qloo_unavailable"; @@ -185,7 +199,13 @@ export async function executeResearchRun(args: ResearchRunArgs): Promise { // A resumed run sends its kept steps again first, so the panel is the one the performer left, then carries on. const kept = resume && waitingBefore ? keptSteps(resume.stored, waitingBefore.stepId) : []; const stored: StoredLearnStep[] = [...kept]; + const timeouts = args.timeouts ?? RESEARCH_TIMEOUTS; + /** Set once the run's `end` is on its way (by the steps, an error or the watchdog): nothing is pushed after it. */ + let ended = false; + /** The step going on now, for the watchdog to close. */ + let openStep: { id: string; tool: string; key: ResearchStepKey; label: string } | null = null; const push = (event: LearnStreamEvent) => { + if (ended) return; run.push(event); if (event.type === "status" || event.type === "note" || event.type === "tool-start" || event.type === "tool-end" || event.type === "totals") stored.push(event); }; @@ -219,6 +239,9 @@ export async function executeResearchRun(args: ResearchRunArgs): Promise { const usageTotal = (): QlooUsage => ({ calls: usageNow.calls + (base?.qlooCalls ?? 0), cacheHits: usageNow.cacheHits + (base?.qlooCacheHits ?? 0) }); const finish = async (end: { status: "done" | "failed" | "fallback"; endCode: RunRecord["endCode"]; saved: Awaited>; usage: RunRecord["usage"]; waiting?: ResearchWaiting }) => { + // One end per run: the steps, an error and the watchdog may race to it. + if (ended) return; + ended = true; // Steps never started this run are skipped (a full run that stopped early; "more" leaves the others as they were). A paused // run's are not started yet, so they stay out of the snapshot. if (!end.waiting) for (const k of mode === "more" ? MORE_STEPS : RESEARCH_STEPS) if (!steps[k]) (steps as Record)[k] = { status: "skipped", output: null }; @@ -249,19 +272,31 @@ export async function executeResearchRun(args: ResearchRunArgs): Promise { /** One step: tool-start, the work, tool-end, then the running totals. A thrown error is a failed step (and the run goes on). */ async function step(key: K, input: unknown, work: (via: (source: MentionSource) => void) => Promise): Promise { + // The watchdog already ended the run: no further step starts. + if (ended) throw new StepTimeout(RESEARCH_LABELS[key]); const id = `rs-${++n}`; lastStepId = id; const tool = RESEARCH_TOOLS[key]; + openStep = { id, tool, key, label: RESEARCH_LABELS[key] }; push({ type: "tool-start", id, tool, label: RESEARCH_LABELS[key], input }); + if (args.cutAt === key) { + ended = true; + throw new RunCut(); + } // `via`: a step with a backup source says which one it is on now, so its running mark shows that source. const via = (source: MentionSource) => push({ type: "tool-source", id, source }); // Every external call the step makes (provider, endpoint, params, status, count, ms, cached), for its RAW panel. const calls: CallRecord[] = []; const withCalls = () => (calls.length ? { calls } : {}); + let timer: ReturnType | undefined; try { if (args.stepDelayMs) await new Promise((resolve) => setTimeout(resolve, args.stepDelayMs)); if (args.budgetFault) throw new QlooDown("Today's Qloo budget is spent.", true); - const output = await recordCalls(calls, () => work(via)); + // The work can't be cancelled; past the step's time it is left behind and the run ends without it. + const late = new Promise((_, reject) => { + timer = setTimeout(() => reject(new StepTimeout(RESEARCH_LABELS[key])), timeouts.stepMs); + }); + const output = await Promise.race([recordCalls(calls, () => work(via)), late]); const waits = Boolean((output as { waiting?: unknown }).waiting); (steps as Record)[key] = { status: waits ? "waiting" : "done", output }; push({ type: "tool-end", id, tool, ok: true, summary: output.summary, output, ...withCalls() }); @@ -270,9 +305,11 @@ export async function executeResearchRun(args: ResearchRunArgs): Promise { const message = error instanceof Error ? error.message : String(error); (steps as Record)[key] = { status: "failed", output: null }; push({ type: "tool-end", id, tool, ok: false, summary: message.slice(0, 300), output: { error: message.slice(0, 600) }, ...withCalls() }); - if (error instanceof QlooDown) throw error; + if (error instanceof QlooDown || error instanceof StepTimeout) throw error; return null; } finally { + clearTimeout(timer); + if (openStep?.id === id) openStep = null; push({ type: "totals", totals: researchTotals(steps, usageTotal()) }); } } @@ -315,6 +352,23 @@ export async function executeResearchRun(args: ResearchRunArgs): Promise { return playlist.soundsLike.filter((a) => a.qlooId).slice(0, RESEARCH_SIGNAL_SOUND_ALIKES) as (SoundsLike & { qlooId: string })[]; } + // The run watchdog: before the function's maxDuration, a run still going is ended as failed (end code "timeout"), its open + // step closed as too long, so the stream gets its `end` and the row never stays `running`. + const watchdog = setTimeout(() => { + if (ended) return; + log.warn("research run ran out of time", { runId: record.id, step: openStep?.key ?? null }); + if (openStep) { + const message = new StepTimeout(openStep.label).message; + (steps as Record)[openStep.key] = { status: "failed", output: null }; + push({ type: "tool-end", id: openStep.id, tool: openStep.tool, ok: false, summary: message, output: { error: message } }); + openStep = null; + } + void finish({ status: "failed", endCode: "timeout", saved: [], usage: null }) + .catch(() => undefined) + .finally(() => { + if (!run.done) run.finish(); + }); + }, timeouts.runMs); try { const today = serverToday(CITIES.melbourne.timeZone); await withQlooUsage(async (usage) => { @@ -392,8 +446,11 @@ export async function executeResearchRun(args: ResearchRunArgs): Promise { tracks = top.map((t) => ({ id: null, title: t.title, artist: profile.name, own: true, year: null, durationMs: null, album: null, image: null, href: null })); } const out: SongsOutput = { summary: "", source, songs: [], added: 0, already: 0, ...(spotifyDown ? { spotifyDown: limited ? ("limited" as const) : ("down" as const) } : {}) }; - for (const t of tracks) { - const res = await playlists.addSong(playlist.id, { + // All of the act's own songs (U16), written in one batch: one transaction per song took minutes for a big catalogue + // and the function was stopped mid-step. + const results = await playlists.addSongs( + playlist.id, + tracks.map((t) => ({ title: t.title.slice(0, 120), artist: t.artist.slice(0, 80), own: true, @@ -404,7 +461,10 @@ export async function executeResearchRun(args: ResearchRunArgs): Promise { // Covers persist (migration 010): the track or album art, and the act's own photo. ...(t.image && /^https:\/\//.test(t.image) && t.image.length <= 500 ? { image: t.image } : {}), ...(t.own && profile.image && /^https:\/\//.test(profile.image) && profile.image.length <= 500 ? { artistImage: profile.image } : {}), - }); + })), + ); + for (const [i, t] of tracks.entries()) { + const res = results[i]; if (res.ok) out.added++; else if (res.reason === "duplicate") out.already++; else break; @@ -847,9 +907,13 @@ export async function executeResearchRun(args: ResearchRunArgs): Promise { }); } catch (error) { const down = error instanceof QlooDown; - if (!down) log.error("research run failed", { error: error instanceof Error ? error.message : "unknown" }); - await finish({ status: "failed", endCode: down ? qlooEndCode((error as QlooDown).budget) : null, saved: [], usage: null }).catch(() => undefined); + const late = error instanceof StepTimeout; + // research-cut: no end and no finishRun, so the row stays running like a function stopped mid-step. + if (error instanceof RunCut) return; + if (!down && !late) log.error("research run failed", { error: error instanceof Error ? error.message : "unknown" }); + await finish({ status: "failed", endCode: down ? qlooEndCode((error as QlooDown).budget) : late ? "timeout" : null, saved: [], usage: null }).catch(() => undefined); } finally { - run.finish(); + clearTimeout(watchdog); + if (!run.done) run.finish(); } } diff --git a/tests/e2e/layout-helpers.ts b/tests/e2e/layout-helpers.ts index cda1b17a..eb598de8 100644 --- a/tests/e2e/layout-helpers.ts +++ b/tests/e2e/layout-helpers.ts @@ -65,7 +65,10 @@ async function clippedPlaceholders(page: Page): Promise { /** Visible, in view and on top: the element at its centre is the control itself or inside it. */ export async function expectOnTop(control: Locator, what: string) { - await control.scrollIntoViewIfNeeded(); + // The page can re-render a node after hydration: scroll through the locator again until it sticks, never on a stale handle. + await expect(async () => { + await control.scrollIntoViewIfNeeded({ timeout: 2_000 }); + }, `${what}: scroll into view`).toPass({ timeout: 10_000 }); await expect(control, what).toBeVisible(); await expect .poll( diff --git a/tests/e2e/repertoire-research.spec.ts b/tests/e2e/repertoire-research.spec.ts index 2a3fcfb1..d9b5e026 100644 --- a/tests/e2e/repertoire-research.spec.ts +++ b/tests/e2e/repertoire-research.spec.ts @@ -45,6 +45,19 @@ test.describe("Repertoire research (offline)", () => { await expect(page.getByTestId("research-notice")).toContainText("isn't a Spotify artist link"); }); + test("a run cut off mid-step (the stream ends with no end) shows as stopped: the step cut, the Stopped tag and Try again", async ({ page }) => { + // `faults=research-cut`: the server drops the run right after step 2 ("Your songs") starts, as a stopped function would. + await research(page, LINK, "research-cut"); + const stream = page.getByTestId("agent-stream").first(); + await expect(stream.getByTestId("agent-tag")).toHaveText("Stopped", { timeout: 20_000 }); + await expect(page.getByTestId("research-page")).not.toHaveAttribute("data-phase", "running"); + await expect(stream.getByText("Stopped before it finished.")).toHaveCount(1); + await expect(stream.getByText("The research stopped before finishing")).toBeVisible(); + await expect(stream.getByRole("button", { name: "Try again" })).toBeVisible(); + // Never "still going": no live heading, no running spinner left behind. + await expect(stream.getByRole("heading", { name: "Researching your act" })).toHaveCount(0); + }); + test("the run streams with the research on top, then the results land in order and the research folds below", async ({ page }) => { await research(page, LINK, "research-slow"); const stream = page.getByTestId("agent-stream").first(); diff --git a/tests/unit/adapters/repertoire-pacer.test.ts b/tests/unit/adapters/repertoire-pacer.test.ts index 660c9a3d..e000a64c 100644 --- a/tests/unit/adapters/repertoire-pacer.test.ts +++ b/tests/unit/adapters/repertoire-pacer.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { countSpotifyCall, DAILY_MAX_CALLS, MAX_QUEUE_WAIT_MS, MemoryBackoffStore, MemoryUsageStore, paceSpotify, PACE_MS, WINDOW_MAX_CALLS, WINDOW_MS, resetPacerForTests, spotifyLimitedUntil } from "@/server/adapters/repertoire/pacer"; +import { BudgetExhausted, CallBudget, CATALOGUE_DEADLINE_MS, countSpotifyCall, DAILY_MAX_CALLS, MAX_QUEUE_WAIT_MS, MemoryBackoffStore, MemoryUsageStore, paceSpotify, PACE_MS, WINDOW_MAX_CALLS, WINDOW_MS, resetPacerForTests, spotifyLimitedUntil } from "@/server/adapters/repertoire/pacer"; import { json, LIVE, mockFetch, route } from "../../helpers"; const LINK = "https://open.spotify.com/artist/0SNWoGaDlrCompmg9rXeNq"; @@ -83,13 +83,54 @@ describe("Spotify pacer, budget and breaker (U34)", () => { expect(times[4] - times[0]).toBe(2000); }); + it("a budget with a deadline refuses calls past it and caps waits to the time left", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-10-23T10:00:00Z")); + const budget = new CallBudget(60, Date.now() + 20_000); + budget.take(); + expect(budget.capWait(30_000)).toBe(20_000); + vi.advanceTimersByTime(15_000); + expect(budget.capWait(30_000)).toBe(5_000); + expect(budget.capWait(1_000)).toBe(1_000); + vi.advanceTimersByTime(5_000); + expect(budget.remainingMs()).toBe(0); + expect(() => budget.take()).toThrow(BudgetExhausted); + expect(() => budget.take()).toThrow(/time for this Spotify fetch/); + expect(budget.spent).toBe(1); + // Without a deadline only the call count limits it. + expect(new CallBudget(1).capWait(30_000)).toBe(30_000); + }); + + it("stops at the catalogue's 20 s deadline and returns the newest songs read so far (U34 pacing on a big catalogue)", async () => { + expect(CATALOGUE_DEADLINE_MS).toBe(20_000); + vi.useFakeTimers(); + // Real pace (500 ms) and window (20 calls per 30 s): the window fills in about 10 s, the next slot is past the deadline. + resetPacerForTests({ dailyMax: Infinity }); + const { getArtistCatalogue } = await fresh(); + const { calls } = mockFetch([tokenRoute, artistAlbums(450), batch()]); + const started = Date.now(); + let doneAt = 0; + const pending = getArtistCatalogue(LINK, "AU", LIVE).finally(() => (doneAt = Date.now())); + await vi.advanceTimersByTimeAsync(60_000); + const result = await pending; + expect(result.mode).toBe("live"); + // 20 calls fit: 14 release pages and 6 batches (7 would need the 21st call), so 120 songs, newest releases first. + expect(spotifyCalls(calls)).toHaveLength(20); + expect(albumCalls(calls)).toHaveLength(6); + expect(result.data).toHaveLength(120); + expect(result.data[0].title).toBe("Song al0"); + // It gave up as soon as the next free slot was past the deadline, rather than queueing 30 s for it. + expect(doneAt - started).toBeLessThan(CATALOGUE_DEADLINE_MS); + }); + it("stops at the per-fetch call budget (60) and returns what it has", async () => { const { getArtistCatalogue } = await fresh(); const { calls } = mockFetch([tokenRoute, artistAlbums(450), batch()]); const result = await getArtistCatalogue(LINK, "AU", LIVE); expect(spotifyCalls(calls)).toHaveLength(60); - expect(albumCalls(calls)).toHaveLength(15); - expect(result.data).toHaveLength(300); + // Tracks are read as the release pages come in (two pages, then one batch of 20): 40 pages + 20 batches = 400 songs. + expect(albumCalls(calls)).toHaveLength(20); + expect(result.data).toHaveLength(400); }); it("a long Retry-After pauses all Spotify calls until it passes, and survives a restart", async () => { diff --git a/tests/unit/playlist/store.test.ts b/tests/unit/playlist/store.test.ts index cee9f31d..f450e62d 100644 --- a/tests/unit/playlist/store.test.ts +++ b/tests/unit/playlist/store.test.ts @@ -190,6 +190,45 @@ describe("PostgresPlaylistStore SQL shape", () => { expect(calls[2].text).toMatch(/revision = revision \+ 1/); }); + it("adds many songs in ONE transaction: lock, fill in repeats, one multi-row INSERT … ON CONFLICT with the learned mark and one bump", async () => { + const row2 = { ...songRow, id: "8", song_key: "artist|song 2", title: "Song 2" }; + const { store, calls } = fake([[{ id: "pid" }], [{ song_key: "artist|song 3" }], [songRow, row2]]); + const r = await store.addSongs("pid", [song(1), song(2), song(3), song(1)]); + expect(r).toEqual([ + { ok: true, song: expect.objectContaining({ id: "7", title: "Song 1" }) }, + { ok: true, song: expect.objectContaining({ id: "8", title: "Song 2" }) }, + { ok: false, reason: "duplicate" }, + // The same song twice in a batch: the second copy is a duplicate of the first. + { ok: false, reason: "duplicate" }, + ]); + expect(calls).toHaveLength(3); + expect(calls[0].text).toMatch(/FOR UPDATE$/); + expect(calls[1].text).toMatch(/^UPDATE playlist_songs ps SET/); + expect(calls[1].text).toMatch(/FROM unnest\(/); + expect(calls[2].text).toMatch(/INSERT INTO playlist_songs/); + expect(calls[2].text).toMatch(/ON CONFLICT \(playlist_id, song_key\) DO NOTHING/); + expect(calls[2].text).toMatch(/UPDATE learn_suggestions SET status = 'learned'/); + expect(calls[2].text).toMatch(/revision = revision \+ 1/); + // One row per distinct song, as arrays (the repeat is not sent). + expect(calls[2].params[1]).toEqual(["artist|song 1", "artist|song 2", "artist|song 3"]); + expect(calls[2].params).toHaveLength(11); + }); + + it("addSongs splits a big catalogue into 500-row statements and reports a missing Repertoire for every song", async () => { + const many = Array.from({ length: 1200 }, (_, i) => song(i)); + const big = fake([]); + const r = await big.store.addSongs("pid", many); + // The fake's empty lock answer reads as a missing Repertoire: the first chunk says so, and the rest are not sent. + expect(r).toHaveLength(1200); + expect(r.every((x) => !x.ok && x.reason === "missing")).toBe(true); + expect(big.calls).toHaveLength(3); + expect((big.calls[2].params[1] as string[]).length).toBe(500); + const ok = fake([[{ id: "pid" }], [], [], [{ id: "pid" }], [], [], [{ id: "pid" }], [], []]); + await ok.store.addSongs("pid", many); + expect(ok.calls.filter((c) => /INSERT INTO playlist_songs/.test(c.text)).map((c) => (c.params[1] as string[]).length)).toEqual([500, 500, 200]); + expect(await ok.store.addSongs("pid", [])).toEqual([]); + }); + it("tells a repeat from a missing Repertoire when the INSERT returns nothing", async () => { const dup = fake([[{ id: "pid" }], [], [{ one: 1 }]]); expect(await dup.store.addSong("pid", song(1))).toEqual({ ok: false, reason: "duplicate" }); @@ -197,6 +236,16 @@ describe("PostgresPlaylistStore SQL shape", () => { expect(await gone.store.addSong("pid", song(1))).toEqual({ ok: false, reason: "missing" }); }); + it("the memory store's addSongs gives addSong's result per song, in order", async () => { + const store = new MemoryPlaylistStore(); + const p = await store.create(hashToken(newToken())); + await store.addSong(p.id, song(2)); + const r = await store.addSongs(p.id, [song(1), song(2), song(3), song(1)]); + expect(r.map((x) => (x.ok ? x.song.title : x.reason))).toEqual(["Song 1", "duplicate", "Song 3", "duplicate"]); + expect((await store.songs(p.id)).map((x) => x.title)).toEqual(["Song 2", "Song 1", "Song 3"]); + expect(await store.addSongs("missing", [song(1)])).toEqual([{ ok: false, reason: "missing" }]); + }); + it("the memory store lets simultaneous saves past 60 songs all land", async () => { const store = new MemoryPlaylistStore(); const p = await store.create(hashToken(newToken())); diff --git a/tests/unit/playlist/unknown-act-sounds-like.test.ts b/tests/unit/playlist/unknown-act-sounds-like.test.ts new file mode 100644 index 00000000..e5569a00 --- /dev/null +++ b/tests/unit/playlist/unknown-act-sounds-like.test.ts @@ -0,0 +1,159 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { startingArtists, type FormPrefill } from "@/components/landing/performer-form"; +import { planRequestSchema } from "@/lib/plan-schema"; +import { actGenresOf, type ResearchSnapshot } from "@/lib/repertoire-research"; +import { CATALOG, genreTagId } from "@/server/qloo-mock/catalog"; +import { mockFetch, OFFLINE } from "../../helpers"; + +/** + * An act Qloo doesn't know, whose research saved "who you sound like" and genres (v1.1.1): the busk form starts with those + * sound-alikes, the suggestions come from them, and the plan's Qloo taste reads carry the genres as signal.interests.tags. + */ + +/** Every Qloo request the in-process mock answered, as URLs. */ +const qlooUrls: URL[] = []; +vi.mock("@/server/qloo-mock/fetch", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + qlooMockFetch: (input: string | URL | Request, init?: RequestInit) => { + qlooUrls.push(new URL(input instanceof Request ? input.url : String(input))); + return original.qlooMockFetch(input, init); + }, + }; +}); + +const [A, B] = [CATALOG[1], CATALOG[2]]; +const UNKNOWN_ID = "4gzpq5DPGxSnKTe4SA8HAU"; + +describe("the busk form's starting artists", () => { + const rep = { soundsLike: [{ name: "Mariah Carey", qlooId: "q-mariah", image: "https://i.example/m.jpg" }, { name: "Celia Cruz" }] }; + const prefill = (artists: string[]): FormPrefill => ({ + performer: { name: "Act", artists, spotifyArtists: [], songs: [] } as unknown as FormPrefill["performer"], + permit: "general", + anyTime: true, + }); + + it("starts with the Repertoire's sound-alikes", () => { + expect(startingArtists(undefined, rep).map((a) => a.name)).toEqual(["Mariah Carey", "Celia Cruz"]); + expect(startingArtists(undefined, rep)[0]).toMatchObject({ qlooId: "q-mariah", image: "https://i.example/m.jpg", spotifyId: null }); + }); + + it("planning again: the plan's artists first, then the Repertoire's it lacks, by name", () => { + const got = startingArtists(prefill(["Shakira", "mariah carey"]), rep); + expect(got.map((a) => a.name)).toEqual(["Shakira", "mariah carey", "Celia Cruz"]); + expect(got[1]).toMatchObject({ qlooId: "q-mariah", image: "https://i.example/m.jpg" }); + }); + + it("an empty plan list never wipes the Repertoire's", () => { + expect(startingArtists(prefill([]), rep).map((a) => a.name)).toEqual(["Mariah Carey", "Celia Cruz"]); + }); + + it("stops at the picker's maximum", () => { + const many = Array.from({ length: 10 }, (_, i) => `Artist ${i}`); + expect(startingArtists(prefill(many), rep)).toHaveLength(10); + }); + + it("no Repertoire, no plan: empty", () => { + expect(startingArtists(undefined, null)).toEqual([]); + }); +}); + +describe("actGenresOf: the genres the performer answered count", () => { + const snapshot = (qloo: object, genres?: { id: string; name: string }[]) => + ({ steps: { qloo: { output: qloo }, ...(genres ? { genres: { output: { tags: genres } } } : {}) } }) as unknown as ResearchSnapshot; + it("an unknown act's answered genre tags lead, then step 8's, once each", () => { + const latin = { id: "urn:tag:genre:music:latin", name: "Latin" }; + const pop = { id: "urn:tag:genre:music:pop", name: "Pop" }; + expect(actGenresOf(snapshot({ matched: false, genreTags: [latin, pop] }, [pop]))).toEqual([latin, pop]); + }); + it("a matched act keeps step 8's only", () => { + expect(actGenresOf(snapshot({ matched: true, genreTags: [{ id: "urn:tag:x", name: "X" }] }))).toEqual([]); + }); +}); + +describe("suggestions for an act Qloo doesn't know (offline)", () => { + beforeEach(() => vi.unstubAllEnvs()); + + async function load() { + vi.resetModules(); + vi.stubEnv("BUSKMAP_OFFLINE", "1"); + vi.stubEnv("VERCEL", ""); + const store = await import("@/server/playlist/store"); + store.resetPlaylistStoresForTests(); + return { root: await import("@/app/api/playlist/route"), profile: await import("@/app/api/playlist/profile/route"), soundsLike: await import("@/app/api/playlist/sounds-like/route") }; + } + const req = (path: string, method: string, body?: unknown, cookie?: string) => + new Request(`http://localhost:3100${path}`, { + method, + headers: { "content-type": "application/json", host: "localhost:3100", ...(cookie ? { cookie } : {}) }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + + it("are Qloo's related artists of the saved sound-alikes, never the saved ones themselves", async () => { + const { root, profile, soundsLike } = await load(); + const cookie = (await root.POST(req("/api/playlist", "POST", { spotifyArtistId: UNKNOWN_ID }))).headers.get("set-cookie")?.split(";")[0] ?? ""; + const loaded = (await (await profile.POST(req("/api/playlist/profile", "POST", undefined, cookie))).json()).playlist; + expect(loaded.qlooArtistId).toBeNull(); + await root.PATCH(req("/api/playlist", "PATCH", { revision: loaded.revision, soundsLike: [{ name: A.name, qlooId: A.id }, { name: B.name, qlooId: B.id }] }, cookie)); + const got = await (await soundsLike.GET(req("/api/playlist/sounds-like", "GET", undefined, cookie))).json(); + expect(got.status).toBe("ok"); + expect(got.artists.length).toBeGreaterThan(0); + expect(got.artists.length).toBeLessThanOrEqual(10); + const names = got.artists.map((a: { name: string }) => a.name); + expect(names).not.toContain(A.name); + expect(names).not.toContain(B.name); + }); + + it("sound-alikes Qloo doesn't know either: still unknown_artist", async () => { + const { root, profile, soundsLike } = await load(); + const cookie = (await root.POST(req("/api/playlist", "POST", { spotifyArtistId: UNKNOWN_ID }))).headers.get("set-cookie")?.split(";")[0] ?? ""; + const loaded = (await (await profile.POST(req("/api/playlist/profile", "POST", undefined, cookie))).json()).playlist; + await root.PATCH(req("/api/playlist", "PATCH", { revision: loaded.revision, soundsLike: [{ name: "Nobody Knows" }] }, cookie)); + expect(await (await soundsLike.GET(req("/api/playlist/sounds-like", "GET", undefined, cookie))).json()).toEqual({ status: "unknown_artist", artists: [] }); + }); +}); + +describe("the plan's Qloo taste reads carry the act's genres when Qloo doesn't know the act (offline)", () => { + const POP = { id: genreTagId("pop"), name: "Pop" }; + const SOUL = { id: genreTagId("soul"), name: "Soul" }; + const request = (extra: object) => + planRequestSchema.parse({ + city: "melbourne", + slot: { date: "2026-10-30", startHour: 17, endHour: 20 }, + permit: "general", + performer: { name: "Unknown act", artists: ["Zzq Nobody Knows This Name"], spotifyArtists: [], songs: [], ...extra }, + }); + + async function taste(req: ReturnType) { + qlooUrls.length = 0; + mockFetch([]); + const { PlanContext } = await import("@/server/agent/context"); + const ctx = new PlanContext(req, OFFLINE, "2026-10-23"); + const [cells] = await Promise.all([ctx.tasteCells(), ctx.places(), ctx.crowdProfile(), ctx.localArtists()]); + const reads = (type: string) => qlooUrls.filter((u) => u.pathname === "/v2/insights" && u.searchParams.get("filter.type") === type); + return { cells, reads }; + } + + it("genres only (no sound-alike Qloo knows): heatmap, places, crowd and local artists are signalled by the tags", async () => { + const { cells, reads } = await taste(request({ genreTags: [POP, SOUL] })); + expect(cells.length).toBeGreaterThan(0); + for (const type of ["urn:heatmap", "urn:entity:place", "urn:demographics", "urn:entity:artist"]) { + const sent = reads(type).filter((u) => u.searchParams.get("signal.interests.tags")); + expect(sent.length, type).toBeGreaterThan(0); + expect(sent[0].searchParams.get("signal.interests.tags")).toBe(`${POP.id},${SOUL.id}`); + } + }); + + it("no genres: no tags param anywhere (the calls stay as they were)", async () => { + const { reads } = await taste(request({})); + expect(qlooUrls.some((u) => u.searchParams.has("signal.interests.tags"))).toBe(false); + expect(reads("urn:heatmap")).toHaveLength(0); + }); + + it("an act Qloo knows (own artist): its genres don't join the taste reads", async () => { + const { reads } = await taste(request({ genreTags: [POP], ownArtist: { qlooId: A.id, name: A.name } })); + expect(reads("urn:heatmap").length).toBeGreaterThan(0); + expect(reads("urn:heatmap").every((u) => !u.searchParams.has("signal.interests.tags"))).toBe(true); + }); +}); diff --git a/tests/unit/research/run-timeouts.test.ts b/tests/unit/research/run-timeouts.test.ts new file mode 100644 index 00000000..1b69b6d4 --- /dev/null +++ b/tests/unit/research/run-timeouts.test.ts @@ -0,0 +1,103 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { learnStreamEventSchema, type LearnStreamEvent } from "@/lib/learn-schema"; +import { RESEARCH_TOOLS } from "@/lib/repertoire-research"; +import { CATALOG } from "@/server/qloo-mock/catalog"; + +/** + * The research hang (v1.1.1): a step that never returns (a big Spotify catalogue) must not leave the run `running` with no + * `end`. Step 2's catalogue fetch hangs here; the step timeout, the run watchdog and the `research-cut` fault are shortened + * or switched on through the shared objects below. + */ +const T = vi.hoisted(() => ({ timeouts: { stepMs: 200, runMs: 5_000 }, hang: true })); +vi.mock("@/server/playlist/config", async (importOriginal) => ({ ...(await importOriginal()), RESEARCH_TIMEOUTS: T.timeouts })); +vi.mock("@/server/adapters/repertoire", async (importOriginal) => { + const original = await importOriginal(); + return { ...original, getArtistCatalogue: (...a: Parameters) => (T.hang ? new Promise(() => {}) : original.getArtistCatalogue(...a)) }; +}); + +async function load() { + vi.resetModules(); + vi.stubEnv("BUSKMAP_OFFLINE", "1"); + vi.stubEnv("BUSKMAP_ALLOW_FAULTS", "1"); + vi.stubEnv("VERCEL", ""); + (await import("@/server/playlist/store")).resetPlaylistStoresForTests(); + (await import("@/server/learn/store")).resetLearnStoresForTests(); + (await import("@/server/learn/runs")).clearLearnRunsForTests(); + return { + research: await import("@/app/api/playlist/research/route"), + learn: await import("@/app/api/playlist/learn/route"), + events: await import("@/app/api/playlist/learn/runs/[runId]/events/route"), + }; +} + +const JULIA = CATALOG.find((a) => a.name === "Julia Jacklin")!; +const LINK = `https://open.spotify.com/artist/${JULIA.spotifyId}`; +const base = "http://localhost:3100/api/playlist"; +const req = (method: string, path: string, body?: unknown, cookie?: string, headers: Record = {}) => + new Request(base + path, { method, headers: { "content-type": "application/json", host: "localhost:3100", ...(cookie ? { cookie } : {}), ...headers }, body: body === undefined ? undefined : JSON.stringify(body) }); +const ctx = (v: Record) => ({ params: Promise.resolve(v) }) as never; + +async function readEvents(res: Response): Promise { + return (await res.text()) + .split("\n") + .filter((l) => l.trim()) + .map((l) => { + const parsed = learnStreamEventSchema.safeParse(JSON.parse(l)); + expect(parsed.success, l.slice(0, 160)).toBe(true); + return parsed.data!; + }); +} +const songsEnd = (events: LearnStreamEvent[]) => events.find((e) => e.type === "tool-end" && e.tool === RESEARCH_TOOLS.songs) as Extract | undefined; + +beforeEach(() => { + vi.unstubAllEnvs(); + T.hang = true; +}); + +describe("research run time limits", () => { + it("a step past its time fails as 'took too long' and the run ends cleanly (failed, timeout), its row no longer running", async () => { + Object.assign(T.timeouts, { stepMs: 200, runMs: 5_000 }); + const m = await load(); + const started = Date.now(); + const res = await m.research.POST(req("POST", "/research", { link: LINK })); + const cookie = res.headers.get("set-cookie")?.split(";")[0] ?? ""; + const events = await readEvents(res); + expect(Date.now() - started).toBeLessThan(3_000); + expect(songsEnd(events)).toMatchObject({ ok: false, summary: expect.stringMatching(/took too long$/) }); + // No step after the cut one starts. + expect(events.filter((e) => e.type === "tool-start").map((e) => (e as { tool: string }).tool)).toEqual([RESEARCH_TOOLS.profile, RESEARCH_TOOLS.songs]); + expect(events.at(-1)).toMatchObject({ type: "end", ok: false, code: "timeout" }); + const list = await (await m.learn.GET(req("GET", "/learn", undefined, cookie))).json(); + expect(list.running).toBeNull(); + expect(list.runs[0]).toMatchObject({ status: "failed" }); + }); + + it("the run watchdog ends a run still going before maxDuration: the open step is closed and `end` (timeout) is pushed", async () => { + Object.assign(T.timeouts, { stepMs: 60_000, runMs: 300 }); + const m = await load(); + const started = Date.now(); + const res = await m.research.POST(req("POST", "/research", { link: LINK })); + const events = await readEvents(res); + expect(Date.now() - started).toBeLessThan(3_000); + expect(songsEnd(events)).toMatchObject({ ok: false, summary: expect.stringMatching(/took too long$/) }); + const ends = events.filter((e) => e.type === "end"); + expect(ends).toHaveLength(1); + expect(events.at(-1)).toMatchObject({ type: "end", ok: false, code: "timeout" }); + }); + + it("research-cut: the stream ends after step 2 starts with no end, the row stays running and a join is told running_elsewhere", async () => { + Object.assign(T.timeouts, { stepMs: 60_000, runMs: 60_000 }); + T.hang = false; + const m = await load(); + const res = await m.research.POST(req("POST", "/research", { link: LINK }, undefined, { "x-buskmap-faults": "research-cut" })); + const cookie = res.headers.get("set-cookie")?.split(";")[0] ?? ""; + const events = await readEvents(res); + expect(events.at(-1)).toMatchObject({ type: "tool-start", tool: RESEARCH_TOOLS.songs }); + expect(events.some((e) => e.type === "end")).toBe(false); + const list = await (await m.learn.GET(req("GET", "/learn", undefined, cookie))).json(); + expect(list.running).toMatchObject({ runId: expect.any(String) }); + const join = await m.events.GET(req("GET", `/learn/runs/${list.running.runId}/events`, undefined, cookie), ctx({ runId: list.running.runId })); + expect(join.status).toBe(409); + expect((await join.json()).error.code).toBe("running_elsewhere"); + }); +}); diff --git a/tests/unit/research/view.test.ts b/tests/unit/research/view.test.ts index 89baf93c..478b2af7 100644 --- a/tests/unit/research/view.test.ts +++ b/tests/unit/research/view.test.ts @@ -63,6 +63,16 @@ describe("toRepertoireView", () => { expect(v.totals.toLearn).toBeNull(); }); + it("cut: a stream that ended with no end is over, not running; the step it was on is failed (cut) and the rest skipped", async () => { + const { events } = await fullRun(); + const mid = [...cutAfter(events, "research_qloo"), events.find((e) => e.type === "tool-start" && e.tool === "research_sounds_like")!]; + const v = toRepertoireView({ events: mid, cut: true }); + expect(v.phase).toBe("done"); + expect(v.end).toBeNull(); + expect(statuses(v)).toMatchObject({ profile: "done", songs: "done", qloo: "done", soundsLike: "failed", fans: "skipped", summary: "skipped" }); + expect(toRepertoireView({ events: mid }).phase).toBe("running"); + }); + it("Hive Identity: the songs step says its live source (Spotify first) for its running mark, and a done step keeps the source it came from", async () => { const { events } = await fullRun(); const startSongs = events.findIndex((e) => e.type === "tool-start" && e.tool === "research_songs"); From f85475ac12c96ff77ba86750bdb8b63aaccba29d Mon Sep 17 00:00:00 2001 From: T04435 Date: Fri, 23 Oct 2026 23:14:07 +1100 Subject: [PATCH 2/3] fix(research): an unknown act's Last.fm sound-alikes are who it sounds like, and are saved When Qloo confirmed none of Last.fm's similar artists by name, the Hive fell back to the act's genre tags and step 4 listed Qloo's top artists for the genre (Jay-Z, OutKast... for "hip hop"), saving nothing. - Last.fm's similar artists are looked up on Spotify (exact name: id and photo); a Spotify id is Qloo's certain lookup, so any Qloo knows that way become the signal as in v1.0.2 (fallback "soundsLike"). - Otherwise step 4 lists them (QlooOutput.similar) while the genres stay the Qloo signal. - Sound-alikes the run finds are saved as the Hive's "Artists you sound like", merged up to 10, never over ones the performer picked. - Step 2 names its source in words beside the logo ("from Spotify"). Co-Authored-By: Claude Opus 5.5 --- src/components/plan/tool-results.tsx | 5 +- src/lib/repertoire-research.ts | 5 +- src/server/research/run.ts | 51 +++++++++++++-- tests/unit/research/qloo-paths.test.ts | 13 ++-- .../unit/research/unknown-act-lastfm.test.ts | 65 +++++++++++++++++++ 5 files changed, 126 insertions(+), 13 deletions(-) create mode 100644 tests/unit/research/unknown-act-lastfm.test.ts diff --git a/src/components/plan/tool-results.tsx b/src/components/plan/tool-results.tsx index f4ef80db..649f4deb 100644 --- a/src/components/plan/tool-results.tsx +++ b/src/components/plan/tool-results.tsx @@ -217,10 +217,11 @@ const RESEARCH_UI: Record = { research_songs: researchUi((o) => { const songs = arr(o.songs).filter((x) => str(x.title)); if (!songs.length) return str(o.summary) ?? ""; - const from = str(o.source) === "spotify" || str(o.source) === "lastfm" ? [" from ", mentionSource(str(o.source) as "spotify" | "lastfm")] : []; + // The source's logo is an image: its name follows in words, so the line never reads "from ,". + const from = str(o.source) === "spotify" || str(o.source) === "lastfm" ? [" from ", mentionSource(str(o.source) as "spotify" | "lastfm"), str(o.source) === "spotify" ? " Spotify" : " Last.fm"] : []; // Spotify gave no catalogue (rate-limited or down): say so plainly, then what was added from Last.fm. if (o.spotifyDown === "limited" || o.spotifyDown === "down") { - return [o.spotifyDown === "limited" ? "Spotify isn't answering right now, so " : "Spotify didn't answer, so ", `${num(o.added) ?? songs.length} of your songs come from `, mentionSource("lastfm"), ", including ", ...listOf(songs.slice(0, 3).map((x) => mentionSong(str(x.title) as string, { artist: str(x.artist), image: str(x.image), href: str(x.href) })))]; + return [o.spotifyDown === "limited" ? "Spotify isn't answering right now, so " : "Spotify didn't answer, so ", `${num(o.added) ?? songs.length} of your songs come from `, mentionSource("lastfm"), " Last.fm, including ", ...listOf(songs.slice(0, 3).map((x) => mentionSong(str(x.title) as string, { artist: str(x.artist), image: str(x.image), href: str(x.href) })))]; } return [ `Added ${num(o.added) ?? songs.length} of your songs`, diff --git a/src/lib/repertoire-research.ts b/src/lib/repertoire-research.ts index 9a03c5db..77d4b3cf 100644 --- a/src/lib/repertoire-research.ts +++ b/src/lib/repertoire-research.ts @@ -86,6 +86,8 @@ export type QlooOutput = { foundByHive?: boolean; /** fallback "genres": the Qloo genre tags used as the signal. */ genreTags?: { id: string; name: string }[]; + /** Last.fm's similar artists the Hive found for the act when Qloo confirmed none of them: step 4's "who you sound like". */ + similar?: { name: string; spotifyId?: string; image?: string }[]; /** A genre the performer picked when nothing else was found; kept for this link so a refresh uses it again. */ genrePick?: string; }; @@ -97,7 +99,8 @@ export type ResearchArtistOut = { name: string; qlooId: string; spotifyId?: stri export type ResearchAsk = { asks: number; tried: { artists: string[]; genres: string[] } }; export type SoundsLikeOutput = BasedOn & { summary: string; - artists: ResearchArtistOut[]; + /** qlooId is absent for a Last.fm sound-alike Qloo doesn't know (QlooOutput.similar). */ + artists: (Omit & { qlooId?: string })[]; /** True when they were saved as your "Artists you sound like" (only when that list was empty). */ saved: boolean; /** The run is paused here: Qloo has no taste signal for the act, so the step asks for artists you sound like, or a genre. */ diff --git a/src/server/research/run.ts b/src/server/research/run.ts index c077ca2f..24fdba86 100644 --- a/src/server/research/run.ts +++ b/src/server/research/run.ts @@ -39,7 +39,7 @@ import { recordCalls } from "@/server/adapters/call-log"; import type { AdapterContext } from "@/server/adapters/core"; import * as qloo from "@/server/adapters/qloo"; import { withQlooUsage, type QlooUsage } from "@/server/adapters/qloo/usage"; -import { getArtistCatalogue, getArtistProfile } from "@/server/adapters/repertoire"; +import { findArtists, getArtistCatalogue, getArtistProfile } from "@/server/adapters/repertoire"; import { qlooBudgetSpent } from "@/server/adapters/qloo/budget"; import { spotifyLimitedUntil } from "@/server/adapters/repertoire/pacer"; import { getSongs } from "@/server/adapters/songs"; @@ -327,6 +327,19 @@ export async function executeResearchRun(args: ResearchRunArgs): Promise { return false; } + /** + * Sound-alikes the run found by itself (the Hive's Last.fm similar artists) saved as "Artists you sound like", marked as the + * Hive's: merged after the ones already there, up to SOUNDS_LIKE_MAX, and only while the performer has picked none of their own. + */ + async function saveHiveAlikes(list: readonly SoundsLike[]): Promise { + const fresh = (await playlists.get(playlist.id)) ?? playlist; + if (fresh.soundsLike.some((a) => a.foundBy !== "hive")) return false; + const have = new Set(fresh.soundsLike.map((a) => nameKey(a.name))); + const added = list.filter((a) => !have.has(nameKey(a.name)) && have.add(nameKey(a.name))).map((a): SoundsLike => ({ ...a, name: a.name.slice(0, 80), foundBy: "hive" })); + if (added.length === 0) return false; + return patch({ soundsLike: [...fresh.soundsLike, ...added].slice(0, SOUNDS_LIKE_MAX) }); + } + /** What stands in for the act when Qloo doesn't know it. alikes: artists like it (the performer's own, or found by the Hive); tags: its Spotify genres as Qloo tags; neither: no taste data. */ const signalAlikes: { current: (SoundsLike & { qlooId: string })[] | null } = { current: null }; const genreSignal: { tags: { id: string; name: string }[] } = { tags: [] }; @@ -557,19 +570,44 @@ export async function executeResearchRun(args: ResearchRunArgs): Promise { const found = hive.alikes.map((a): SoundsLike & { qlooId: string } => ({ name: a.name, qlooId: a.qlooId, ...(a.spotifyId ? { spotifyId: a.spotifyId } : {}), ...(a.image ? { image: a.image } : {}), foundBy: "hive" })); signalAlikes.current = found; // Saved as "Artists you sound like" (marked as the Hive's) only when the performer has none of their own: theirs are never overwritten. - if (playlist.soundsLike.length === 0) await patch({ soundsLike: found.slice(0, RESEARCH_SOUNDS_LIKE_SAVED) }); + await saveHiveAlikes(found.slice(0, RESEARCH_SOUNDS_LIKE_SAVED)); return { ...empty, summary: R.fallback.label(found.slice(0, 3).map((a) => a.name)), fallback: "soundsLike", foundByHive: true, ...declinedMark, ...genreMark }; } + // Last.fm knows artists like the act but Qloo confirmed none of them by name: each is looked up on Spotify (an exact name + // gives its id and photo), and a Spotify id is Qloo's certain lookup. Any Qloo knows that way are the signal, as above; + // otherwise they are still who the act sounds like (step 4 lists them and they are saved) and the act's genres are the signal. + const market = CITIES[(playlist.city in CITIES ? playlist.city : "melbourne") as keyof typeof CITIES].countryCode; + const similar: SoundsLike[] = []; + for (const name of hive.similar.slice(0, RESEARCH_SOUNDS_LIKE_SAVED)) { + const res = await findArtists(name, market, adapters).catch(() => null); + const hit = res && res.mode !== "fallback" ? res.data.find((a) => nameKey(a.name) === nameKey(name)) : undefined; + similar.push({ + name: name.slice(0, 80), + ...(hit && SPOTIFY_ID.test(hit.id) && hit.id !== args.spotifyId ? { spotifyId: hit.id } : {}), + ...(hit?.image && /^https:\/\//.test(hit.image) && hit.image.length <= 500 ? { image: hit.image } : {}), + }); + } + const byId = similar.filter((a) => a.spotifyId); + const confirmed = byId.length ? await resolveSoundAlikes(byId, { name: profile.name, actName: actLabel, spotifyId: args.spotifyId }, adapters) : null; + if (confirmed && !confirmed.unavailable && confirmed.found.length > 0) { + const qlooOf = new Map(byId.flatMap((a, i) => (confirmed.matches[i] ? [[nameKey(a.name), confirmed.matches[i]!.qlooId] as const] : []))); + const all = similar.map((a): SoundsLike => (qlooOf.has(nameKey(a.name)) ? { ...a, qlooId: qlooOf.get(nameKey(a.name)) } : a)); + signalAlikes.current = all.filter((a): a is SoundsLike & { qlooId: string } => Boolean(a.qlooId)).map((a) => ({ ...a, foundBy: "hive" as const })); + await saveHiveAlikes(all); + return { ...empty, summary: R.fallback.label(signalAlikes.current.slice(0, 3).map((a) => a.name)), fallback: "soundsLike", foundByHive: true, ...declinedMark, ...genreMark }; + } + const similarMark = similar.length ? { similar } : {}; + if (similar.length) await saveHiveAlikes(similar); if (hive.source === "genres") { genreSignal.tags = hive.tags; - return { ...empty, summary: R.fallback.genresLabel(hive.tags.slice(0, 3).map((t) => t.name)), fallback: "genres", genreTags: hive.tags, ...declinedMark, ...genreMark }; + return { ...empty, summary: R.fallback.genresLabel(hive.tags.slice(0, 3).map((t) => t.name)), fallback: "genres", genreTags: hive.tags, ...declinedMark, ...genreMark, ...similarMark }; } // Nothing else to go on: a weak match (a name Qloo has, with no genre) is better than no taste data at all. if (weak) { by = weak.by; return matchedOut(weak); } - return { ...empty, summary: R.fallback.noneLabel, fallback: "none", ...declinedMark, ...genreMark }; + return { ...empty, summary: R.fallback.noneLabel, fallback: "none", ...declinedMark, ...genreMark, ...similarMark }; }); if (!found) { await finish({ status: "failed", endCode: qlooEndCode(), saved: [], usage: null }); @@ -715,6 +753,11 @@ export async function executeResearchRun(args: ResearchRunArgs): Promise { .slice(0, RESEARCH_SOUNDS_LIKE_SAVED) .map((e) => ({ name: e.name, qlooId: e.id, ...(e.spotifyId && SPOTIFY_ID.test(e.spotifyId) ? { spotifyId: e.spotifyId } : {}), ...(e.image && /^https:\/\//.test(e.image) ? { image: e.image } : {}), ...(e.tags.length ? { tags: e.tags.slice(0, 3).map((t) => t.name) } : {}), affinity: r2(e.affinity) })); genreAlikes = artists.slice(0, RESEARCH_SIGNAL_SOUND_ALIKES).map((a) => ({ qlooId: a.qlooId, name: a.name })); + // The Hive's Last.fm sound-alikes, when it found some, are who the act sounds like; the genre's artists only stand in as the signal. + if (found.similar?.length) { + const lastfm = found.similar.map((a) => ({ ...a, affinity: null })); + return { summary: `You sound like ${lastfm.slice(0, 3).map((a) => a.name).join(", ")}`, artists: lastfm, saved: playlist.soundsLike.some((a) => a.foundBy === "hive"), basedOn: "genres" }; + } return { summary: artists.length ? basedOnGenre(andList(artists.slice(0, 3).map((a) => a.name))) : "Qloo found no artists in your genre", artists, saved: false, basedOn: "genres" }; } if (!own) { diff --git a/tests/unit/research/qloo-paths.test.ts b/tests/unit/research/qloo-paths.test.ts index 23d6c866..37aa9693 100644 --- a/tests/unit/research/qloo-paths.test.ts +++ b/tests/unit/research/qloo-paths.test.ts @@ -241,13 +241,14 @@ describe("path 4: Qloo doesn't know the act, so the Hive finds artists like it a const s = v.research.steps as ResearchSnapshot["steps"]; expect(s.qloo!.output).toMatchObject({ matched: false, fallback: "genres", genreTags: [{ id: expect.stringMatching(/^urn:tag:genre/), name: "Indie Folk" }], summary: PLAYLIST_COPY.research.fallback.genresLabel(["Indie Folk"]) }); expect(s.qloo!.output!.summary).toBe("Qloo doesn't know you yet, so this is based on your genres: Indie Folk."); - // The genre's artists on Qloo stand in for who the act sounds like (never saved as the performer's), and every Qloo step asks Qloo. - expect(v.soundsLike).toHaveLength(0); - expect(s.soundsLike!.output).toMatchObject({ basedOn: "genres" }); - const genreAlikes = s.soundsLike!.output!.artists.map((a) => a.name); - expect(genreAlikes.length).toBeGreaterThan(0); + // Last.fm's similar artists are who the act sounds like (listed, and saved as the Hive's); the genre's Qloo artists are the signal. + const similar: string[] = [...OFFLINE_TEST_ACTS.unresolved.similar]; + expect(s.qloo!.output!.similar!.map((a) => a.name)).toEqual(similar); + expect(v.soundsLike.map((a: { name: string; foundBy?: string }) => [a.name, a.foundBy])).toEqual(similar.map((name) => [name, "hive"])); + expect(s.soundsLike!.output).toMatchObject({ basedOn: "genres", saved: true, summary: `You sound like ${similar.join(", ")}` }); + expect(s.soundsLike!.output!.artists.map((a) => a.name)).toEqual(similar); expect(s.fansLove!.output).toMatchObject({ basedOn: "genres", signalledBy: ["Indie Folk"] }); - expect(s.fansLove!.output!.artists.some((a) => genreAlikes.includes(a.name))).toBe(false); + expect(s.fansLove!.output!.artists.some((a) => similar.includes(a.name))).toBe(false); expect(s.fansLove!.output!.artists.length).toBeGreaterThan(0); expect(s.genres!.output).toMatchObject({ basedOn: "genres" }); expect(s.genres!.output!.tags.map((t) => t.name)).toEqual(["Indie Folk"]); diff --git a/tests/unit/research/unknown-act-lastfm.test.ts b/tests/unit/research/unknown-act-lastfm.test.ts new file mode 100644 index 00000000..ac113ad4 --- /dev/null +++ b/tests/unit/research/unknown-act-lastfm.test.ts @@ -0,0 +1,65 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { ResearchSnapshot } from "@/lib/repertoire-research"; + +/** + * An act Qloo doesn't know (Al2 El Aldeano): the Hive asks Last.fm for similar artists under the act's Spotify name, and those + * are "who you sound like" and are saved as the Repertoire's "Artists you sound like", even when Qloo confirms none of them. + */ +const SPOTIFY_ID = "6zYGkCfNmXiKyyHTbNgYcW"; +const SIMILAR = ["Calle 13", "Los Aldeanos", "El B", "Silvito El Libre"]; +const asked = vi.hoisted(() => [] as string[]); + +vi.mock("@/server/adapters/repertoire", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + getArtistProfile: async () => ({ mode: "live" as const, data: { name: "Al2 El Aldeano", images: [], genres: [], fetchedAt: new Date().toISOString() } }), + getArtistCatalogue: async () => ({ mode: "fallback" as const, data: [] }), + }; +}); +vi.mock("@/server/adapters/songs", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + getSimilarArtists: async (name: string) => { + asked.push(name); + return { mode: "live" as const, data: name === "Al2 El Aldeano" ? SIMILAR.map((n) => ({ name: n, match: 0.9 })) : [] }; + }, + getTopTags: async () => ({ mode: "live" as const, data: [{ name: "indie folk", count: 100 }] }), + }; +}); + +const base = "http://localhost:3100/api/playlist"; +const req = (method: string, path: string, body?: unknown, cookie?: string) => + new Request(base + path, { method, headers: { "content-type": "application/json", host: "localhost:3100", ...(cookie ? { cookie } : {}) }, body: body === undefined ? undefined : JSON.stringify(body) }); + +beforeEach(() => { + vi.unstubAllEnvs(); + asked.length = 0; +}); + +describe("an act Qloo doesn't know, with Last.fm sound-alikes", () => { + it("asks Last.fm under the act's Spotify name, lists the sound-alikes and saves them", async () => { + vi.resetModules(); + vi.stubEnv("BUSKMAP_OFFLINE", "1"); + vi.stubEnv("VERCEL", ""); + (await import("@/server/playlist/store")).resetPlaylistStoresForTests(); + (await import("@/server/learn/store")).resetLearnStoresForTests(); + (await import("@/server/learn/runs")).clearLearnRunsForTests(); + const research = await import("@/app/api/playlist/research/route"); + const playlist = await import("@/app/api/playlist/route"); + const res = await research.POST(req("POST", "/research", { link: SPOTIFY_ID })); + const cookie = res.headers.get("set-cookie")?.split(";")[0] ?? ""; + await res.text(); + // The act-name source: the Spotify profile's name (a lookup, never a model input), not empty and not Qloo's. + expect(asked[0]).toBe("Al2 El Aldeano"); + const v = (await (await playlist.GET(req("GET", "", undefined, cookie))).json()).playlist; + const s = v.research.steps as ResearchSnapshot["steps"]; + expect(s.qloo!.output).toMatchObject({ matched: false }); + expect(s.qloo!.output!.similar!.map((a) => a.name)).toEqual(SIMILAR); + expect(s.soundsLike!.output!.artists.map((a) => a.name)).toEqual(SIMILAR); + expect(s.soundsLike!.output!.summary).toBe("You sound like Calle 13, Los Aldeanos, El B"); + expect(v.soundsLike.map((a: { name: string }) => a.name)).toEqual(SIMILAR); + expect(v.soundsLike.every((a: { foundBy?: string }) => a.foundBy === "hive")).toBe(true); + }); +}); From c0fb5559c3c210ebe5f3c5064e9dd4764377e739 Mon Sep 17 00:00:00 2001 From: T04435 Date: Fri, 23 Oct 2026 23:17:43 +1100 Subject: [PATCH 3/3] fix(research): give the Spotify catalogue 35 s Spotify refuses Get Several Albums for this app, so every release costs one paced call; 20 s read only about 12 songs for a singles-heavy act. 35 s fits about 31 calls; the step limit goes to 50 s, the run watchdog stays at 110 s. Co-Authored-By: Claude Opus 5.5 --- src/server/adapters/repertoire/pacer.ts | 2 +- src/server/playlist/config.ts | 2 +- tests/unit/adapters/repertoire-pacer.test.ts | 13 ++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/server/adapters/repertoire/pacer.ts b/src/server/adapters/repertoire/pacer.ts index 39eb37ec..63083477 100644 --- a/src/server/adapters/repertoire/pacer.ts +++ b/src/server/adapters/repertoire/pacer.ts @@ -36,7 +36,7 @@ export const CATALOGUE_CALL_BUDGET = 60; * Wall-clock time one catalogue fetch may take. At 20 s it stops and returns the songs found so far (a partial catalogue), * so research step 2 stays well inside the function's 120 s (a big catalogue paced at 2 calls a second took minutes). */ -export const CATALOGUE_DEADLINE_MS = 20_000; +export const CATALOGUE_DEADLINE_MS = 35_000; const ADAPTER = "spotify"; /** How often an instance re-reads the shared row while it believes Spotify is fine. */ const RECHECK_MS = 30_000; diff --git a/src/server/playlist/config.ts b/src/server/playlist/config.ts index cf02399f..37fd04ec 100644 --- a/src/server/playlist/config.ts +++ b/src/server/playlist/config.ts @@ -113,7 +113,7 @@ export const SUGGEST_RELATED_TAKE = 20; * that takes longer than `stepMs` fails ("took too long") and ends the run; `runMs` after the start a watchdog ends the run * as failed (end code "timeout") so the stream always gets its `end` before the function is stopped. */ -export const RESEARCH_TIMEOUTS = { stepMs: 45_000, runMs: 110_000 } as const; +export const RESEARCH_TIMEOUTS = { stepMs: 50_000, runMs: 110_000 } as const; /** Sound-alikes (step 4) saved as your "Artists you sound like" when that list is empty. */ export const RESEARCH_SOUNDS_LIKE_SAVED = 6; /** Sound-alikes that join your own artist in step 6's signal. */ diff --git a/tests/unit/adapters/repertoire-pacer.test.ts b/tests/unit/adapters/repertoire-pacer.test.ts index e000a64c..e3e9f2bb 100644 --- a/tests/unit/adapters/repertoire-pacer.test.ts +++ b/tests/unit/adapters/repertoire-pacer.test.ts @@ -101,8 +101,8 @@ describe("Spotify pacer, budget and breaker (U34)", () => { expect(new CallBudget(1).capWait(30_000)).toBe(30_000); }); - it("stops at the catalogue's 20 s deadline and returns the newest songs read so far (U34 pacing on a big catalogue)", async () => { - expect(CATALOGUE_DEADLINE_MS).toBe(20_000); + it("stops at the catalogue's 35 s deadline and returns the newest songs read so far (U34 pacing on a big catalogue)", async () => { + expect(CATALOGUE_DEADLINE_MS).toBe(35_000); vi.useFakeTimers(); // Real pace (500 ms) and window (20 calls per 30 s): the window fills in about 10 s, the next slot is past the deadline. resetPacerForTests({ dailyMax: Infinity }); @@ -114,13 +114,12 @@ describe("Spotify pacer, budget and breaker (U34)", () => { await vi.advanceTimersByTimeAsync(60_000); const result = await pending; expect(result.mode).toBe("live"); - // 20 calls fit: 14 release pages and 6 batches (7 would need the 21st call), so 120 songs, newest releases first. - expect(spotifyCalls(calls)).toHaveLength(20); - expect(albumCalls(calls)).toHaveLength(6); - expect(result.data).toHaveLength(120); + // 31 calls fit: a full window, then the next window from 30 s until the deadline. Newest releases first. + expect(spotifyCalls(calls)).toHaveLength(31); + expect(result.data.length).toBeGreaterThan(120); expect(result.data[0].title).toBe("Song al0"); // It gave up as soon as the next free slot was past the deadline, rather than queueing 30 s for it. - expect(doneAt - started).toBeLessThan(CATALOGUE_DEADLINE_MS); + expect(doneAt - started).toBeLessThanOrEqual(CATALOGUE_DEADLINE_MS); }); it("stops at the per-fetch call budget (60) and returns what it has", async () => {