diff --git a/apps/web/components/cms/cms-workspace.tsx b/apps/web/components/cms/cms-workspace.tsx index f0544e1..cfb5cef 100644 --- a/apps/web/components/cms/cms-workspace.tsx +++ b/apps/web/components/cms/cms-workspace.tsx @@ -20,6 +20,7 @@ import { } from "@/components/cms/post-dialogs"; import { RightPanel } from "@/components/cms/right-panel"; import { PublicationsDashboard } from "@/components/cms/publications-dashboard"; +import { ResearchSection } from "@/components/cms/research-section"; import { WorkspaceHeader } from "@/components/cms/workspace-header"; import { FeedbackSection } from "@/components/cms/feedback-section"; import { MobileWorkspaceFooter } from "@/components/cms/mobile-workspace-footer"; @@ -682,6 +683,8 @@ export function CmsWorkspace() { isSyncing={isSyncing} onSync={syncPublications} /> + ) : activeView === "research" ? ( + ) : activeView === "feedback" ? ( ) : ( diff --git a/apps/web/components/cms/mobile-workspace-footer.tsx b/apps/web/components/cms/mobile-workspace-footer.tsx index ce29430..a773b34 100644 --- a/apps/web/components/cms/mobile-workspace-footer.tsx +++ b/apps/web/components/cms/mobile-workspace-footer.tsx @@ -1,6 +1,6 @@ "use client"; -import { BookOpenIcon, LibraryIcon, MessageSquareTextIcon, RocketIcon, SaveIcon } from "lucide-react"; +import { BookOpenIcon, LibraryIcon, MessageSquareTextIcon, RocketIcon, SaveIcon, TelescopeIcon } from "lucide-react"; import { Button } from "@/components/ui/button"; import { cn } from "@/lib/utils"; import type { DraftSaveState } from "@/lib/draft-editor"; @@ -49,17 +49,18 @@ export function MobileWorkspaceFooter({ } const views = [ - { value: "posts", label: "Posts", icon: BookOpenIcon }, - { value: "publications", label: "Publications", icon: LibraryIcon }, - { value: "feedback", label: "Feedback", icon: MessageSquareTextIcon }, + { value: "posts", label: "Posts", mobileLabel: "Posts", icon: BookOpenIcon }, + { value: "publications", label: "Publications", mobileLabel: "Sites", icon: LibraryIcon }, + { value: "research", label: "Research", mobileLabel: "Research", icon: TelescopeIcon }, + { value: "feedback", label: "Feedback", mobileLabel: "Feedback", icon: MessageSquareTextIcon }, ] as const; return ( diff --git a/apps/web/components/cms/research-section.tsx b/apps/web/components/cms/research-section.tsx new file mode 100644 index 0000000..05c0eef --- /dev/null +++ b/apps/web/components/cms/research-section.tsx @@ -0,0 +1,251 @@ +"use client"; + +import * as React from "react"; +import { format, parseISO } from "date-fns"; +import { + ArrowUpRightIcon, + BookMarkedIcon, + HighlighterIcon, + LibraryBigIcon, + LoaderCircleIcon, + RefreshCwIcon, +} from "lucide-react"; +import { + loadResearch, + type MarginResearchAnnotation, + type ResearchResponse, + type SembleResearchCard, + type SembleResearchCollection, +} from "@/lib/research-api"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader } from "@/components/ui/card"; +import { Empty, EmptyDescription, EmptyTitle } from "@/components/ui/empty"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; + +export function ResearchSection() { + const [research, setResearch] = React.useState(null); + const [error, setError] = React.useState(""); + const [isLoading, setIsLoading] = React.useState(true); + const [refreshVersion, setRefreshVersion] = React.useState(0); + + const refresh = React.useCallback(() => { + setIsLoading(true); + setError(""); + setRefreshVersion((version) => version + 1); + }, []); + + React.useEffect(() => { + const controller = new AbortController(); + loadResearch(controller.signal) + .then(setResearch) + .catch((loadError: unknown) => { + if (loadError instanceof DOMException && loadError.name === "AbortError") return; + setError(loadError instanceof Error ? loadError.message : "Could not load research."); + }) + .finally(() => { + if (!controller.signal.aborted) setIsLoading(false); + }); + return () => controller.abort(); + }, [refreshVersion]); + + return ( +
+
+
+
+

Your AT Protocol library

+

Research

+

+ Browse your Semble collections and Margin annotations when you are looking for something worth developing. +

+
+ +
+ + {isLoading && !research ? ( +
+ Loading your research… +
+ ) : error ? ( + + Research is unavailable + {error} + + + ) : research ? ( + + + + + Semble {research.semble.collections.length} + + + + Margin {research.margin.annotations.length} + + + + + + + + + + + + ) : null} +
+
+ ); +} + +function SourceNotice({ message }: { message?: string }) { + return message ? ( +
+ {message} +
+ ) : null; +} + +function SembleCollections({ collections }: { collections: SembleResearchCollection[] }) { + if (!collections.length) { + return ( + + No Semble collections found + Collections saved by this linked account will appear here. + + ); + } + + return ( +
+ {collections.map((collection) => ( + + +
+
+

{collection.name}

+ {collection.description ? {collection.description} : null} +
+
+ {collection.accessType.toLowerCase()} + {collection.cards.length} {collection.cards.length === 1 ? "item" : "items"} +
+
+ + + {collection.cards.length ? collection.cards.map((card) => ( + + )) : ( +

This collection is empty.

+ )} + + + ))} +
+ ); +} + +function SembleCard({ card }: { card: SembleResearchCard }) { + const title = card.title || card.note || card.url || "Untitled saved item"; + return ( +
+
+
+

{title}

+ {card.siteName ? {card.siteName} : null} +
+ {card.author ?

By {card.author}

: null} + {card.note && card.note !== title ?

{card.note}

: null} + {card.description ?

{card.description}

: null} + {card.createdAt ? : null} +
+ {safeHTTPURL(card.url) ? ( + + ) : null} +
+ ); +} + +function MarginAnnotations({ annotations }: { annotations: MarginResearchAnnotation[] }) { + if (!annotations.length) { + return ( + + No Margin annotations found + Notes, highlights, and bookmarks from this linked account will appear here. + + ); + } + + return ( +
+ {annotations.map((annotation) => ( + + +
+ + {annotation.motivation} + + +
+

{annotation.title || sourceHost(annotation.source)}

+ + + {annotation.quote ? ( +
“{annotation.quote}”
+ ) : null} + {annotation.body ?

{annotation.body}

: null} + {annotation.tags.length ? ( +
+ {annotation.tags.map((tag) => {tag})} +
+ ) : null} + {safeHTTPURL(annotation.source) ? ( + + ) : null} + + + ))} +
+ ); +} + +function ResearchDate({ value }: { value: string }) { + let label: string; + try { + label = format(parseISO(value), "MMM d, yyyy"); + } catch { + return null; + } + return ; +} + +function safeHTTPURL(value?: string) { + if (!value) return false; + try { + const url = new URL(value); + return url.protocol === "http:" || url.protocol === "https:"; + } catch { + return false; + } +} + +function sourceHost(value: string) { + try { + return new URL(value).hostname.replace(/^www\./, ""); + } catch { + return "Untitled annotation"; + } +} diff --git a/apps/web/components/cms/workspace-header.tsx b/apps/web/components/cms/workspace-header.tsx index 459606e..9fed5f0 100644 --- a/apps/web/components/cms/workspace-header.tsx +++ b/apps/web/components/cms/workspace-header.tsx @@ -11,6 +11,7 @@ import { RocketIcon, Settings2Icon, SunIcon, + TelescopeIcon, } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -183,6 +184,7 @@ function WorkspaceNavigation({ const views = [ { value: "posts", label: "Posts", icon: BookOpenIcon }, { value: "publications", label: "Publications", icon: LibraryIcon }, + { value: "research", label: "Research", icon: TelescopeIcon }, { value: "feedback", label: "Feedback", icon: MessageSquareTextIcon }, ] as const; @@ -208,7 +210,13 @@ function WorkspaceNavigation({ } function mobileViewTitle(view: WorkspaceView) { - return view === "posts" ? "Posts" : view === "publications" ? "Publications" : "Feedback"; + return view === "posts" + ? "Posts" + : view === "publications" + ? "Publications" + : view === "research" + ? "Research" + : "Feedback"; } function MobileAccountSheet({ diff --git a/apps/web/lib/research-api.ts b/apps/web/lib/research-api.ts new file mode 100644 index 0000000..d6b0871 --- /dev/null +++ b/apps/web/lib/research-api.ts @@ -0,0 +1,50 @@ +import { apiFetch } from "@/lib/api"; + +export type SembleResearchCard = { + uri: string; + url?: string; + title?: string; + description?: string; + siteName?: string; + author?: string; + note?: string; + createdAt?: string; +}; + +export type SembleResearchCollection = { + uri: string; + name: string; + description?: string; + accessType: string; + createdAt?: string; + updatedAt?: string; + cards: SembleResearchCard[]; +}; + +export type MarginResearchAnnotation = { + uri: string; + motivation: string; + source: string; + title?: string; + body?: string; + quote?: string; + tags: string[]; + color?: string; + createdAt: string; + modifiedAt?: string; +}; + +export type ResearchResponse = { + semble: { + collections: SembleResearchCollection[]; + error?: string; + }; + margin: { + annotations: MarginResearchAnnotation[]; + error?: string; + }; +}; + +export function loadResearch(signal?: AbortSignal) { + return apiFetch("/api/research", { signal }); +} diff --git a/apps/web/lib/workspace-navigation.ts b/apps/web/lib/workspace-navigation.ts index 43d705d..9f83187 100644 --- a/apps/web/lib/workspace-navigation.ts +++ b/apps/web/lib/workspace-navigation.ts @@ -1,4 +1,4 @@ -export type WorkspaceView = "posts" | "publications" | "feedback"; +export type WorkspaceView = "posts" | "publications" | "research" | "feedback"; export type MobileWorkspacePane = "list" | "write" | "details" | "schedule"; export type WorkspaceNavigationState = { @@ -7,7 +7,7 @@ export type WorkspaceNavigationState = { pane: MobileWorkspacePane; }; -const workspaceViews = new Set(["posts", "publications", "feedback"]); +const workspaceViews = new Set(["posts", "publications", "research", "feedback"]); const editorPanes = new Set(["write", "details", "schedule"]); export const defaultWorkspaceNavigation: WorkspaceNavigationState = { diff --git a/apps/web/test/mobile-workspace-ui.test.tsx b/apps/web/test/mobile-workspace-ui.test.tsx index 46df36a..2f437cf 100644 --- a/apps/web/test/mobile-workspace-ui.test.tsx +++ b/apps/web/test/mobile-workspace-ui.test.tsx @@ -102,6 +102,8 @@ describe("mobile workspace controls", () => { fireEvent.click(screen.getByRole("button", { name: "Open Feedback" })); expect(onViewChange).toHaveBeenCalledWith("feedback"); + fireEvent.click(screen.getByRole("button", { name: "Open Research" })); + expect(onViewChange).toHaveBeenCalledWith("research"); }); it("keeps save and publish actions in the editing footer", () => { diff --git a/apps/web/test/research-api.test.ts b/apps/web/test/research-api.test.ts new file mode 100644 index 0000000..92e8e8b --- /dev/null +++ b/apps/web/test/research-api.test.ts @@ -0,0 +1,23 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { loadResearch } from "@/lib/research-api"; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("research API", () => { + it("loads the authenticated account research inventory", async () => { + const payload = { + semble: { collections: [] }, + margin: { annotations: [] }, + }; + const fetchMock = vi.fn().mockResolvedValue(Response.json(payload)); + vi.stubGlobal("fetch", fetchMock); + + await expect(loadResearch()).resolves.toEqual(payload); + expect(fetchMock).toHaveBeenCalledWith( + "http://localhost:8080/api/research", + expect.objectContaining({ credentials: "include" }), + ); + }); +}); diff --git a/apps/web/test/research-section.test.tsx b/apps/web/test/research-section.test.tsx new file mode 100644 index 0000000..4c52384 --- /dev/null +++ b/apps/web/test/research-section.test.tsx @@ -0,0 +1,115 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { ResearchSection } from "@/components/cms/research-section"; + +const mocks = vi.hoisted(() => ({ loadResearch: vi.fn() })); + +vi.mock("@/lib/research-api", async (importOriginal) => { + const original = await importOriginal(); + return { ...original, loadResearch: mocks.loadResearch }; +}); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.loadResearch.mockResolvedValue({ + semble: { + collections: [{ + uri: "at://did:plc:writer/network.cosmik.collection/research", + name: "Things to write about", + description: "Saved prompts and source material.", + accessType: "CLOSED", + cards: [{ + uri: "at://did:plc:writer/network.cosmik.card/source", + url: "https://example.com/source", + title: "A promising source", + description: "A useful description.", + siteName: "Example", + author: "A. Writer", + note: "Revisit the central claim.", + createdAt: "2026-08-29T12:00:00.000Z", + }], + }], + }, + margin: { + annotations: [{ + uri: "at://did:plc:writer/at.margin.note/note", + motivation: "highlighting", + source: "https://example.org/essay", + title: "An essay worth revisiting", + body: "Connect this to the publishing workflow.", + quote: "A highlighted passage", + tags: ["publishing"], + createdAt: "2026-08-30T12:00:00.000Z", + }], + }, + }); +}); + +describe("research workspace", () => { + it("browses the linked user's Semble collections and Margin annotations", async () => { + render(); + + expect(await screen.findByRole("heading", { name: "Things to write about" })).toBeInTheDocument(); + expect(screen.getByText("A promising source")).toBeInTheDocument(); + expect(screen.getByRole("link", { name: /Open source/ })).toHaveAttribute("href", "https://example.com/source"); + + const marginTab = screen.getByRole("tab", { name: /Margin 1/ }); + fireEvent.mouseDown(marginTab, { button: 0, ctrlKey: false }); + fireEvent.click(marginTab); + expect(screen.getByRole("heading", { name: "An essay worth revisiting" })).toBeInTheDocument(); + expect(screen.getByText("“A highlighted passage”")).toBeInTheDocument(); + expect(screen.getByText("Connect this to the publishing workflow.")).toBeInTheDocument(); + }); + + it("keeps one source usable when the other reports a partial failure", async () => { + mocks.loadResearch.mockResolvedValue({ + semble: { collections: [], error: "Semble records could not be loaded." }, + margin: { annotations: [] }, + }); + + render(); + + expect(await screen.findByText("Semble records could not be loaded.")).toBeInTheDocument(); + expect(screen.getByText("No Semble collections found")).toBeInTheDocument(); + const marginTab = screen.getByRole("tab", { name: /Margin 0/ }); + fireEvent.mouseDown(marginTab, { button: 0, ctrlKey: false }); + fireEvent.click(marginTab); + expect(screen.getByText("No Margin annotations found")).toBeInTheDocument(); + }); + + it("offers retry after the inventory request fails", async () => { + mocks.loadResearch + .mockRejectedValueOnce(new Error("Research service unavailable")) + .mockResolvedValueOnce({ semble: { collections: [] }, margin: { annotations: [] } }); + + render(); + + expect(await screen.findByText("Research service unavailable")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Try again" })); + await waitFor(() => expect(mocks.loadResearch).toHaveBeenCalledTimes(2)); + expect(await screen.findByText("No Semble collections found")).toBeInTheDocument(); + }); + + it("does not expose unsafe source schemes as links", async () => { + mocks.loadResearch.mockResolvedValue({ + semble: { + collections: [{ + uri: "at://did:plc:writer/network.cosmik.collection/unsafe", + name: "Unsafe links", + accessType: "OPEN", + cards: [{ + uri: "at://did:plc:writer/network.cosmik.card/unsafe", + url: "javascript:alert(1)", + title: "Untrusted item", + }], + }], + }, + margin: { annotations: [] }, + }); + + render(); + + expect(await screen.findByText("Untrusted item")).toBeInTheDocument(); + expect(screen.queryByRole("link", { name: /Open source/ })).not.toBeInTheDocument(); + }); +}); diff --git a/apps/web/test/workspace-navigation.test.ts b/apps/web/test/workspace-navigation.test.ts index 61f99c8..09190ba 100644 --- a/apps/web/test/workspace-navigation.test.ts +++ b/apps/web/test/workspace-navigation.test.ts @@ -32,6 +32,16 @@ describe("workspace navigation", () => { }); }); + it("supports the research inventory as a global destination", () => { + expect(parseWorkspaceNavigation("?view=research&draft=draft-1&pane=details")).toEqual({ + view: "research", + draftID: "", + pane: "list", + }); + expect(workspaceNavigationURL({ view: "research", draftID: "ignored", pane: "schedule" })) + .toBe("/editor?view=research"); + }); + it("serializes stable, minimal editor URLs", () => { expect(workspaceNavigationURL(defaultWorkspaceNavigation)).toBe("/editor"); expect(workspaceNavigationURL({ view: "publications", draftID: "ignored", pane: "details" })) diff --git a/services/backend/Sources/App/Controllers/ResearchController.swift b/services/backend/Sources/App/Controllers/ResearchController.swift new file mode 100644 index 0000000..c93e54f --- /dev/null +++ b/services/backend/Sources/App/Controllers/ResearchController.swift @@ -0,0 +1,12 @@ +import Vapor + +struct ResearchController: RouteCollection { + func boot(routes: RoutesBuilder) throws { + routes.get("research", use: show) + } + + func show(req: Request) async throws -> ResearchResponse { + let account = try await req.authenticatedContext().account + return await req.application.research.load(account: account, req: req) + } +} diff --git a/services/backend/Sources/App/Services/ResearchService.swift b/services/backend/Sources/App/Services/ResearchService.swift new file mode 100644 index 0000000..1357f8f --- /dev/null +++ b/services/backend/Sources/App/Services/ResearchService.swift @@ -0,0 +1,476 @@ +import Foundation +import Vapor + +protocol RepositoryRecordPageListing: Sendable { + func listRecordsPage( + account: LinkedAccount, + collection: String, + cursor: String?, + client: Client + ) async throws -> ListRecordsResponse +} + +extension ATProtoXRPCClient: RepositoryRecordPageListing {} + +struct RepositoryRecordPaginator: Sendable { + private let records: any RepositoryRecordPageListing + + init(records: any RepositoryRecordPageListing = ATProtoXRPCClient()) { + self.records = records + } + + func listAll( + account: LinkedAccount, + collection: String, + client: Client + ) async throws -> [RepositoryRecord] { + var cursor: String? + var seenCursors = Set() + var result: [RepositoryRecord] = [] + + while true { + let page = try await records.listRecordsPage( + account: account, + collection: collection, + cursor: cursor, + client: client + ) + result.append(contentsOf: page.records) + + guard !page.records.isEmpty, let nextCursor = page.cursor else { break } + guard seenCursors.insert(nextCursor).inserted else { + throw Abort(.badGateway, reason: "PDS repeated a \(collection) listing cursor") + } + cursor = nextCursor + } + + return result + } +} + +protocol ResearchLoading: Sendable { + func load(account: LinkedAccount, req: Request) async -> ResearchResponse +} + +struct ResearchService: ResearchLoading, Sendable { + private enum Collection { + static let sembleCollection = "network.cosmik.collection" + static let sembleCollectionLink = "network.cosmik.collectionLink" + static let sembleCollectionLinkRemoval = "network.cosmik.collectionLinkRemoval" + static let sembleCard = "network.cosmik.card" + static let marginNote = "at.margin.note" + } + + private let paginator: RepositoryRecordPaginator + + init(records: any RepositoryRecordPageListing = ATProtoXRPCClient()) { + paginator = RepositoryRecordPaginator(records: records) + } + + func load(account: LinkedAccount, req: Request) async -> ResearchResponse { + async let sembleFetch = fetchSemble(account: account, req: req) + async let marginFetch = fetchMargin(account: account, req: req) + + let semble: SembleResearchResponse + do { + semble = SembleResearchResponse( + collections: try await sembleFetch, + error: nil + ) + } catch { + req.logger.warning("Semble research refresh failed", metadata: [ + "accountDID": "\(account.did)", + "error": "\(error)", + ]) + semble = SembleResearchResponse( + collections: [], + error: "Semble research is temporarily unavailable." + ) + } + + let margin: MarginResearchResponse + do { + margin = MarginResearchResponse( + annotations: try await marginFetch, + error: nil + ) + } catch { + req.logger.warning("Margin research refresh failed", metadata: [ + "accountDID": "\(account.did)", + "error": "\(error)", + ]) + margin = MarginResearchResponse( + annotations: [], + error: "Margin research is temporarily unavailable." + ) + } + + return ResearchResponse(semble: semble, margin: margin) + } + + func fetchSemble(account: LinkedAccount, req: Request) async throws -> [SembleCollectionResponse] { + async let collectionFetch = paginator.listAll( + account: account, + collection: Collection.sembleCollection, + client: req.client + ) + async let linkFetch = paginator.listAll( + account: account, + collection: Collection.sembleCollectionLink, + client: req.client + ) + async let removalFetch = paginator.listAll( + account: account, + collection: Collection.sembleCollectionLinkRemoval, + client: req.client + ) + async let cardFetch = paginator.listAll( + account: account, + collection: Collection.sembleCard, + client: req.client + ) + let (collectionRecords, linkRecords, removalRecords, cardRecords) = try await ( + collectionFetch, + linkFetch, + removalFetch, + cardFetch + ) + + let collections: [SembleCollection] = collectionRecords.compactMap { record -> SembleCollection? in + guard let collection = SembleCollection(record: record, accountDID: account.did) else { + logMalformed(record, source: "Semble collection", req: req) + return nil + } + return collection + } + let cards = cardRecords.compactMap { record -> SembleCardResponse? in + guard let card = SembleCardResponse(record: record, accountDID: account.did) else { + logMalformed(record, source: "Semble card", req: req) + return nil + } + return card + } + let cardsByURI = Dictionary(cards.map { ($0.uri, $0) }, uniquingKeysWith: { _, latest in latest }) + let removedLinkURIs = Set(removalRecords.compactMap { record -> String? in + guard let removal = SembleCollectionLinkRemoval(record: record, accountDID: account.did) else { + logMalformed(record, source: "Semble collection-link removal", req: req) + return nil + } + return removal.removedLinkURI + }) + + var cardsByCollectionURI: [String: [SembleCardResponse]] = [:] + var linkedCardURIsByCollectionURI: [String: Set] = [:] + for record in linkRecords { + guard !removedLinkURIs.contains(record.uri) else { continue } + guard let link = SembleCollectionLink(record: record, accountDID: account.did), + let card = cardsByURI[link.cardURI] + else { + logMalformed(record, source: "Semble collection link", req: req) + continue + } + guard linkedCardURIsByCollectionURI[link.collectionURI, default: []].insert(link.cardURI).inserted else { + continue + } + cardsByCollectionURI[link.collectionURI, default: []].append(card) + } + + return collections.map { collection in + SembleCollectionResponse( + uri: collection.uri, + name: collection.name, + description: collection.description, + accessType: collection.accessType, + createdAt: collection.createdAt, + updatedAt: collection.updatedAt, + cards: (cardsByCollectionURI[collection.uri] ?? []).sorted { + newestFirst($0.createdAt, $1.createdAt, lhsTieBreaker: $0.uri, rhsTieBreaker: $1.uri) + } + ) + }.sorted { + newestFirst( + $0.updatedAt ?? $0.createdAt, + $1.updatedAt ?? $1.createdAt, + lhsTieBreaker: $0.uri, + rhsTieBreaker: $1.uri + ) + } + } + + func fetchMargin(account: LinkedAccount, req: Request) async throws -> [MarginAnnotationResponse] { + let records = try await paginator.listAll( + account: account, + collection: Collection.marginNote, + client: req.client + ) + return records.compactMap { record in + guard let annotation = MarginAnnotationResponse(record: record, accountDID: account.did) else { + logMalformed(record, source: "Margin annotation", req: req) + return nil + } + return annotation + }.sorted { + newestFirst( + $0.modifiedAt ?? $0.createdAt, + $1.modifiedAt ?? $1.createdAt, + lhsTieBreaker: $0.uri, + rhsTieBreaker: $1.uri + ) + } + } + + private func logMalformed(_ record: RepositoryRecord, source: String, req: Request) { + req.logger.warning("Skipping malformed or unrelated \(source) record", metadata: ["uri": "\(record.uri)"]) + } +} + +private func newestFirst( + _ lhsTimestamp: String?, + _ rhsTimestamp: String?, + lhsTieBreaker: String, + rhsTieBreaker: String +) -> Bool { + let lhsDate = researchDate(lhsTimestamp) + let rhsDate = researchDate(rhsTimestamp) + if lhsDate != rhsDate { + if let lhsDate, let rhsDate { return lhsDate > rhsDate } + return lhsDate != nil + } + return lhsTieBreaker < rhsTieBreaker +} + +private func researchDate(_ timestamp: String?) -> Date? { + guard let timestamp else { return nil } + let fractional = ISO8601DateFormatter() + fractional.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + if let date = fractional.date(from: timestamp) { return date } + let standard = ISO8601DateFormatter() + standard.formatOptions = [.withInternetDateTime] + return standard.date(from: timestamp) +} + +struct ResearchResponse: Content, Equatable, Sendable { + let semble: SembleResearchResponse + let margin: MarginResearchResponse +} + +struct SembleResearchResponse: Content, Equatable, Sendable { + let collections: [SembleCollectionResponse] + let error: String? +} + +struct SembleCollectionResponse: Content, Equatable, Sendable { + let uri: String + let name: String + let description: String? + let accessType: String + let createdAt: String? + let updatedAt: String? + let cards: [SembleCardResponse] +} + +struct SembleCardResponse: Content, Equatable, Sendable { + let uri: String + let url: String? + let title: String? + let description: String? + let siteName: String? + let author: String? + let note: String? + let createdAt: String? + + init?( + record: RepositoryRecord, + accountDID: String + ) { + guard record.belongs(to: accountDID, collection: "network.cosmik.card"), + let value = record.value.objectValue, + value.hasType("network.cosmik.card"), + let cardType = value.nonEmptyString("type"), + let content = value["content"]?.objectValue + else { return nil } + + let metadata = content["metadata"]?.objectValue + switch cardType { + case "URL": + guard content.hasType("network.cosmik.card#urlContent"), + let contentURL = content.nonEmptyString("url") + else { return nil } + url = contentURL + title = metadata?.nonEmptyString("title") + description = metadata?.nonEmptyString("description") + siteName = metadata?.nonEmptyString("siteName") + author = metadata?.nonEmptyString("author") + note = nil + case "NOTE": + guard content.hasType("network.cosmik.card#noteContent"), + let text = content.nonEmptyString("text") + else { return nil } + url = value.nonEmptyString("url") + title = nil + description = nil + siteName = nil + author = nil + note = text + default: + return nil + } + + uri = record.uri + createdAt = value.nonEmptyString("createdAt") + } +} + +struct MarginResearchResponse: Content, Equatable, Sendable { + let annotations: [MarginAnnotationResponse] + let error: String? +} + +struct MarginAnnotationResponse: Content, Equatable, Sendable { + let uri: String + let motivation: String + let source: String + let title: String? + let body: String? + let quote: String? + let tags: [String] + let color: String? + let createdAt: String + let modifiedAt: String? + + init?(record: RepositoryRecord, accountDID: String) { + guard record.belongs(to: accountDID, collection: "at.margin.note"), + let value = record.value.objectValue, + value.hasType("at.margin.note"), + let motivation = value.nonEmptyString("motivation"), + let target = value["target"]?.objectValue, + let source = target.nonEmptyString("source"), + let createdAt = value.nonEmptyString("createdAt") + else { return nil } + + uri = record.uri + self.motivation = motivation + self.source = source + title = target.nonEmptyString("title") + body = value["body"]?.objectValue?.nonEmptyString("value") + quote = Self.exactQuote(in: target["selector"]) + tags = value["tags"]?.arrayValue?.compactMap(\.stringValue) ?? [] + color = value.nonEmptyString("color") + self.createdAt = createdAt + modifiedAt = value.nonEmptyString("modifiedAt") + } + + private static func exactQuote(in selector: JSONValue?) -> String? { + if let selectors = selector?.arrayValue { + return selectors.lazy.compactMap(exactQuote).first + } + guard let selector = selector?.objectValue else { return nil } + if let exact = selector.nonEmptyString("exact") { return exact } + return exactQuote(in: selector["refinedBy"]) + } +} + +private struct SembleCollection: Sendable { + let uri: String + let name: String + let description: String? + let accessType: String + let createdAt: String? + let updatedAt: String? + + init?(record: RepositoryRecord, accountDID: String) { + guard record.belongs(to: accountDID, collection: "network.cosmik.collection"), + let value = record.value.objectValue, + value.hasType("network.cosmik.collection"), + let name = value.nonEmptyString("name"), + let accessType = value.nonEmptyString("accessType") + else { return nil } + uri = record.uri + self.name = name + description = value.nonEmptyString("description") + self.accessType = accessType + createdAt = value.nonEmptyString("createdAt") + updatedAt = value.nonEmptyString("updatedAt") + } +} + +private struct SembleCollectionLink: Sendable { + let collectionURI: String + let cardURI: String + + init?(record: RepositoryRecord, accountDID: String) { + guard record.belongs(to: accountDID, collection: "network.cosmik.collectionLink"), + let value = record.value.objectValue, + value.hasType("network.cosmik.collectionLink"), + value.nonEmptyString("addedBy") != nil, + value.nonEmptyString("addedAt") != nil, + let collectionURI = value.strongReferenceURI("collection"), + let cardURI = value.strongReferenceURI("card"), + collectionURI.belongs(to: accountDID, collection: "network.cosmik.collection"), + cardURI.belongs(to: accountDID, collection: "network.cosmik.card") + else { return nil } + self.collectionURI = collectionURI + self.cardURI = cardURI + } +} + +private struct SembleCollectionLinkRemoval: Sendable { + let removedLinkURI: String + + init?(record: RepositoryRecord, accountDID: String) { + guard record.belongs(to: accountDID, collection: "network.cosmik.collectionLinkRemoval"), + let value = record.value.objectValue, + value.hasType("network.cosmik.collectionLinkRemoval"), + value.nonEmptyString("removedAt") != nil, + let collectionURI = value.strongReferenceURI("collection"), + let removedLinkURI = value.strongReferenceURI("removedLink"), + collectionURI.belongs(to: accountDID, collection: "network.cosmik.collection") + else { return nil } + self.removedLinkURI = removedLinkURI + } +} + +private extension RepositoryRecord where Value == JSONValue { + func belongs(to accountDID: String, collection: String) -> Bool { + uri.belongs(to: accountDID, collection: collection) + } +} + +private extension String { + func belongs(to accountDID: String, collection: String) -> Bool { + guard let reference = try? ATRecordReference(uri: self) else { return false } + return reference.repo == accountDID && reference.collection == collection + } +} + +private extension Dictionary where Key == String, Value == JSONValue { + func nonEmptyString(_ key: String) -> String? { + guard let value = self[key]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines), + !value.isEmpty + else { return nil } + return value + } + + func hasType(_ expected: String) -> Bool { + self["$type"] == nil || self["$type"]?.stringValue == expected + } + + func strongReferenceURI(_ key: String) -> String? { + guard let reference = self[key]?.objectValue, + let uri = reference.nonEmptyString("uri"), + reference.nonEmptyString("cid") != nil + else { return nil } + return uri + } +} + +private struct ResearchLoadingKey: StorageKey { + typealias Value = any ResearchLoading +} + +extension Application { + var research: any ResearchLoading { + get { storage[ResearchLoadingKey.self] ?? ResearchService() } + set { storage[ResearchLoadingKey.self] = newValue } + } +} diff --git a/services/backend/Sources/App/routes.swift b/services/backend/Sources/App/routes.swift index 4d15d61..4ccd511 100644 --- a/services/backend/Sources/App/routes.swift +++ b/services/backend/Sources/App/routes.swift @@ -12,6 +12,7 @@ func routes(_ app: Application) throws { try api.register(collection: AssetController()) try api.register(collection: UnsplashController()) try api.register(collection: FeedbackController()) + try api.register(collection: ResearchController()) app.get("health") { _ in HealthResponse(ok: true) diff --git a/services/backend/Tests/AppTests/ResearchTests.swift b/services/backend/Tests/AppTests/ResearchTests.swift new file mode 100644 index 0000000..02a6cee --- /dev/null +++ b/services/backend/Tests/AppTests/ResearchTests.swift @@ -0,0 +1,403 @@ +@testable import App +import Foundation +import Testing +import Vapor +import VaporTesting + +@Suite("Research") +struct ResearchTests { + @Test("Semble aggregation keeps current own records and applies link removals") + func sembleAggregation() async throws { + try await withApp(configure: configure) { app in + let did = "did:plc:research-semble" + let collectionURI = researchURI(did, "network.cosmik.collection", "ideas") + let olderCollectionURI = researchURI(did, "network.cosmik.collection", "older") + let urlCardURI = researchURI(did, "network.cosmik.card", "url-card") + let noteCardURI = researchURI(did, "network.cosmik.card", "note-card") + let removedCardURI = researchURI(did, "network.cosmik.card", "removed-card") + let removedLinkURI = researchURI(did, "network.cosmik.collectionLink", "removed-link") + let lister = StubResearchRecordLister(pages: [ + researchPageKey("network.cosmik.collection"): ListRecordsResponse(records: [ + researchRecord(uri: olderCollectionURI, value: [ + "$type": .string("network.cosmik.collection"), + "name": .string("Older ideas"), + "accessType": .string("OPEN"), + "createdAt": .string("2026-07-01T00:00:00Z"), + ]), + researchRecord(uri: collectionURI, value: [ + "$type": .string("network.cosmik.collection"), + "name": .string("Draft ideas"), + "description": .string("Things to explore"), + "accessType": .string("CLOSED"), + "createdAt": .string("2026-08-01T00:00:00Z"), + "updatedAt": .string("2026-08-02T00:00:00Z"), + "ignoredExtension": .bool(true), + ]), + researchRecord( + uri: researchURI(did, "network.cosmik.collection", "malformed"), + value: ["accessType": .string("OPEN")] + ), + researchRecord( + uri: researchURI("did:plc:someone-else", "network.cosmik.collection", "other"), + value: ["name": .string("Other"), "accessType": .string("OPEN")] + ), + ], cursor: nil), + researchPageKey("network.cosmik.card"): ListRecordsResponse(records: [ + researchRecord(uri: urlCardURI, value: [ + "$type": .string("network.cosmik.card"), + "type": .string("URL"), + "content": .object([ + "$type": .string("network.cosmik.card#urlContent"), + "url": .string("https://example.com/article"), + "metadata": .object([ + "$type": .string("network.cosmik.card#urlMetadata"), + "title": .string("An article"), + "description": .string("Useful context"), + "siteName": .string("Example"), + "author": .string("A. Writer"), + ]), + ]), + "createdAt": .string("2026-08-03T00:00:00Z"), + ]), + researchRecord(uri: noteCardURI, value: [ + "type": .string("NOTE"), + "url": .string("https://example.com/note-source"), + "content": .object([ + "$type": .string("network.cosmik.card#noteContent"), + "text": .string("A connection worth developing"), + ]), + "createdAt": .string("2026-08-04T00:00:00Z"), + ]), + researchRecord(uri: removedCardURI, value: [ + "type": .string("URL"), + "content": .object(["url": .string("https://example.com/removed")]), + ]), + ], cursor: nil), + researchPageKey("network.cosmik.collectionLink"): ListRecordsResponse(records: [ + researchCollectionLink( + uri: researchURI(did, "network.cosmik.collectionLink", "url-link"), + collectionURI: collectionURI, + cardURI: urlCardURI, + did: did + ), + researchCollectionLink( + uri: researchURI(did, "network.cosmik.collectionLink", "note-link"), + collectionURI: collectionURI, + cardURI: noteCardURI, + did: did + ), + researchCollectionLink( + uri: removedLinkURI, + collectionURI: collectionURI, + cardURI: removedCardURI, + did: did + ), + ], cursor: nil), + researchPageKey("network.cosmik.collectionLinkRemoval"): ListRecordsResponse(records: [ + researchRecord( + uri: researchURI(did, "network.cosmik.collectionLinkRemoval", "removal"), + value: [ + "$type": .string("network.cosmik.collectionLinkRemoval"), + "collection": researchStrongRef(collectionURI), + "removedLink": researchStrongRef(removedLinkURI), + "removedAt": .string("2026-08-05T00:00:00Z"), + ] + ), + ], cursor: nil), + ]) + let request = Request(application: app, on: app.eventLoopGroup.next()) + + let collections = try await ResearchService(records: lister).fetchSemble( + account: researchAccount(did: did), + req: request + ) + + let collection = try #require(collections.first) + #expect(collections.map(\.uri) == [collectionURI, olderCollectionURI]) + #expect(collection.uri == collectionURI) + #expect(collection.name == "Draft ideas") + #expect(collection.description == "Things to explore") + #expect(collection.accessType == "CLOSED") + #expect(collection.createdAt == "2026-08-01T00:00:00Z") + #expect(collection.updatedAt == "2026-08-02T00:00:00Z") + #expect(collection.cards.map(\.uri) == [noteCardURI, urlCardURI]) + #expect(collection.cards[0].note == "A connection worth developing") + #expect(collection.cards[0].url == "https://example.com/note-source") + #expect(collection.cards[1].url == "https://example.com/article") + #expect(collection.cards[1].title == "An article") + #expect(collection.cards[1].description == "Useful context") + #expect(collection.cards[1].siteName == "Example") + #expect(collection.cards[1].author == "A. Writer") + } + } + + @Test("Margin aggregation maps the unified note schema and skips malformed or other-user records") + func marginAggregation() async throws { + try await withApp(configure: configure) { app in + let did = "did:plc:research-margin" + let uri = researchURI(did, "at.margin.note", "annotation") + let olderURI = researchURI(did, "at.margin.note", "older") + let lister = StubResearchRecordLister(pages: [ + researchPageKey("at.margin.note"): ListRecordsResponse(records: [ + researchRecord(uri: olderURI, value: [ + "motivation": .string("bookmarking"), + "target": .object(["source": .string("https://example.com/older")]), + "createdAt": .string("2026-07-01T00:00:00Z"), + ]), + researchRecord(uri: uri, value: [ + "$type": .string("at.margin.note"), + "motivation": .string("highlighting"), + "target": .object([ + "source": .string("https://example.com/essay"), + "title": .string("An essay"), + "selector": .object([ + "type": .string("TextQuoteSelector"), + "exact": .string("A precise passage"), + "prefix": .string("before"), + "suffix": .string("after"), + ]), + ]), + "body": .object([ + "value": .string("This could become a section."), + "format": .string("text/plain"), + ]), + "tags": .array([.string("draft"), .string("research")]), + "color": .string("yellow"), + "createdAt": .string("2026-08-06T00:00:00Z"), + "modifiedAt": .string("2026-08-07T00:00:00Z"), + ]), + researchRecord( + uri: researchURI(did, "at.margin.note", "malformed"), + value: ["motivation": .string("commenting")] + ), + researchRecord( + uri: researchURI("did:plc:someone-else", "at.margin.note", "other"), + value: [ + "motivation": .string("commenting"), + "target": .object(["source": .string("https://other.example")]), + "createdAt": .string("2026-08-01T00:00:00Z"), + ] + ), + ], cursor: nil), + ]) + let request = Request(application: app, on: app.eventLoopGroup.next()) + + let annotations = try await ResearchService(records: lister).fetchMargin( + account: researchAccount(did: did), + req: request + ) + + let annotation = try #require(annotations.first) + #expect(annotations.map(\.uri) == [uri, olderURI]) + #expect(annotation.uri == uri) + #expect(annotation.motivation == "highlighting") + #expect(annotation.source == "https://example.com/essay") + #expect(annotation.title == "An essay") + #expect(annotation.body == "This could become a section.") + #expect(annotation.quote == "A precise passage") + #expect(annotation.tags == ["draft", "research"]) + #expect(annotation.color == "yellow") + #expect(annotation.createdAt == "2026-08-06T00:00:00Z") + #expect(annotation.modifiedAt == "2026-08-07T00:00:00Z") + } + } + + @Test("Generic paginator reads all pages and rejects cursor cycles") + func paginationAndCycleSafety() async throws { + try await withApp(configure: configure) { app in + let did = "did:plc:research-pages" + let first = researchRecord( + uri: researchURI(did, "at.margin.note", "first"), + value: ["value": .string("first")] + ) + let second = researchRecord( + uri: researchURI(did, "at.margin.note", "second"), + value: ["value": .string("second")] + ) + let lister = StubResearchRecordLister(pages: [ + researchPageKey("at.margin.note"): ListRecordsResponse(records: [first], cursor: "two"), + researchPageKey("at.margin.note", cursor: "two"): ListRecordsResponse(records: [second], cursor: nil), + ]) + + let records = try await RepositoryRecordPaginator(records: lister).listAll( + account: researchAccount(did: did), + collection: "at.margin.note", + client: app.client + ) + #expect(records.map(\.uri) == [first.uri, second.uri]) + + let cycling = StubResearchRecordLister(pages: [ + researchPageKey("network.cosmik.card"): ListRecordsResponse(records: [first], cursor: "same"), + researchPageKey("network.cosmik.card", cursor: "same"): ListRecordsResponse(records: [second], cursor: "same"), + ]) + await #expect(throws: (any Error).self) { + try await RepositoryRecordPaginator(records: cycling).listAll( + account: researchAccount(did: did), + collection: "network.cosmik.card", + client: app.client + ) + } + } + } + + @Test("Source failures are isolated and legacy Margin collections are not queried") + func partialSourceFailure() async throws { + try await withApp(configure: configure) { app in + let did = "did:plc:research-partial" + let marginURI = researchURI(did, "at.margin.note", "available") + let lister = StubResearchRecordLister( + pages: [ + researchPageKey("at.margin.note"): ListRecordsResponse(records: [ + researchRecord(uri: marginURI, value: [ + "motivation": .string("bookmarking"), + "target": .object(["source": .string("https://available.example")]), + "createdAt": .string("2026-08-08T00:00:00Z"), + ]), + ], cursor: nil), + ], + failingCollections: ["network.cosmik.collection"] + ) + let request = Request(application: app, on: app.eventLoopGroup.next()) + + let response = await ResearchService(records: lister).load( + account: researchAccount(did: did), + req: request + ) + + #expect(response.semble.collections.isEmpty) + #expect(response.semble.error != nil) + #expect(response.margin.error == nil) + #expect(response.margin.annotations.map(\.uri) == [marginURI]) + let requested = Set(await lister.requestedCollections()) + let currentCollections: Set = [ + "network.cosmik.collection", + "network.cosmik.collectionLink", + "network.cosmik.collectionLinkRemoval", + "network.cosmik.card", + "at.margin.note", + ] + #expect(requested.contains("network.cosmik.collection")) + #expect(requested.contains("at.margin.note")) + #expect(requested.isSubset(of: currentCollections)) + } + } + + @Test("Research endpoint requires the browser session and uses its linked account") + func authenticatedEndpoint() async throws { + try await withApp(configure: configure) { app in + let did = "did:plc:research-endpoint" + let cookie = try await authenticatedCookie(for: did, app: app) + app.research = FixedResearchLoader(expectedDID: did) + + try await app.testing().test(.GET, "/api/research") { _ in + } afterResponse: { response in + #expect(response.status == .unauthorized) + } + + try await app.testing().test(.GET, "/api/research") { request in + request.headers.replaceOrAdd(name: .cookie, value: cookie) + } afterResponse: { response in + #expect(response.status == .ok) + expectContent(ResearchResponse.self, response) { research in + #expect(research.semble.collections.first?.name == "Own research") + #expect(research.margin.annotations.isEmpty) + } + } + } + } +} + +private actor StubResearchRecordLister: RepositoryRecordPageListing { + let pages: [String: ListRecordsResponse] + let failingCollections: Set + private var requested: [String] = [] + + init( + pages: [String: ListRecordsResponse], + failingCollections: Set = [] + ) { + self.pages = pages + self.failingCollections = failingCollections + } + + func listRecordsPage( + account: LinkedAccount, + collection: String, + cursor: String?, + client: Client + ) async throws -> ListRecordsResponse { + requested.append(collection) + if failingCollections.contains(collection) { + throw Abort(.badGateway, reason: "Stubbed \(collection) failure") + } + return pages[researchPageKey(collection, cursor: cursor)] ?? ListRecordsResponse(records: [], cursor: nil) + } + + func requestedCollections() -> [String] { requested } +} + +private struct FixedResearchLoader: ResearchLoading { + let expectedDID: String + + func load(account: LinkedAccount, req: Request) async -> ResearchResponse { + ResearchResponse( + semble: SembleResearchResponse(collections: [ + SembleCollectionResponse( + uri: researchURI(expectedDID, "network.cosmik.collection", "own"), + name: account.did == expectedDID ? "Own research" : "Wrong account", + description: nil, + accessType: "CLOSED", + createdAt: nil, + updatedAt: nil, + cards: [] + ), + ], error: nil), + margin: MarginResearchResponse(annotations: [], error: nil) + ) + } +} + +private func researchAccount(did: String) -> LinkedAccount { + LinkedAccount( + did: did, + handle: "researcher.example", + pdsURL: "https://pds.example", + scope: "atproto", + accessToken: "plain:access", + refreshToken: "plain:refresh" + ) +} + +private func researchURI(_ did: String, _ collection: String, _ rkey: String) -> String { + "at://\(did)/\(collection)/\(rkey)" +} + +private func researchPageKey(_ collection: String, cursor: String? = nil) -> String { + "\(collection)|\(cursor ?? "")" +} + +private func researchRecord( + uri: String, + value: [String: JSONValue] +) -> RepositoryRecord { + RepositoryRecord(uri: uri, cid: "test-cid", value: .object(value)) +} + +private func researchStrongRef(_ uri: String) -> JSONValue { + .object(["uri": .string(uri), "cid": .string("test-cid")]) +} + +private func researchCollectionLink( + uri: String, + collectionURI: String, + cardURI: String, + did: String +) -> RepositoryRecord { + researchRecord(uri: uri, value: [ + "$type": .string("network.cosmik.collectionLink"), + "collection": researchStrongRef(collectionURI), + "card": researchStrongRef(cardURI), + "addedBy": .string(did), + "addedAt": .string("2026-08-05T00:00:00Z"), + ]) +}