release: test → main (#741 social scrape posts depth)

lib/apify/__tests__/scrapeProfileUrl.test.ts

@@ -26,9 +26,13 @@ beforeEach(() => {
 describe("scrapeProfileUrl", () => {
   it("dispatches linkedin.com to the LinkedIn scraper", async () => {
     const r = await scrapeProfileUrl("https://www.linkedin.com/in/drew-thurlow", "drew-thurlow");
-    expect(li).toHaveBeenCalledWith("drew-thurlow");
+    expect(li).toHaveBeenCalledWith("drew-thurlow", undefined);
     expect(r).toMatchObject({ supported: true, runId: "li-run", datasetId: "li-ds" });
   });
+  it("forwards posts to the platform scraper", async () => {
+    await scrapeProfileUrl("https://www.linkedin.com/in/drew-thurlow", "drew-thurlow", 20);
+    expect(li).toHaveBeenCalledWith("drew-thurlow", 20);
+  });
   it("returns null for an unsupported platform (spotify)", async () => {
     const r = await scrapeProfileUrl("https://open.spotify.com/artist/x", "x");
     expect(r).toBeNull();

lib/apify/__tests__/scrapeProfileUrlBatch.test.ts

@@ -0,0 +1,35 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+import { scrapeProfileUrlBatch } from "@/lib/apify/scrapeProfileUrlBatch";
+import { scrapeProfileUrl } from "@/lib/apify/scrapeProfileUrl";
+
+vi.mock("@/lib/apify/scrapeProfileUrl", () => ({ scrapeProfileUrl: vi.fn() }));
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  vi.mocked(scrapeProfileUrl).mockResolvedValue({
+    runId: "r1",
+    datasetId: "d1",
+    error: null,
+    supported: true,
+  });
+});
+
+describe("scrapeProfileUrlBatch", () => {
+  it("forwards posts to every scrapeProfileUrl call", async () => {
+    await scrapeProfileUrlBatch(
+      [
+        { profileUrl: "https://x.com/a", username: "a" },
+        { profileUrl: "https://youtube.com/@b", username: "b" },
+      ],
+      20,
+    );
+    expect(scrapeProfileUrl).toHaveBeenCalledWith("https://x.com/a", "a", 20);
+    expect(scrapeProfileUrl).toHaveBeenCalledWith("https://youtube.com/@b", "b", 20);
+  });
+
+  it("passes posts as undefined when omitted (legacy behavior)", async () => {
+    await scrapeProfileUrlBatch([{ profileUrl: "https://x.com/a", username: "a" }]);
+    expect(scrapeProfileUrl).toHaveBeenCalledWith("https://x.com/a", "a", undefined);
+  });
+});

lib/apify/scrapeProfileUrl.ts

@@ -7,7 +7,10 @@ import startFacebookProfileScraping from "@/lib/apify/facebook/startFacebookProf
 import startLinkedinProfileScraping from "@/lib/apify/linkedin/startLinkedinProfileScraping";
 import { getUsernameFromProfileUrl } from "@/lib/socials/getUsernameFromProfileUrl";
 
-type ScrapeRunner = (handle: string) => Promise<{
+type ScrapeRunner = (
+  handle: string,
+  posts?: number,
+) => Promise<{
   runId: string;
   datasetId: string;
   error?: string;
@@ -61,6 +64,7 @@ const PLATFORM_SCRAPERS: Array<{
 export const scrapeProfileUrl = async (
   profileUrl: string | null | undefined,
   username: string,
+  posts?: number,
 ): Promise<ScrapeProfileResult | null> => {
   if (!profileUrl) {
     return null;
@@ -76,7 +80,7 @@ export const scrapeProfileUrl = async (
   const finalUsername = username || getUsernameFromProfileUrl(profileUrl);
 
   try {
-    const result = await platform.scraper(finalUsername);
+    const result = await platform.scraper(finalUsername, posts);
 
     if (!result) {
       return {

lib/apify/scrapeProfileUrlBatch.ts

@@ -7,9 +7,12 @@ type ScrapeProfileUrlBatchInput = {
 
 export const scrapeProfileUrlBatch = async (
   inputs: ScrapeProfileUrlBatchInput[],
+  posts?: number,
 ): Promise<ProfileScrapeResult[]> => {
   const results = await Promise.all(
-    inputs.map(({ profileUrl, username }) => scrapeProfileUrl(profileUrl ?? null, username ?? "")),
+    inputs.map(({ profileUrl, username }) =>
+      scrapeProfileUrl(profileUrl ?? null, username ?? "", posts),
+    ),
   );
 
   return results

lib/apify/twitter/__tests__/startTwitterProfileScraping.test.ts

@@ -0,0 +1,36 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import apifyClient from "@/lib/apify/client";
+import startTwitterProfileScraping from "@/lib/apify/twitter/startTwitterProfileScraping";
+
+const start = vi.fn();
+vi.mock("@/lib/apify/client", () => ({ default: { actor: vi.fn(() => ({ start })) } }));
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  start.mockResolvedValue({ id: "run-1", defaultDatasetId: "ds-1", status: "RUNNING" });
+});
+
+describe("startTwitterProfileScraping", () => {
+  it("defaults to the legacy single-item snapshot (maxItems: 1) when posts is omitted", async () => {
+    const r = await startTwitterProfileScraping("sweetman_eth");
+    expect(apifyClient.actor).toHaveBeenCalledWith("apidojo/twitter-scraper-lite");
+    expect(start).toHaveBeenCalledWith(
+      { twitterHandles: ["sweetman_eth"], sort: "Latest", maxItems: 1 },
+      { webhooks: expect.any(Array) },
+    );
+    expect(r).toEqual({ runId: "run-1", datasetId: "ds-1" });
+  });
+
+  it("passes posts through as maxItems", async () => {
+    await startTwitterProfileScraping("sweetman_eth", 20);
+    expect(start).toHaveBeenCalledWith(
+      { twitterHandles: ["sweetman_eth"], sort: "Latest", maxItems: 20 },
+      { webhooks: expect.any(Array) },
+    );
+  });
+
+  it("throws on an empty handle", async () => {
+    await expect(startTwitterProfileScraping("  ")).rejects.toThrow(/Invalid Twitter handle/);
+    expect(start).not.toHaveBeenCalled();
+  });
+});

lib/apify/twitter/startTwitterProfileScraping.ts

@@ -2,7 +2,10 @@ import apifyClient from "@/lib/apify/client";
 import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks";
 import { ApifyRunInfo } from "@/lib/apify/types";
 
-const startTwitterProfileScraping = async (handle: string): Promise<ApifyRunInfo | null> => {
+const startTwitterProfileScraping = async (
+  handle: string,
+  posts?: number,
+): Promise<ApifyRunInfo | null> => {
   const cleanHandle = handle.trim();
 
   if (!cleanHandle) {
@@ -12,7 +15,7 @@ const startTwitterProfileScraping = async (handle: string): Promise<ApifyRunInfo
   const input = {
     twitterHandles: [cleanHandle],
     sort: "Latest",
-    maxItems: 1,
+    maxItems: posts ?? 1,
   };
 
   const run = await apifyClient

lib/apify/youtube/__tests__/startYoutubeProfileScraping.test.ts

@@ -0,0 +1,44 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import apifyClient from "@/lib/apify/client";
+import startYoutubeProfileScraping from "@/lib/apify/youtube/startYoutubeProfileScraping";
+
+const start = vi.fn();
+vi.mock("@/lib/apify/client", () => ({ default: { actor: vi.fn(() => ({ start })) } }));
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  start.mockResolvedValue({ id: "run-1", defaultDatasetId: "ds-1", status: "RUNNING" });
+});
+
+describe("startYoutubeProfileScraping", () => {
+  it("defaults to the legacy snapshot (maxResults: 1, Shorts excluded) when posts is omitted", async () => {
+    const r = await startYoutubeProfileScraping("@mycowtf");
+    expect(apifyClient.actor).toHaveBeenCalledWith("streamers/youtube-scraper");
+    expect(start).toHaveBeenCalledWith(
+      expect.objectContaining({
+        startUrls: [{ url: "https://www.youtube.com/@mycowtf" }],
+        maxResults: 1,
+        maxResultsShorts: 0,
+      }),
+      { webhooks: expect.any(Array) },
+    );
+    expect(r).toEqual({ runId: "run-1", datasetId: "ds-1" });
+  });
+
+  it("passes posts through as maxResults AND maxResultsShorts (Shorts included)", async () => {
+    await startYoutubeProfileScraping("mycowtf", 20);
+    expect(start).toHaveBeenCalledWith(
+      expect.objectContaining({
+        startUrls: [{ url: "https://www.youtube.com/@mycowtf" }],
+        maxResults: 20,
+        maxResultsShorts: 20,
+      }),
+      { webhooks: expect.any(Array) },
+    );
+  });
+
+  it("throws on an empty handle", async () => {
+    await expect(startYoutubeProfileScraping("  ")).rejects.toThrow(/Invalid YouTube handle/);
+    expect(start).not.toHaveBeenCalled();
+  });
+});

lib/apify/youtube/startYoutubeProfileScraping.ts

@@ -24,7 +24,10 @@ const DEFAULT_INPUT = {
   subtitlesFormat: "srt",
 };
 
-const startYoutubeProfileScraping = async (handle: string): Promise<ApifyRunInfo | null> => {
+const startYoutubeProfileScraping = async (
+  handle: string,
+  posts?: number,
+): Promise<ApifyRunInfo | null> => {
   const cleanHandle = handle.trim().replace(/^@/, "");
 
   if (!cleanHandle) {
@@ -36,6 +39,9 @@ const startYoutubeProfileScraping = async (handle: string): Promise<ApifyRunInfo
   const input = {
     ...DEFAULT_INPUT,
     startUrls: [{ url: targetUrl }],
+    // The legacy snapshot excludes Shorts; a requested posts depth includes them.
+    maxResults: posts ?? DEFAULT_INPUT.maxResults,
+    maxResultsShorts: posts ?? DEFAULT_INPUT.maxResultsShorts,
   };
 
   const run = await apifyClient

lib/artist/__tests__/postArtistSocialsScrapeHandler.test.ts

@@ -0,0 +1,63 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { NextRequest } from "next/server";
+
+import { postArtistSocialsScrapeHandler } from "../postArtistSocialsScrapeHandler";
+import { scrapeProfileUrlBatch } from "@/lib/apify/scrapeProfileUrlBatch";
+import { selectAccountSocials } from "@/lib/supabase/account_socials/selectAccountSocials";
+
+vi.mock("@/lib/apify/scrapeProfileUrlBatch", () => ({ scrapeProfileUrlBatch: vi.fn() }));
+vi.mock("@/lib/supabase/account_socials/selectAccountSocials", () => ({
+  selectAccountSocials: vi.fn(),
+}));
+vi.mock("@/lib/networking/getCorsHeaders", () => ({ getCorsHeaders: () => ({}) }));
+
+const ARTIST_ID = "660e8400-e29b-41d4-a716-446655440000";
+
+const makeRequest = (body: unknown) =>
+  new NextRequest("http://localhost/api/artist/socials/scrape", {
+    method: "POST",
+    body: JSON.stringify(body),
+  });
+
+describe("postArtistSocialsScrapeHandler", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(selectAccountSocials).mockResolvedValue([
+      { social: { profile_url: "https://x.com/a", username: "a" } } as never,
+    ]);
+    vi.mocked(scrapeProfileUrlBatch).mockResolvedValue([
+      { runId: "r1", datasetId: "d1", error: null },
+    ]);
+  });
+
+  it("returns 400 when artist_account_id is missing", async () => {
+    const res = await postArtistSocialsScrapeHandler(makeRequest({}));
+    expect(res.status).toBe(400);
+    expect(scrapeProfileUrlBatch).not.toHaveBeenCalled();
+  });
+
+  it("scrapes without a posts depth by default (legacy behavior)", async () => {
+    const res = await postArtistSocialsScrapeHandler(makeRequest({ artist_account_id: ARTIST_ID }));
+    expect(res.status).toBe(200);
+    expect(scrapeProfileUrlBatch).toHaveBeenCalledWith(
+      [{ profileUrl: "https://x.com/a", username: "a" }],
+      undefined,
+    );
+  });
+
+  it("forwards posts to scrapeProfileUrlBatch", async () => {
+    await postArtistSocialsScrapeHandler(makeRequest({ artist_account_id: ARTIST_ID, posts: 20 }));
+    expect(scrapeProfileUrlBatch).toHaveBeenCalledWith(
+      [{ profileUrl: "https://x.com/a", username: "a" }],
+      20,
+    );
+  });
+
+  it("returns [] when the artist has no socials", async () => {
+    vi.mocked(selectAccountSocials).mockResolvedValue([]);
+    const res = await postArtistSocialsScrapeHandler(makeRequest({ artist_account_id: ARTIST_ID }));
+    expect(res.status).toBe(200);
+    expect(await res.json()).toEqual([]);
+    expect(scrapeProfileUrlBatch).not.toHaveBeenCalled();
+  });
+});

lib/artist/__tests__/validateArtistSocialsScrapeBody.test.ts

@@ -0,0 +1,36 @@
+import { describe, it, expect, vi } from "vitest";
+import { NextResponse } from "next/server";
+
+import { validateArtistSocialsScrapeBody } from "../validateArtistSocialsScrapeBody";
+
+vi.mock("@/lib/networking/getCorsHeaders", () => ({ getCorsHeaders: () => ({}) }));
+
+const ARTIST_ID = "660e8400-e29b-41d4-a716-446655440000";
+
+describe("validateArtistSocialsScrapeBody", () => {
+  it("returns 400 when artist_account_id is missing", () => {
+    const res = validateArtistSocialsScrapeBody({}) as NextResponse;
+    expect(res.status).toBe(400);
+  });
+
+  it("accepts a body without posts (legacy behavior)", () => {
+    expect(validateArtistSocialsScrapeBody({ artist_account_id: ARTIST_ID })).toEqual({
+      artist_account_id: ARTIST_ID,
+    });
+  });
+
+  it("accepts a valid posts field", () => {
+    expect(validateArtistSocialsScrapeBody({ artist_account_id: ARTIST_ID, posts: 20 })).toEqual({
+      artist_account_id: ARTIST_ID,
+      posts: 20,
+    });
+  });
+
+  it.each([[0], [101], [1.5], ["20"]])("returns 400 for invalid posts %s", posts => {
+    const res = validateArtistSocialsScrapeBody({
+      artist_account_id: ARTIST_ID,
+      posts,
+    }) as NextResponse;
+    expect(res.status).toBe(400);
+  });
+});

lib/artist/postArtistSocialsScrapeHandler.ts

@@ -36,6 +36,7 @@ export async function postArtistSocialsScrapeHandler(request: NextRequest): Prom
         profileUrl: social.social?.profile_url,
         username: social.social?.username,
       })),
+      validatedBody.posts,
     );
 
     return NextResponse.json(results, {

lib/artist/validateArtistSocialsScrapeBody.ts

@@ -4,6 +4,12 @@ import { z } from "zod";
 
 export const artistSocialsScrapeBodySchema = z.object({
   artist_account_id: z.string().min(1, "artist_account_id body parameter is required"),
+  posts: z
+    .number()
+    .int("posts must be an integer")
+    .min(1, "posts must be between 1 and 100")
+    .max(100, "posts must be between 1 and 100")
+    .optional(),
 });
 
 export type ArtistSocialsScrapeBody = z.infer<typeof artistSocialsScrapeBodySchema>;

lib/socials/__tests__/postSocialScrapeHandler.test.ts

@@ -45,6 +45,17 @@ describe("postSocialScrapeHandler", () => {
     const res = await postSocialScrapeHandler(request, SOCIAL_ID);
     expect(res.status).toBe(200);
     expect(await res.json()).toEqual({ runId: "r1", datasetId: "d1" });
+    expect(scrapeProfileUrl).toHaveBeenCalledWith(social.profile_url, social.username, undefined);
+  });
+
+  it("forwards validated posts to scrapeProfileUrl", async () => {
+    vi.mocked(validatePostSocialScrapeRequest).mockResolvedValue({
+      social_id: SOCIAL_ID,
+      posts: 20,
+    });
+    vi.mocked(scrapeProfileUrl).mockResolvedValue({ runId: "r1", datasetId: "d1" } as never);
+    await postSocialScrapeHandler(request, SOCIAL_ID);
+    expect(scrapeProfileUrl).toHaveBeenCalledWith(social.profile_url, social.username, 20);
   });
 
   it("returns 400 when platform unsupported (scrapeProfileUrl returns null)", async () => {

lib/socials/__tests__/validatePostSocialScrapeRequest.test.ts

@@ -73,9 +73,34 @@ describe("validatePostSocialScrapeRequest", () => {
   it("returns validated payload when caller has access to an owning artist", async () => {
     expect(await validatePostSocialScrapeRequest(makeRequest(), SOCIAL_ID)).toEqual({
       social_id: SOCIAL_ID,
+      posts: undefined,
     });
   });
 
+  it("parses a valid posts query param", async () => {
+    const req = new NextRequest(`http://localhost/api/socials/${SOCIAL_ID}/scrape?posts=20`, {
+      method: "POST",
+      headers: { "x-api-key": "k" },
+    });
+    expect(await validatePostSocialScrapeRequest(req, SOCIAL_ID)).toEqual({
+      social_id: SOCIAL_ID,
+      posts: 20,
+    });
+  });
+
+  it.each([["0"], ["101"], ["abc"], ["1.5"]])(
+    "returns 400 for invalid posts query param %s",
+    async posts => {
+      const req = new NextRequest(
+        `http://localhost/api/socials/${SOCIAL_ID}/scrape?posts=${posts}`,
+        { method: "POST", headers: { "x-api-key": "k" } },
+      );
+      const res = (await validatePostSocialScrapeRequest(req, SOCIAL_ID)) as NextResponse;
+      expect(res.status).toBe(400);
+      expect(validateAuthContext).not.toHaveBeenCalled();
+    },
+  );
+
   it("propagates DB error from selectAccountSocials (fails closed as 500)", async () => {
     vi.mocked(selectAccountSocials).mockRejectedValue(new Error("db blew up"));
     await expect(validatePostSocialScrapeRequest(makeRequest(), SOCIAL_ID)).rejects.toThrow(