Read every older project's site again, once, and re-judge its leads

scripts/reseed-profiles.ts

@@ -2,9 +2,11 @@
  * One-off for projects made before 2026-09-19 (PR #73): reads each product page
  * again, drops the "<platform> api" and "<platform> scraper" searches from a
  * product that does not sell platform data, writes the competitors the reading
- * names, fills the exclusions and not-buyers of a project that has none, and
- * queues the discovery that rebuilds the plan from them. No facts a person
- * edited are touched; a project whose lists were filled is judged again.
+ * names, replaces a profile nobody has touched with the new reading, fills the
+ * exclusions and not-buyers of an edited one that has none, and queues the
+ * discovery that rebuilds the plan. No facts a person edited are touched. The
+ * deployed app does the same from its own `profile_reseed` job at boot; this
+ * is for a database you can reach, and for --dry-run.
  *
  * The discovery jobs are spaced out, because they run on the same workers as a
  * new signup's first sweep. Run it only once the deployed app has PR #73: an
@@ -55,6 +57,7 @@ for (const row of chosen) {
         problemPhrasings: parseTextList(row.problemPhrasings),
         exclusions: parseTextList(row.exclusions),
         notBuyers: parseTextList(row.notBuyers),
+        profileVersion: row.profileVersion,
       },
       { dryRun },
     );
@@ -66,7 +69,7 @@ for (const row of chosen) {
       `${row.name}: sells platform data ${result.sellsPlatformData}, ` +
         `dropped ${result.droppedPhrasings.length} searches, ` +
         `competitors ${result.competitors.join(", ") || "none"}, ` +
-        `filled ${result.exclusions.length} exclusions and ${result.notBuyers.length} not-buyers`,
+        `${result.refreshed ? "profile replaced, " : ""}wrote ${result.exclusions.length} exclusions and ${result.notBuyers.length} not-buyers`,
     );
   } catch (error) {
     // One page that will not load is not a reason to leave the rest as they are.

src/jobs/registry.ts

@@ -4,6 +4,8 @@ import { jobs, projects } from "@/db/schema";
 import { CADENCE_MS } from "@/lib/alerts/select";
 import { runDiscoveryRefresh } from "@/lib/discovery/refresh";
 import { runInitialDiscovery } from "@/lib/discovery/initial";
+import { parseTextList } from "@/lib/discovery/store";
+import { reseedFromPage } from "@/lib/profile";
 import { deleteExpiredPosts } from "@/lib/retention";
 import { discoveryBudget } from "@/lib/discovery/run";
 import { runBackfill } from "@/lib/scan/backfill";
@@ -83,6 +85,38 @@ export const JOB_HANDLERS: Record<string, JobHandler> = {
     }
     await runRescore(job.projectId, job.id);
   },
+  /**
+   * One new reading of the site for a project whose profile an older prompt
+   * made, queued at boot and never again. When a fact changed, every verdict
+   * the project holds was made about a different product, so they are judged
+   * again; the text is already here, so that buys no Reddit data.
+   */
+  profile_reseed: async (job) => {
+    if (!job.projectId) {
+      throw new Error("A profile reseed needs a project");
+    }
+    const [project] = await db().select().from(projects).where(eq(projects.id, job.projectId));
+    if (!project?.url) {
+      return;
+    }
+    const result = await reseedFromPage({
+      id: project.id,
+      userId: project.userId,
+      url: project.url,
+      problemPhrasings: parseTextList(project.problemPhrasings),
+      exclusions: parseTextList(project.exclusions),
+      notBuyers: parseTextList(project.notBuyers),
+      profileVersion: project.profileVersion,
+    });
+    if (result.refreshed || result.exclusions.length > 0 || result.notBuyers.length > 0) {
+      await enqueueOnce("rescore", new Date(), project.id);
+    }
+    // Searches for a platform's API that the product never sold are in the plan
+    // itself, and only a new plan takes them out.
+    if (result.droppedPhrasings.length > 0) {
+      await enqueueOnce("discovery_initial", new Date(), project.id);
+    }
+  },
   discovery_refresh: async (job) => {
     if (!job.projectId) {
       throw new Error("A discovery refresh needs a project");

src/jobs/scheduler.ts

@@ -1,6 +1,7 @@
 import { Cron } from "croner";
+import { eq } from "drizzle-orm";
 import { db } from "@/db";
-import { projects } from "@/db/schema";
+import { jobs, projects } from "@/db/schema";
 import { config } from "@/lib/config";
 import { projectsWithStaleEvaluations } from "@/lib/scan/rescore";
 import { enqueueOnce, lastRunJob } from "./enqueue";
@@ -78,12 +79,25 @@ async function pump(workers: number, watchedWorkers: number): Promise<void> {
  */
 export async function seedProjectScans(): Promise<void> {
   const rows = await db()
-    .select({ id: projects.id, discoveredAt: projects.discoveredAt })
+    .select({
+      id: projects.id,
+      discoveredAt: projects.discoveredAt,
+      createdAt: projects.createdAt,
+      url: projects.url,
+    })
     .from(projects);
   const stale = await projectsWithStaleEvaluations();
+  const reseeded = new Set(
+    (
+      await db()
+        .selectDistinct({ projectId: jobs.projectId })
+        .from(jobs)
+        .where(eq(jobs.kind, "profile_reseed"))
+    ).map((row) => row.projectId),
+  );
   for (const row of rows) {
     try {
-      await seedProject(row, stale.has(row.id));
+      await seedProject(row, stale.has(row.id), reseeded.has(row.id));
     } catch (error) {
       /** The project was deleted between the read above and its insert. Nothing to seed. */
       if (!isMissingProject(error)) {
@@ -99,7 +113,19 @@ function isMissingProject(error: unknown): boolean {
   return (cause as { code?: string } | null)?.code === "23503";
 }
 
-async function seedProject(row: { id: string; discoveredAt: Date | null }, stale: boolean): Promise<void> {
+/**
+ * Profiles made before this were read from the homepage alone, by a prompt that
+ * asked for a product's limits only where the page spelled them out. Each gets
+ * one new reading, spread over a few hours so the scrapes and the rescores
+ * behind them never crowd a signup's first sweep.
+ */
+const PROFILES_READ_WHOLE_SINCE = new Date("2026-09-20T00:00:00Z");
+const RESEED_GAP_MS = 2 * 60 * 1000;
+let reseedsQueued = 0;
+
+type SeedRow = { id: string; discoveredAt: Date | null; createdAt: Date; url: string | null };
+
+async function seedProject(row: SeedRow, stale: boolean, reseeded: boolean): Promise<void> {
   if (!row.discoveredAt) {
     /**
      * A project whose first discovery never finished has no plan at all, so
@@ -122,6 +148,10 @@ async function seedProject(row: { id: string; discoveredAt: Date | null }, stale
   if (stale) {
     await enqueueOnce("rescore", new Date(), row.id);
   }
+  if (row.url && row.createdAt < PROFILES_READ_WHOLE_SINCE && !reseeded) {
+    await enqueueOnce("profile_reseed", new Date(Date.now() + reseedsQueued * RESEED_GAP_MS), row.id);
+    reseedsQueued += 1;
+  }
 }
 
 /**

src/lib/profile.ts

@@ -55,7 +55,9 @@ const readingSchema = profileSchema.extend({
 function flat(text: string): string {
   return text
     .toLowerCase()
-    .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
+    // The address may not hold a space: a page cut mid-link leaves "[text](https://a.com"
+    // open, and an address that ran on to the next ")" took a whole page of text with it.
+    .replace(/\[([^\]]*)\]\([^)\s]*\)/g, "$1")
     .replace(/[*_#`>|~\\]/g, "")
     .replace(/[\u2018\u2019]/g, "'")
     .replace(/[\u201C\u201D]/g, '"')
@@ -148,6 +150,12 @@ async function scrapeProduct(projectId: string, userId: string, url: string) {
 
 /** The pages that say what a homepage leaves out, in the order they are worth reading. */
 const SITE_PAGES = [/pric|plans/i, /feature|product|solution|how-it-works|services/i, /faq|help/i, /about/i];
+/**
+ * Where a site keeps writing about things rather than the thing it sells. A
+ * post titled "2024 popular javascript products" is not the product page its
+ * address looks like (ihatereading.in, 2026-09-19).
+ */
+const NOT_A_SITE_PAGE = /^(blogs?|posts?|news|articles?|stories|docs?|guides?|changelog|legal|privacy|terms|tag|category)$/i;
 const MAX_SITE_PAGES = 3;
 const HOME_CHARS = 12000;
 const PAGE_CHARS = 8000;
@@ -172,9 +180,11 @@ export function sitePageLinks(pageUrl: string, markdown: string): string[] {
     if (link.host !== home.host || path === home.pathname.replace(/\/$/, "") || !/^https?:$/.test(link.protocol)) {
       continue;
     }
+    const segments = path.split("/").filter(Boolean);
     const rank = SITE_PAGES.findIndex((pattern) => pattern.test(path));
     const key = `${link.origin}${path}`;
-    if (rank !== -1 && path.split("/").length <= 3 && !found.has(key)) {
+    const written = segments.some((segment) => NOT_A_SITE_PAGE.test(segment));
+    if (rank !== -1 && segments.length <= 2 && !written && !found.has(key)) {
       found.set(key, rank);
     }
   }
@@ -316,6 +326,8 @@ export type PageReseed = {
   sellsPlatformData: boolean;
   droppedPhrasings: string[];
   competitors: string[];
+  /** True when nobody had touched the profile, so the new reading replaced all of it. */
+  refreshed: boolean;
   /** The exclusions and not-buyers written, which is none when the project already had its own. */
   exclusions: string[];
   notBuyers: string[];
@@ -325,16 +337,16 @@ export type PageReseed = {
 const PLATFORM_PHRASING = / (?:api|scraper)$/;
 
 /**
- * Brings an older project up to what a new one gets, and touches nothing else.
- * The page is read again for what it was never asked: the platform searches are
- * dropped when the product does not sell its platforms' data, the competitors
- * the reading names are written, and a project with no exclusions or no
- * not-buyers gets the ones the page implies (before 2026-09-19 they were asked
- * for only where the page stated them, and 54 of 79 projects had none). A list
- * that holds anything is a person's or an earlier reading's and stays as it is.
- * Filling either one bumps the profile version, because a verdict made without
- * them is a verdict about a different product. The caller queues the discovery
- * that turns the corrected searches into a plan.
+ * Brings an older project up to what a new one gets. The site is read again:
+ * the platform searches are dropped when the product does not sell its
+ * platforms' data, and the competitors the reading names are written. A profile
+ * nobody has touched (still on its first version) is replaced whole by the new
+ * reading, because the old one was made from the homepage alone and asked for
+ * limits only where the page stated them (54 of 79 projects had none). A profile
+ * somebody edited or rebuilt keeps every fact it has, and only an empty list of
+ * exclusions or not-buyers is filled. Either way the profile version is bumped
+ * when a fact changed, because a verdict made without it is a verdict about a
+ * different product. Where and how buyers ask is left to the weekly refresh.
  */
 export async function reseedFromPage(
   project: {
@@ -344,6 +356,7 @@ export async function reseedFromPage(
     problemPhrasings: string[];
     exclusions: string[];
     notBuyers: string[];
+    profileVersion: number;
   },
   options: { dryRun?: boolean } = {},
 ): Promise<PageReseed> {
@@ -352,8 +365,9 @@ export async function reseedFromPage(
   const droppedPhrasings = profile.sellsPlatformData
     ? []
     : project.problemPhrasings.filter((item) => PLATFORM_PHRASING.test(item));
-  const exclusions = project.exclusions.length === 0 ? profile.exclusions : [];
-  const notBuyers = project.notBuyers.length === 0 ? profile.notBuyers : [];
+  const refreshed = project.profileVersion === 1 && profile.pain.trim() !== "";
+  const exclusions = refreshed || project.exclusions.length === 0 ? profile.exclusions : [];
+  const notBuyers = refreshed || project.notBuyers.length === 0 ? profile.notBuyers : [];
   const { limits } = await tierForUser(project.userId);
   const competitors = capped(
     pageCompetitors(profile.name, profile.competitors),
@@ -367,7 +381,22 @@ export async function reseedFromPage(
         .set({ problemPhrasings: project.problemPhrasings.filter((item) => !dropped.has(item)) })
         .where(eq(projects.id, project.id));
     }
-    if (exclusions.length > 0 || notBuyers.length > 0) {
+    if (refreshed) {
+      await db()
+        .update(projects)
+        .set({
+          pain: profile.pain,
+          solution: profile.solution,
+          targetUsers: profile.targetUsers,
+          geography: profile.serviceGeography || null,
+          budgetFit: profile.budgetFit,
+          capabilities: profile.capabilities,
+          exclusions,
+          notBuyers,
+          profileVersion: sql`${projects.profileVersion} + 1`,
+        })
+        .where(eq(projects.id, project.id));
+    } else if (exclusions.length > 0 || notBuyers.length > 0) {
       await db()
         .update(projects)
         .set({
@@ -383,6 +412,7 @@ export async function reseedFromPage(
     sellsPlatformData: profile.sellsPlatformData,
     droppedPhrasings,
     competitors: competitors.map((item) => item.name),
+    refreshed,
     exclusions,
     notBuyers,
   };

src/lib/prompts.ts

@@ -21,7 +21,7 @@ Describe only what the pages support. Use the page's own words wherever you can,
 - solution: what the product does about that, one sentence.
 - targetUsers: who buys it, one sentence.
 - capabilities: what the product does for its buyer, one short phrase each, each an outcome the buyer gets rather than the mechanism behind it ("see where the month's money went", not "upload a CSV"). Leave out what every product has - sign-in, billing, support, a free sample, a newsletter - and leave out the maker's biography and anything else on the page that is not the thing being sold.
-- exclusions: the limits the site itself states, each as { text, sourceText }: text is the limit in one short phrase, sourceText is the exact words on the site it rests on, copied character for character and no longer than fifteen words. Two kinds. What a buyer must already have or be for it to work at all: the device, system or account it runs on ("iPhone only", from "Download on the App Store" where no other store is offered), the country, language, currency or law it is built around, the smallest customer or price it starts at. And what it says in so many words it does not do. You are reading several of the site's pages, and a site sells things its homepage never mentions: a thing no page mentions is not a limit, a plan or a service on any page is something it does, and a free plan or a build the site says is coming means those people are its buyers. Leave the list empty rather than write a limit you cannot quote.
+- exclusions: the limits on who can use this product at all, each as { text, sourceText }: text is the limit in one short phrase, sourceText is the exact words on the site that show it, copied character for character and no longer than fifteen words. Look for each of these five, and write the ones the site shows. What a buyer must already have for it to work: the device or system it runs on ("iPhone only" from "Download on the App Store" when no other store is offered, "Mac and Windows desktop only"), and the platform, hardware or account it plugs into when it offers no other ("needs a Shopify store", "needs a Garmin watch", "needs your own Rithmic account"). The country, region or law it is built for: "Spain only" from a page about Spanish payroll tax, "US immigration filings only", "servers in Amsterdam only". The language: when the whole site is written in one language other than English and offers no other, "Spanish-speaking customers only", quoting any sentence of it. The currency it prices in when that is not US dollars: "prices in MXN". The smallest customer or price it starts at, when there is no free plan: "from $20 a month, no free plan", "bulk orders only, by quote". After those, anything the site says in so many words it does not do. Not a limit: what a cheaper plan leaves out, a usage cap, needing an account, needing to pay by card, or anything else true of most products. You are reading several of the site's pages, and a site sells things its homepage never mentions: a thing no page mentions is not a limit, a plan or a service on any page is something it does, and a free plan or a build the site says is coming means those people are its buyers. Leave the list empty rather than write a limit you cannot quote.
 - notBuyers: the kinds of person who share this product's vocabulary but would not buy it, each as { text, sourceText }, where sourceText is the exact words on the site that show it: who it says it is for, what it costs, how it is delivered. For something sold to businesses, the consumer on the other side of that market (a hotel guest, for hotel software); for a done-for-you service with no self-serve plan, the person who will only do it themselves; for a product whose cheapest plan has a price, the person who says they will pay nothing, unless the site has a free plan. Never someone a page of the site sells to or invites. Empty list when the site gives no ground.
 - serviceGeography: where the product itself works - the places it covers or operates in. This is not where its buyers live. Empty string when the page binds it to nowhere.
 - destinations: the individual places this product serves, each with the exact page text you read it from. Take them only from the page's own navigation links or body text. Never add a place the page does not name, however obvious it seems. Return an empty list when the page names none.