Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/plans/rfc-official-web-page-sources.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@ Example:
`<h5>`, so the heading selector covers `h5`/`h6`, and they repeat category and author as short
unpunctuated paragraphs, which `isDescription` drops: a card without a description reports no
summary instead of one built from its labels.
- Claude Resources: `https://claude.com/resources/articles`, entries under `/resources/articles/`.
Cards print the date as plain text (`Nov 10, 2026`) in a `<span>` and use no `<time>` element at
all, so extraction falls back to the first descendant whose entire text is one calendar date. A
headline or sentence that merely mentions a date does not count, and a card that declares no date
is still skipped rather than given an invented one.

These listing routes allow automated access in their robots policies. Production-client and CLI
dry-run checks cover dated links, canonical URLs and source health. Blog card titles omit author and
Expand Down
7 changes: 7 additions & 0 deletions industry-feeds.json
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,13 @@
"pathPrefixes": ["/blog/"],
"tags": ["Articles", "Design", "Frontend"]
},
{
"type": "page",
"name": "Claude Resources",
"url": "https://claude.com/resources/articles",
"pathPrefixes": ["/resources/articles/"],
"tags": ["Articles", "AI"]
},
{
"name": "Hugging Face Blog",
"url": "https://huggingface.co/blog/feed.xml",
Expand Down
22 changes: 21 additions & 1 deletion src/integrations/web-page/dto.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ function normalizedDate(value: string | undefined): string | undefined {
}

const SENTENCE_END = /[.!?。!?…]["'”’)\]]*$/u;
/** A date a card prints as plain text, e.g. `Nov 10, 2026` or `2026-11-10`. */
const BARE_DATE = /^(?:\d{4}-\d{2}-\d{2}|[A-Z][a-z]{2} \d{1,2}, \d{4})$/u;
/** Longer than a category or author label, so a short card stays description-less. */
const MIN_DESCRIPTION_WORDS = 4;

Expand Down Expand Up @@ -121,7 +123,25 @@ export function extractWebPageEntries(html: string, source: WebPageSubscription)

const current = entries.get(url) ?? { titleScore: 0 };
const time = element.find("time").first();
const publishedAt = normalizedDate(time.attr("datetime") ?? cleanText(time.text()));
let publishedAt = normalizedDate(time.attr("datetime") ?? cleanText(time.text()));
if (!publishedAt) {
// Some cards print a bare date in a plain element instead of a `<time>`. Only a descendant
// whose whole text is one calendar date counts, so a headline or sentence that mentions a
// date cannot become the publication date.
element.find("*").each((_, node) => {
if (publishedAt) {
return;
}
const candidate = $(node);
if (candidate.children().length > 0) {
return;
}
const text = cleanText(candidate.text());
if (text && BARE_DATE.test(text)) {
publishedAt = normalizedDate(text);
}
});
}
const titleElement = element
.find("h1, h2, h3, h4, h5, h6, [role='heading'], [class*='title'], [class*='headline']")
.first();
Expand Down
30 changes: 30 additions & 0 deletions src/integrations/web-page/mapper.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,36 @@ describe("web page sources", () => {
]);
});

it("reads a card that prints its date as plain text instead of a time element", () => {
const entries = extractWebPageEntries(
`<a href="/news/connectors">
<div><span>Article</span><span>Nov 10, 2026</span></div>
<h3>Your apps can now open in Claude</h3>
<p>Partners can now embed their apps directly in Claude, so both work in the same tool.</p>
</a>`,
source
);
expect(entries).toEqual([
{
url: "https://example.com/news/connectors",
title: "Your apps can now open in Claude",
publishedAt: "2026-11-10T12:00:00.000Z",
summary: "Partners can now embed their apps directly in Claude, so both work in the same tool."
}
]);
});

it("does not read a date out of a headline or a longer sentence", () => {
const entries = extractWebPageEntries(
`<a href="/news/no-date">
<h3>What changed on Nov 10, 2026</h3>
<p>This release note mentions Nov 10, 2026 without declaring the entry date.</p>
</a>`,
source
);
expect(entries).toEqual([]);
});

it("keeps the body of each changelog entry with its own dated links", () => {
const entries = extractWebPageEntries(
`<article>
Expand Down
Loading