From c0770c07aae875d810a9bc8ce2f1291f5e1df7a2 Mon Sep 17 00:00:00 2001 From: Dan Date: Fri, 28 Aug 2026 12:49:33 +0300 Subject: [PATCH] Normalize MCP responses and Notes pagination --- README.md | 8 +- docs/mcp-analytics.md | 30 ++- package.json | 2 +- src/core/client.ts | 3 +- src/core/index.ts | 1 + src/core/types.ts | 7 +- src/mcp-server.ts | 507 +++++++++++++++++++++++++++++------ src/resources/notes/index.ts | 6 +- test/client.test.ts | 4 +- test/mcp.test.ts | 272 ++++++++++++++++--- 10 files changed, 703 insertions(+), 137 deletions(-) diff --git a/README.md b/README.md index 356fd26..b4e7c23 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Copy [`.dev.vars.example`](.dev.vars.example) to `.dev.vars` and replace the pla ## MCP server -The package includes a read-only STDIO MCP server for publication, post, Note, subscriber, and activity analytics. It exposes both bounded raw endpoint data and compact full-history summaries. Set `SUBSTACK_SESSION_TOKEN` and `SUBSTACK_PUBLICATION_URL`, then configure Codex: +The package includes a read-only STDIO MCP server for publication, post, Note, subscriber, and activity analytics. It returns compact, normalized model-facing data by default, with explicit raw-data opt-ins where supported. Set `SUBSTACK_SESSION_TOKEN` and `SUBSTACK_PUBLICATION_URL`, then configure Codex: ```toml [mcp_servers.substack] @@ -67,15 +67,15 @@ Keep the session token local and out of source control. All MCP tools are read-o | `get_publication_analytics` | Full-history totals, average upstream rates, audience/section/type breakdowns, top posts, and optional raw rows. | | `get_post_engagement` | Post content engagement and a bounded visible-comment sample. | | `get_post_analytics` | Combined author analytics, delivery, conversion, media, links, referrers, comparison data, and visible engagement. | -| `get_notes` | Bounded Notes page from authenticated profile or optional `profile_id`. | -| `get_profile_notes` | Bounded profile Notes page with raw per-Note metrics. | +| `get_notes` | Compact, body-first Notes from the authenticated profile or optional `profile_id`; supports guarded `fetch_all`. | +| `get_profile_notes` | Compact, body-first profile Notes with cursor paging or guarded `fetch_all`. | | `get_note_engagement` | Reactions, restacks, viewer state, and fully paginated direct/nested reply totals. | | `get_subscriber_summary` | Privacy-safe subscriber totals. Raw records require explicit `include_records: true`. | | `get_activity` | Bounded activity filtered by all events, replies and mentions, or restacks. | | `get_unread_activity` | Bounded unread activity plus unread metadata. | | `analyze_content` | Compact complete analytics for one post without comment or raw-response payloads. | -`get_publication_analytics` follows every email-stat page before calculating its summary, so it can make several authenticated requests for a large archive. Raw rows are excluded by default and capped when requested. `get_subscriber_summary` excludes subscriber records by default because they can contain email addresses and other personal data. See [MCP analytics](docs/mcp-analytics.md) for output semantics and usage examples. +`get_publication_analytics` follows every email-stat page before calculating its summary, so it can make several authenticated requests for a large archive. Raw rows are excluded by default and capped when requested. `get_notes` and `get_profile_notes` default to 10 complete Note bodies; set `fetch_all: true` to follow every cursor up to `max_items` (default 500, maximum 5,000). `get_subscriber_summary` excludes subscriber records by default because they can contain email addresses and other personal data. See [MCP analytics](docs/mcp-analytics.md) for output semantics and usage examples. ## API diff --git a/docs/mcp-analytics.md b/docs/mcp-analytics.md index 9caf25c..a29cd2e 100644 --- a/docs/mcp-analytics.md +++ b/docs/mcp-analytics.md @@ -1,6 +1,6 @@ # MCP analytics -The package's STDIO MCP server is read-only. It uses the same authenticated `SubstackClient` as the SDK and returns each successful result as both JSON text and structured content under `{ data }`. +The package's STDIO MCP server is read-only. It uses the same authenticated `SubstackClient` as the SDK and returns normalized structured content under `{ data }`. The text content is a short result summary rather than a duplicate JSON payload, which keeps model context smaller. Substack's web API is undocumented and can change without notice. Raw endpoint fields are retained where a tool returns them, while derived fields use explicit names and bounded payloads. @@ -15,9 +15,9 @@ The result contains: - `rowsAnalyzed`: rows remaining after date filtering. - `dateRange`: earliest and latest included raw `post_date` values. - `totals`: sums of additive delivery, engagement, conversion, revenue, podcast, and video fields that were present. -- `averageRates`: arithmetic means of Substack's raw `open_rate`, `click_through_rate`, and `engagement_rate` values. These are not recalculated or weighted because Substack does not document every numerator's semantics. +- `summary`: arithmetic means of Substack's raw `open_rate`, `click_through_rate`, and `engagement_rate` values. These are not recalculated or weighted because Substack does not document every numerator's semantics. - `breakdowns`: post counts grouped by audience, section, and content type. -- `topPosts`: a bounded ranking by the requested metric. +- `top_posts`: a bounded ranking by `top_metric`. - `availableFields`: every raw field present in the included rows, including future fields unknown to the SDK. - `rows`: optional raw rows, capped by `row_limit` at 200. @@ -50,16 +50,32 @@ Example MCP arguments: ## Note analytics -`get_profile_notes` returns raw per-Note fields from a bounded profile feed page. `get_note_engagement` calls `getNoteWithEngagement()`, follows every reply cursor, and reports normalized reactions, restacks, direct replies, nested replies, total replies, viewer state, and `replyCountsComplete`. +`get_notes` and `get_profile_notes` return compact, body-first records with the complete Note `body`, plus `id`, `created_at`, and an upstream permalink when available. Large tracking, publication, theme, palette, and subscription objects are removed. Minimal attachment details are retained only for Notes without a text body. -Only the returned reply sample is capped. The normalized counts still represent every safely loaded page. `rawReplyPages` is excluded unless `include_raw_pages` is true. Note views remain absent unless a future Substack response includes a numeric `views` or `view_count` field. +Both tools default to 10 Notes and accept `limit` from 1 through 50. Their response includes `returned`, `pages_fetched`, `complete`, `has_more`, and a normalized `cursor`. + +Set `fetch_all` to true to follow profile-feed cursors automatically. `max_items` is a safety bound (default 500, maximum 5,000), repeated Note IDs are removed, and repeated cursors stop collection. If the bound is reached, `complete` is false and `cursor` identifies the page from which collection can resume. + +Example full-profile request: + +```json +{ + "profile_id": 44242110, + "fetch_all": true, + "max_items": 500 +} +``` + +`get_note_engagement` calls `getNoteWithEngagement()`, follows every reply cursor, and reports normalized reactions, restacks, direct replies, nested replies, total replies, viewer state, and `replyCountsComplete`. + +Only the returned reply sample is capped. The normalized counts still represent every safely loaded page. `raw_reply_pages` is excluded unless `include_raw_pages` is true. Note views remain absent unless a future Substack response includes a numeric `views` or `view_count` field. ## Subscriber privacy `get_subscriber_summary` returns the upstream aggregate subscriber count when available, the number of records present in the response, numeric and boolean top-level aggregates, and available record field names without exposing record values. This is the default behavior. -Setting `include_records` to true returns up to `record_limit` raw subscriber records and sets `personalDataIncluded` to true. Those records can contain email addresses and other personal data. Use the option only in a trusted local MCP session and do not copy its results into logs, prompts, issues, or source control. +Setting `include_records` to true returns up to `record_limit` raw subscriber records and sets `personal_data_included` to true. Those records can contain email addresses and other personal data. Use the option only in a trusted local MCP session and do not copy its results into logs, prompts, issues, or source control. ## Response limits -List-returning tools cap data returned to the model while retaining upstream pagination metadata when available. The caps reduce context size; they do not change Substack's fixed email-stat page size or the complete pagination needed for publication and Note summaries. +List-returning tools cap and normalize data returned to the model while retaining pagination metadata when available. Activity tools discard large upstream dependency tables and tracking parameters. These reductions do not change Substack's fixed email-stat page size or the complete pagination used for publication analytics and guarded full-profile Note collection. diff --git a/package.json b/package.json index d49d70d..6be35a6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "unofficial-substack-sdk", - "version": "0.3.9", + "version": "0.3.10", "description": "Unofficial, portable TypeScript SDK for Substack's web API", "type": "module", "license": "MIT", diff --git a/src/core/client.ts b/src/core/client.ts index ab31255..3f664f5 100644 --- a/src/core/client.ts +++ b/src/core/client.ts @@ -28,6 +28,7 @@ import type { PostManagementPost, PostWithEngagement, PostWithEngagementOptions, + ProfileNotesOptions, ProfileNotesPage, PublishNoteRequest, ProfilePostsOptions, @@ -207,7 +208,7 @@ export class SubstackClient { getProfileNotes = NoteFeedItem>( id: number | string, - options: CursorOptions = {} + options: ProfileNotesOptions = {} ): Promise> { return getProfileNotes(this.endpoints, id, options) } diff --git a/src/core/index.ts b/src/core/index.ts index 76a17c9..3734433 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -50,6 +50,7 @@ export { type PostWithEngagement, type PostWithEngagementOptions, type ProfileNoteItem, + type ProfileNotesOptions, type ProfileNotesPage, type PublishNoteRequest, type ProfilePostsOptions, diff --git a/src/core/types.ts b/src/core/types.ts index 60845db..1b56be4 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -36,7 +36,12 @@ export interface CursorOptions { cursor?: string } -export interface NotesOptions extends CursorOptions { +export interface ProfileNotesOptions extends CursorOptions { + /** Requested upstream page size. */ + limit?: number +} + +export interface NotesOptions extends ProfileNotesOptions { profileId?: number | string } diff --git a/src/mcp-server.ts b/src/mcp-server.ts index 2bd1647..48d52cb 100644 --- a/src/mcp-server.ts +++ b/src/mcp-server.ts @@ -45,6 +45,9 @@ type PublicationAnalyticsOptions = EmailStatsOptions & { const id = z.union([z.number().int().positive(), z.string().min(1)]) const limit = z.number().int().min(1).max(50).default(20) +const noteLimit = z.number().int().min(1).max(50).default(10) +const fetchAll = z.boolean().default(false) +const maxNoteItems = z.number().int().min(1).max(5_000).default(500) const emailRowLimit = z.number().int().min(1).max(20).default(20) const rawRowLimit = z.number().int().min(1).max(200).default(20) const date = z.string().regex(/^\d{4}-\d{2}-\d{2}$/, 'Use YYYY-MM-DD.') @@ -134,7 +137,8 @@ const authenticatedProfileOutputSchema = { const recentPostsOutputSchema = { data: z .object({ - posts: z.array(z.record(z.string(), z.unknown())).describe('Recent published posts') + posts: z.array(z.record(z.string(), z.unknown())).describe('Recent published posts'), + cursor: z.string().optional().describe('Pagination cursor for the next page') }) .passthrough() } @@ -181,22 +185,39 @@ const postAnalyticsOutputSchema = { .passthrough() } +const compactNoteSchema = z.object({ + id: z.union([z.number(), z.string()]).optional().describe('Note ID'), + body: z.string().describe('Complete Note body'), + created_at: z.string().optional().describe('Note publication timestamp'), + url: z.string().optional().describe('Note permalink when supplied upstream'), + attachments: z + .array(z.record(z.string(), z.unknown())) + .optional() + .describe('Minimal attachment details for Notes without a text body') +}) + const notesOutputSchema = { data: z .object({ - items: z.array(z.record(z.string(), z.unknown())).describe('Notes items'), - cursor: z.string().optional().describe('Pagination cursor for next page') + items: z.array(compactNoteSchema).describe('Body-first Note records'), + returned: z.number().int().nonnegative().describe('Number of Notes returned'), + pages_fetched: z.number().int().positive().describe('Number of upstream pages fetched'), + complete: z.boolean().describe('Whether the requested collection was fully fetched'), + has_more: z.boolean().describe('Whether another page is available'), + cursor: z.string().nullable().describe('Pagination cursor for the next page') }) - .passthrough() } const profileNotesOutputSchema = { data: z .object({ - items: z.array(z.record(z.string(), z.unknown())).describe('Notes published by profile'), - cursor: z.string().optional().describe('Pagination cursor for next page') + items: z.array(compactNoteSchema).describe('Body-first Notes published by the profile'), + returned: z.number().int().nonnegative().describe('Number of Notes returned'), + pages_fetched: z.number().int().positive().describe('Number of upstream pages fetched'), + complete: z.boolean().describe('Whether the requested collection was fully fetched'), + has_more: z.boolean().describe('Whether another page is available'), + cursor: z.string().nullable().describe('Pagination cursor for the next page') }) - .passthrough() } const noteEngagementOutputSchema = { @@ -235,7 +256,8 @@ const subscriberStatsOutputSchema = { const activityOutputSchema = { data: z .object({ - activityItems: z.array(z.record(z.string(), z.unknown())).describe('Activity notification items') + activityItems: z.array(z.record(z.string(), z.unknown())).describe('Activity notification items'), + more: z.boolean().optional().describe('Whether more activity is available upstream') }) .passthrough() } @@ -332,8 +354,17 @@ const growthSourcesGranularity = z function result(data: unknown): ToolResult { const output = { data } + const record = isRecord(data) ? data : undefined + const collectionKey = record + ? ['items', 'posts', 'rows', 'activityItems', 'comments', 'replies', 'top_posts', 'subscribers'].find( + (key) => Array.isArray(record[key]) + ) + : undefined + const summary = collectionKey + ? `Returned ${(record?.[collectionKey] as unknown[]).length} ${collectionKey}.` + : 'Request completed successfully.' return { - content: [{ type: 'text', text: JSON.stringify(output, null, 2) }], + content: [{ type: 'text', text: summary }], structuredContent: output } } @@ -375,6 +406,7 @@ function compactPost(post: Record): Record { 'title', 'subtitle', 'slug', + 'canonical_url', 'post_date', 'audience', 'type', @@ -383,6 +415,229 @@ function compactPost(post: Record): Record { return Object.fromEntries(keys.filter((key) => post[key] !== undefined).map((key) => [key, post[key]])) } +function selectDefined( + record: Record, + keys: readonly string[] +): Record { + return Object.fromEntries( + keys + .filter((key) => record[key] !== undefined && record[key] !== null) + .map((key) => [key, record[key]]) + ) +} + +function compactProfile(profile: unknown): Record { + return isRecord(profile) + ? selectDefined(profile, ['id', 'handle', 'name', 'photo_url', 'bio']) + : {} +} + +function bodyJsonText(value: unknown): string { + if (Array.isArray(value)) return value.map(bodyJsonText).filter(Boolean).join('\n') + if (!isRecord(value)) return '' + if (typeof value.text === 'string') return value.text + return bodyJsonText(value.content) +} + +function compactAttachment(value: unknown): Record | undefined { + if (!isRecord(value)) return undefined + const post = isRecord(value.post) ? value.post : undefined + const comment = isRecord(value.comment) ? value.comment : undefined + const compact = selectDefined(value, ['id', 'type', 'url', 'title']) + const body = typeof comment?.body === 'string' ? comment.body : undefined + const title = typeof post?.title === 'string' ? post.title : undefined + const url = + typeof post?.canonical_url === 'string' + ? post.canonical_url + : typeof comment?.canonical_url === 'string' + ? comment.canonical_url + : undefined + return { + ...compact, + ...(title ? { title } : {}), + ...(body ? { body } : {}), + ...(url ? { url } : {}) + } +} + +function compactNote(value: unknown): Record { + const item = isRecord(value) ? value : {} + const comment = isRecord(item.comment) ? item.comment : item + const context = isRecord(item.context) ? item.context : undefined + const entityKey = typeof item.entity_key === 'string' ? item.entity_key : undefined + const parsedEntityId = entityKey?.startsWith('c-') ? finiteNumber(entityKey.slice(2)) : undefined + const body = + typeof comment.body === 'string' + ? comment.body + : bodyJsonText(comment.body_json) + const createdAt = + typeof comment.date === 'string' + ? comment.date + : typeof comment.created_at === 'string' + ? comment.created_at + : typeof context?.timestamp === 'string' + ? context.timestamp + : undefined + const urlCandidate = [comment.url, comment.canonical_url, item.url, item.canonical_url].find( + (candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0 + ) + const attachments = Array.isArray(comment.attachments) + ? comment.attachments.map(compactAttachment).filter(isRecord) + : [] + + return { + ...(comment.id !== undefined + ? { id: comment.id } + : item.id !== undefined + ? { id: item.id } + : parsedEntityId !== undefined + ? { id: parsedEntityId } + : {}), + body, + ...(createdAt ? { created_at: createdAt } : {}), + ...(urlCandidate ? { url: urlCandidate } : {}), + ...(!body.trim() && attachments.length > 0 ? { attachments } : {}) + } +} + +function compactComment(value: unknown): Record { + if (!isRecord(value)) return {} + const compact = selectDefined(value, [ + 'id', + 'body', + 'date', + 'created_at', + 'name', + 'handle', + 'user_id', + 'parent_id', + 'reaction_count', + 'restacks', + 'children_count' + ]) + if (Array.isArray(value.children)) { + compact.children = value.children.map(compactComment) + } + return compact +} + +function compactReply(value: unknown): Record { + if (!isRecord(value)) return {} + const reply = isRecord(value.comment) ? compactComment(value.comment) : compactComment(value) + if (Array.isArray(value.descendantComments)) { + reply.descendant_replies = value.descendantComments.map((descendant) => + isRecord(descendant) && isRecord(descendant.comment) + ? compactComment(descendant.comment) + : compactComment(descendant) + ) + } + return reply +} + +function compactActivityItem(value: unknown): Record { + if (!isRecord(value)) return {} + return selectDefined(value, [ + 'id', + 'user_id', + 'item_key', + 'type', + 'created_at', + 'updated_at', + 'sender_count', + 'recent_sender_ids', + 'publication_id', + 'comment_id', + 'mention_id', + 'target_user_id', + 'target_post_id', + 'target_comment_id', + 'target_community_post_id', + 'target_community_comment_id', + 'target_live_stream_id', + 'target_media_clip_id', + 'source', + 'source_name', + 'isNew', + 'cta', + 'secondaryCta' + ]) +} + +type CompactNotesOptions = { + cursor?: string + limit: number + fetchAll: boolean + maxItems: number +} + +async function collectCompactNotes( + fetchPage: (cursor: string | undefined, limit: number) => Promise, + options: CompactNotesOptions +): Promise> { + const items: Record[] = [] + const seenIds = new Set() + const seenCursors = new Set() + let cursor = options.cursor + let pagesFetched = 0 + let complete = false + + while (true) { + const pageCursor = cursor + const pageLimit = options.fetchAll + ? Math.max(1, Math.min(50, options.maxItems - items.length)) + : options.limit + const response = await fetchPage(pageCursor, pageLimit) + pagesFetched += 1 + const page = isRecord(response) ? response : {} + const pageItems = Array.isArray(page.items) ? page.items.map(compactNote) : [] + const uniquePageItems = pageItems.filter((item) => { + if (item.id === undefined) return true + const key = String(item.id) + if (seenIds.has(key)) return false + seenIds.add(key) + return true + }) + const nextCursor = + typeof page.nextCursor === 'string' && page.nextCursor.length > 0 + ? page.nextCursor + : undefined + + if (options.fetchAll && items.length > 0 && items.length + uniquePageItems.length > options.maxItems) { + cursor = pageCursor + break + } + + const remaining = options.fetchAll ? options.maxItems - items.length : options.limit + items.push(...uniquePageItems.slice(0, remaining)) + if (!options.fetchAll) { + cursor = nextCursor + complete = !nextCursor + break + } + if (!nextCursor) { + cursor = undefined + complete = true + break + } + if (seenCursors.has(nextCursor)) { + cursor = nextCursor + break + } + seenCursors.add(nextCursor) + cursor = nextCursor + if (items.length >= options.maxItems) break + } + + return { + items, + returned: items.length, + pages_fetched: pagesFetched, + complete, + has_more: !complete, + cursor: cursor ?? null + } +} + function filterRowsByDate( rows: EmailStatsRow[], fromDate?: string, @@ -443,13 +698,14 @@ function summarizeEmailStats( rowsAnalyzed: rows.length, dateRange: dates.length > 0 ? { from: dates[0], to: dates.at(-1) } : undefined, totals, - averageRates: averages, + summary: averages, breakdowns: { byAudience: dimensionCounts(rows, 'audience'), bySection: dimensionCounts(rows, 'section_name'), byType: dimensionCounts(rows, 'type') }, - topPosts: { metric: topMetric, posts: topPosts }, + top_metric: topMetric, + top_posts: topPosts, availableFields: [...new Set(rows.flatMap((row) => Object.keys(row)))].sort(), ...(options.includeRows ? { rows: rows.slice(0, options.rowLimit ?? 20) } : {}) } @@ -471,6 +727,17 @@ function subscriberCount(response: Record, records: unknown[]): return records.length } +function optionalSubscriberCount( + response: Record, + records: unknown[] +): number | undefined { + for (const key of ['total', 'count', 'subscriber_count', 'subscriberCount']) { + const value = finiteNumber(response[key]) + if (value !== undefined && value >= 0) return Math.floor(value) + } + return Array.isArray(response.subscribers) ? records.length : undefined +} + export function createToolHandlers(client: ReadOnlyClient) { const run = async (work: () => Promise) => { try { @@ -480,43 +747,60 @@ export function createToolHandlers(client: ReadOnlyClient) { } } - const getPostAnalytics = ( + const loadPostAnalytics = async ( postId: string | number, commentLimit = 20, includeRaw = false - ) => - run(async () => { - const [postResult, managementDetail] = await Promise.all([ - client.getPostWithEngagement(postId), - client.getPostManagementDetail(postId) - ]) - const managementPost = Array.isArray(managementDetail.posts) - ? managementDetail.posts.find((post) => String(post.id) === String(postId)) ?? managementDetail.posts[0] - : undefined + ) => { + const [postResult, managementDetail] = await Promise.all([ + client.getPostWithEngagement(postId), + client.getPostManagementDetail(postId) + ]) + const managementPost = Array.isArray(managementDetail.posts) + ? managementDetail.posts.find((post) => String(post.id) === String(postId)) ?? managementDetail.posts[0] + : undefined + const managementEngagement = managementPost + ? { + reaction_count: managementPost.reaction_count, + reactions: managementPost.reactions, + comment_count: managementPost.comment_count, + reply_count: managementPost.child_comment_count + } + : undefined - return { - post: compactPost(postResult.post), - analytics: managementPost?.stats, - contentEngagement: postResult.engagement, - managementEngagement: managementPost - ? { - reactionCount: managementPost.reaction_count, - reactions: managementPost.reactions, - commentCount: managementPost.comment_count, - replyCount: managementPost.child_comment_count - } - : undefined, - comments: postResult.commentItems.slice(0, commentLimit), - commentsReturned: Math.min(commentLimit, postResult.commentItems.length), - visibleCommentCount: postResult.commentItems.length, - ...(includeRaw ? { raw: { post: postResult.post, managementDetail } } : {}) - } - }) + return { + post: compactPost(postResult.post), + stats: managementPost?.stats, + engagement: { + content: postResult.engagement, + management: managementEngagement + }, + comments: postResult.commentItems.slice(0, commentLimit).map(compactComment), + comments_returned: Math.min(commentLimit, postResult.commentItems.length), + visible_comment_count: postResult.commentItems.length, + ...(includeRaw ? { raw: { post: postResult.post, managementDetail } } : {}) + } + } + + const getPostAnalytics = ( + postId: string | number, + commentLimit = 20, + includeRaw = false + ) => run(() => loadPostAnalytics(postId, commentLimit, includeRaw)) return { - getAuthenticatedProfile: () => run(() => client.getAuthenticatedProfile()), + getAuthenticatedProfile: () => run(async () => compactProfile(await client.getAuthenticatedProfile())), getRecentPosts: (profileId: string | number, maximum = 20) => - run(async () => capped(await client.getProfilePosts(profileId, { limit: maximum }), maximum)), + run(async () => { + const response = await client.getProfilePosts(profileId, { limit: maximum }) + const record = isRecord(response) ? response : {} + return { + posts: Array.isArray(record.posts) + ? record.posts.slice(0, maximum).map((post) => compactPost(isRecord(post) ? post : {})) + : [], + ...(typeof record.nextCursor === 'string' ? { cursor: record.nextCursor } : {}) + } + }), getEmailStats: (options: EmailStatsOptions, maximum = 20) => run(async () => capped(await client.getEmailStats(options), maximum)), getPublicationAnalytics: (options: PublicationAnalyticsOptions = {}) => @@ -542,13 +826,41 @@ export function createToolHandlers(client: ReadOnlyClient) { getPostEngagement: (postId: string | number, maximum = 20) => run(async () => { const { post, engagement, commentItems } = await client.getPostWithEngagement(postId) - return { post, engagement, comments: commentItems.slice(0, maximum) } + return { + post: compactPost(post), + engagement, + comments: commentItems.slice(0, maximum).map(compactComment) + } }), getPostAnalytics, - getNotes: (cursor: string | undefined, maximum = 20, profileId?: string | number) => - run(async () => capped(await client.getNotes({ cursor, profileId }), maximum)), - getProfileNotes: (profileId: string | number, cursor: string | undefined, maximum = 20) => - run(async () => capped(await client.getProfileNotes(profileId, { cursor }), maximum)), + getNotes: ( + cursor: string | undefined, + maximum = 10, + profileId?: string | number, + fetchEveryPage = false, + maximumItems = 500 + ) => + run(() => + collectCompactNotes( + (pageCursor, pageLimit) => + client.getNotes({ cursor: pageCursor, profileId, limit: pageLimit }), + { cursor, limit: maximum, fetchAll: fetchEveryPage, maxItems: maximumItems } + ) + ), + getProfileNotes: ( + profileId: string | number, + cursor: string | undefined, + maximum = 10, + fetchEveryPage = false, + maximumItems = 500 + ) => + run(() => + collectCompactNotes( + (pageCursor, pageLimit) => + client.getProfileNotes(profileId, { cursor: pageCursor, limit: pageLimit }), + { cursor, limit: maximum, fetchAll: fetchEveryPage, maxItems: maximumItems } + ) + ), getNoteEngagement: ( noteId: string | number, maximum = 20, @@ -557,12 +869,12 @@ export function createToolHandlers(client: ReadOnlyClient) { run(async () => { const noteResult = await client.getNoteWithEngagement(noteId) return { - note: noteResult.note.item, + item: compactNote(noteResult.note.item), engagement: noteResult.engagement, - replyPagesFetched: noteResult.replyPages.length, - replies: noteResult.replies.slice(0, maximum), - repliesReturned: Math.min(maximum, noteResult.replies.length), - ...(includeRawPages ? { rawReplyPages: noteResult.replyPages } : {}) + reply_pages_fetched: noteResult.replyPages.length, + replies: noteResult.replies.slice(0, maximum).map(compactReply), + replies_returned: Math.min(maximum, noteResult.replies.length), + ...(includeRawPages ? { raw_reply_pages: noteResult.replyPages } : {}) } }), getSubscriberSummary: (includeRecords = false, maximum = 20) => @@ -571,37 +883,65 @@ export function createToolHandlers(client: ReadOnlyClient) { const record = isRecord(response) ? response : {} const subscribers = Array.isArray(response.subscribers) ? response.subscribers : [] return { - subscriberCount: subscriberCount(record, subscribers), - recordsReturnedByUpstream: subscribers.length, - upstreamAggregates: safeSubscriberMetadata(record), - availableRecordFields: [ + count: subscriberCount(record, subscribers), + records_returned_by_upstream: subscribers.length, + aggregates: safeSubscriberMetadata(record), + available_record_fields: [ ...new Set( subscribers.flatMap((subscriber) => (isRecord(subscriber) ? Object.keys(subscriber) : [])) ) ].sort(), - personalDataIncluded: includeRecords, + personal_data_included: includeRecords, ...(includeRecords ? { subscribers: subscribers.slice(0, maximum) } : {}) } }), getActivity: (filter: 'all' | 'replies-and-mentions' | 'restacks', maximum = 20) => - run(async () => capped(await client.getActivity(filter), maximum)), + run(async () => { + const response = await client.getActivity(filter) + return { + activityItems: Array.isArray(response.activityItems) + ? response.activityItems.slice(0, maximum).map(compactActivityItem) + : [], + ...(typeof response.more === 'boolean' ? { more: response.more } : {}) + } + }), getUnreadActivity: (maximum = 20) => - run(async () => capped(await client.getUnreadActivity(), maximum)), - analyzeContent: (postId: string | number) => getPostAnalytics(postId, 0, false), - getSubscriberStats: async (): Promise => { - try { - const stats = await client.getSubscriberStats() + run(async () => { + const response = await client.getUnreadActivity() return { - content: [{ type: 'text', text: JSON.stringify(stats, null, 2) }], - structuredContent: stats as Record + activityItems: response.activityItems.slice(0, maximum).map(compactActivityItem), + unread: response.unread, + ...(typeof response.more === 'boolean' ? { more: response.more } : {}) } - } catch (error: any) { + }), + analyzeContent: (postId: string | number) => + run(async () => { + const analytics = await loadPostAnalytics(postId, 0, false) + const post = isRecord(analytics.post) ? analytics.post : {} return { - isError: true, - content: [{ type: 'text', text: `Failed to fetch subscriber stats: ${error.message}` }] + post_id: post.id ?? postId, + ...(typeof post.title === 'string' ? { title: post.title } : {}), + performance: analytics.stats, + engagement: analytics.engagement } - } - }, + }), + getSubscriberStats: () => + run(async () => { + const response = await client.getSubscriberStats>() + const record = isRecord(response) ? response : {} + const subscribers = Array.isArray(record.subscribers) ? record.subscribers : [] + const count = optionalSubscriberCount(record, subscribers) + return { + ...(count !== undefined ? { total_subscribers: count } : {}), + ...selectDefined(record, [ + 'active_subscribers_delivered', + 'recent_signups', + 'open_rate', + 'latest_post_title', + 'derived_from_delivery' + ]) + } + }), getGrowthSources: (options: GrowthSourcesOptions = {}) => run(async () => { const fromDate = options.fromDate ?? options.from_date @@ -615,7 +955,7 @@ export function createToolHandlers(client: ReadOnlyClient) { } export function createMcpServer(client: ReadOnlyClient): McpServer { - const server = new McpServer({ name: 'substack-mcp', version: '0.3.9' }) + const server = new McpServer({ name: 'substack-mcp', version: '0.3.10' }) const tools = createToolHandlers(client) server.registerTool( @@ -733,23 +1073,38 @@ export function createMcpServer(client: ReadOnlyClient): McpServer { { title: 'Get publication Notes', description: - 'Get a bounded page of Notes from the authenticated publication/profile feed or a specified profile ID.', - inputSchema: { profile_id: id.optional(), cursor: z.string().optional(), limit }, + 'Get compact, body-first Notes from the authenticated profile or a specified profile ID. Set fetch_all to follow pagination safely up to max_items.', + inputSchema: { + profile_id: id.optional(), + cursor: z.string().optional(), + limit: noteLimit, + fetch_all: fetchAll, + max_items: maxNoteItems + }, outputSchema: notesOutputSchema, annotations: readOnlyAnnotations }, - ({ profile_id, cursor, limit }) => tools.getNotes(cursor, limit, profile_id) + ({ profile_id, cursor, limit, fetch_all, max_items }) => + tools.getNotes(cursor, limit, profile_id, fetch_all, max_items) ) server.registerTool( 'get_profile_notes', { title: 'Get profile Notes', - description: 'Get a bounded profile Notes page with raw per-Note engagement fields.', - inputSchema: { profile_id: id, cursor: z.string().optional(), limit }, + description: + 'Get compact, body-first Notes for a profile. Set fetch_all to follow pagination safely up to max_items.', + inputSchema: { + profile_id: id, + cursor: z.string().optional(), + limit: noteLimit, + fetch_all: fetchAll, + max_items: maxNoteItems + }, outputSchema: profileNotesOutputSchema, annotations: readOnlyAnnotations }, - ({ profile_id, cursor, limit }) => tools.getProfileNotes(profile_id, cursor, limit) + ({ profile_id, cursor, limit, fetch_all, max_items }) => + tools.getProfileNotes(profile_id, cursor, limit, fetch_all, max_items) ) server.registerTool( 'get_note_engagement', diff --git a/src/resources/notes/index.ts b/src/resources/notes/index.ts index a2490a8..664e607 100644 --- a/src/resources/notes/index.ts +++ b/src/resources/notes/index.ts @@ -17,6 +17,7 @@ import type { NoteRestackOptions, NotesOptions, NoteWithEngagement, + ProfileNotesOptions, ProfileNotesPage, PublishNoteRequest, ScheduleNoteRequest, @@ -79,10 +80,13 @@ export function getProfileNotes< >( context: EndpointContext, id: number | string, - options: CursorOptions = {} + options: ProfileNotesOptions = {} ): Promise> { const profileId = positiveInteger(id, 'Profile ID') const query = new URLSearchParams({ types: 'note' }) + if (options.limit !== undefined) { + query.set('limit', String(positiveInteger(options.limit, 'Profile Notes limit'))) + } if (options.cursor) { query.set('cursor', options.cursor) } diff --git a/test/client.test.ts b/test/client.test.ts index a17b175..4497c37 100644 --- a/test/client.test.ts +++ b/test/client.test.ts @@ -316,10 +316,10 @@ describe('SubstackClient', () => { } }) - await client.getNotes({ profileId: 42, cursor: 'next page' }) + await client.getNotes({ profileId: 42, cursor: 'next page', limit: 10 }) expect(request?.url).toBe( - 'https://allagentsconsidered.substack.com/api/v1/reader/feed/profile/42?types=note&cursor=next+page' + 'https://allagentsconsidered.substack.com/api/v1/reader/feed/profile/42?types=note&limit=10&cursor=next+page' ) }) diff --git a/test/mcp.test.ts b/test/mcp.test.ts index 7c65742..07a3269 100644 --- a/test/mcp.test.ts +++ b/test/mcp.test.ts @@ -55,8 +55,18 @@ const mockClient = (overrides: Record = {}) => ({ commentItems: [{ id: 10 }, { id: 11 }], engagement: { visibleCommentCount: 2 } }), - getNotes: async () => ({ items: [{ id: 1 }, { id: 2 }] }), - getProfileNotes: async () => ({ items: [{ id: 3 }, { id: 4 }] }), + getNotes: async () => ({ + items: [ + { comment: { id: 1, body: 'First Note', date: '2026-08-01T10:00:00Z' } }, + { comment: { id: 2, body: 'Second Note', date: '2026-08-02T10:00:00Z' } } + ] + }), + getProfileNotes: async () => ({ + items: [ + { comment: { id: 3, body: 'Third Note', date: '2026-08-03T10:00:00Z' } }, + { comment: { id: 4, body: 'Fourth Note', date: '2026-08-04T10:00:00Z' } } + ] + }), getNoteWithEngagement: async () => ({ note: { item: { comment: { id: 5, reaction_count: 9, restacks: 2 } } }, replyPages: [{ commentBranches: [] }, { commentBranches: [] }], @@ -106,10 +116,182 @@ describe('MCP tools', () => { data: { rows: [{ post_id: 1 }] } }) expect((await tools.getNotes(undefined, 1)).structuredContent).toEqual({ - data: { items: [{ id: 1 }] } + data: { + items: [{ id: 1, body: 'First Note', created_at: '2026-08-01T10:00:00Z' }], + returned: 1, + pages_fetched: 1, + complete: true, + has_more: false, + cursor: null + } }) expect((await tools.getProfileNotes(7, undefined, 1)).structuredContent).toEqual({ - data: { items: [{ id: 3 }] } + data: { + items: [{ id: 3, body: 'Third Note', created_at: '2026-08-03T10:00:00Z' }], + returned: 1, + pages_fetched: 1, + complete: true, + has_more: false, + cursor: null + } + }) + }) + + test('fetches every profile Note page, normalizes bodies, and deduplicates IDs', async () => { + const cursors: Array = [] + const tools = createToolHandlers( + mockClient({ + getProfileNotes: async (_profileId: number, options: { cursor?: string }) => { + cursors.push(options.cursor) + return options.cursor + ? { + items: [ + { comment: { id: 2, body: 'Duplicate' } }, + { comment: { id: 3, body: 'Final body' } } + ], + nextCursor: null + } + : { + items: [ + { comment: { id: 1, body: 'First body' } }, + { comment: { id: 2, body: 'Second body' } } + ], + nextCursor: 'page-2' + } + } + }) as never + ) + + const result = await tools.getProfileNotes(7, undefined, 10, true, 500) + + expect(cursors).toEqual([undefined, 'page-2']) + expect(result.structuredContent).toEqual({ + data: { + items: [ + { id: 1, body: 'First body' }, + { id: 2, body: 'Second body' }, + { id: 3, body: 'Final body' } + ], + returned: 3, + pages_fetched: 2, + complete: true, + has_more: false, + cursor: null + } + }) + expect(result.content[0].text).toBe('Returned 3 items.') + }) + + test('stops fetch-all collection at max_items with a resumable cursor', async () => { + const tools = createToolHandlers( + mockClient({ + getProfileNotes: async ( + _profileId: number, + options: { cursor?: string; limit?: number } + ) => { + const start = options.cursor === 'page-2' ? 41 : 1 + const count = options.cursor === 'page-2' ? options.limit ?? 10 : 40 + return { + items: Array.from({ length: count }, (_, index) => ({ + comment: { id: start + index, body: `Note ${start + index}` } + })), + nextCursor: options.cursor === 'page-2' ? 'page-3' : 'page-2' + } + } + }) as never + ) + + const result = await tools.getProfileNotes(7, undefined, 10, true, 50) + + expect(result.structuredContent).toMatchObject({ + data: { + returned: 50, + pages_fetched: 2, + complete: false, + has_more: true, + cursor: 'page-3' + } + }) + }) + + test('prunes raw Note and activity metadata from AI-facing responses', async () => { + const tools = createToolHandlers( + mockClient({ + getProfileNotes: async () => ({ + items: [ + { + entity_key: 'c-9', + context: { timestamp: '2026-08-28T10:00:00Z', page_rank: 1 }, + comment: { + id: 9, + body: 'Important body', + tracking_parameters: { large: 'discard me' }, + attachments: [{ type: 'image', url: 'https://example.com/image.png' }] + }, + trackingParameters: { large: 'discard me too' } + }, + { + comment: { + id: 10, + body: '', + attachments: [ + { + id: 'image-1', + type: 'image', + url: 'https://example.com/image.png', + publication: { theme: { large: 'discard me' } } + } + ] + } + } + ], + publications: [{ theme: { large: true } }] + }), + getActivity: async () => ({ + activityItems: [ + { + id: 'activity-1', + type: 'mention', + created_at: '2026-08-28T10:00:00Z', + trackingParams: { large: 'discard me' } + } + ], + users: [{ bio: 'discard me' }], + posts: [{ body_html: 'discard me' }], + more: true + }) + }) as never + ) + + const notes = await tools.getProfileNotes(7, undefined, 10) + const activity = await tools.getActivity('all', 10) + + expect(notes.structuredContent).toEqual({ + data: { + items: [ + { id: 9, body: 'Important body', created_at: '2026-08-28T10:00:00Z' }, + { + id: 10, + body: '', + attachments: [ + { id: 'image-1', type: 'image', url: 'https://example.com/image.png' } + ] + } + ], + returned: 2, + pages_fetched: 1, + complete: true, + has_more: false, + cursor: null + } + }) + expect(activity.structuredContent).toEqual({ + data: { + activityItems: [ + { id: 'activity-1', type: 'mention', created_at: '2026-08-28T10:00:00Z' } + ], + more: true + } }) }) @@ -141,23 +323,21 @@ describe('MCP tools', () => { subscribes: 3, podcast_preview_downloads: 25 }, - averageRates: { engagement_rate: 0.4 }, + summary: { engagement_rate: 0.4 }, breakdowns: { byAudience: { paid: 1 }, bySection: {}, byType: { podcast: 1 } }, - topPosts: { - metric: 'clicks', - posts: [ - { - post_id: 2, - title: 'Second', - post_date: '2026-02-01T10:00:00Z', - clicks: 12 - } - ] - }, + top_metric: 'clicks', + top_posts: [ + { + post_id: 2, + title: 'Second', + post_date: '2026-02-01T10:00:00Z', + clicks: 12 + } + ], availableFields: [ 'audience', 'clicks', @@ -192,17 +372,19 @@ describe('MCP tools', () => { expect(result.structuredContent).toEqual({ data: { post: { id: 1, title: 'Post', subtitle: 'Subtitle' }, - analytics: { delivered: 100, opens: 50, links: [['https://example.com', 4]] }, - contentEngagement: { visibleCommentCount: 2 }, - managementEngagement: { - reactionCount: 8, - reactions: undefined, - commentCount: 3, - replyCount: 2 + stats: { delivered: 100, opens: 50, links: [['https://example.com', 4]] }, + engagement: { + content: { visibleCommentCount: 2 }, + management: { + reaction_count: 8, + reactions: undefined, + comment_count: 3, + reply_count: 2 + } }, comments: [{ id: 10 }], - commentsReturned: 1, - visibleCommentCount: 2 + comments_returned: 1, + visible_comment_count: 2 } }) }) @@ -212,11 +394,13 @@ describe('MCP tools', () => { const data = result.structuredContent?.data as Record expect(data).toMatchObject({ - post: { id: 1, title: 'Post', subtitle: 'Subtitle' }, - analytics: { delivered: 100, opens: 50 }, - contentEngagement: { visibleCommentCount: 2 }, - comments: [], - commentsReturned: 0 + post_id: 1, + title: 'Post', + performance: { delivered: 100, opens: 50 }, + engagement: { + content: { visibleCommentCount: 2 }, + management: { reaction_count: 8, comment_count: 3, reply_count: 2 } + } }) expect(data).not.toHaveProperty('raw') }) @@ -226,7 +410,7 @@ describe('MCP tools', () => { expect(result.structuredContent).toEqual({ data: { - note: { comment: { id: 5, reaction_count: 9, restacks: 2 } }, + item: { id: 5, body: '' }, engagement: { reactionCount: 9, restackCount: 2, @@ -235,9 +419,9 @@ describe('MCP tools', () => { totalReplyCount: 3, replyCountsComplete: true }, - replyPagesFetched: 2, - replies: [{ comment: { id: 6 } }], - repliesReturned: 1 + reply_pages_fetched: 2, + replies: [{ id: 6 }], + replies_returned: 1 } }) }) @@ -251,16 +435,16 @@ describe('MCP tools', () => { > expect(safe).toEqual({ - subscriberCount: 2, - recordsReturnedByUpstream: 2, - upstreamAggregates: { total: 2, has_more: false }, - availableRecordFields: ['user_email_address', 'user_id'], - personalDataIncluded: false + count: 2, + records_returned_by_upstream: 2, + aggregates: { total: 2, has_more: false }, + available_record_fields: ['user_email_address', 'user_id'], + personal_data_included: false }) expect(safe).not.toHaveProperty('subscribers') expect(optedIn).toMatchObject({ - subscriberCount: 2, - personalDataIncluded: true, + count: 2, + personal_data_included: true, subscribers: [{ user_id: 1, user_email_address: 'one@example.com' }] }) }) @@ -270,9 +454,9 @@ describe('MCP tools', () => { const result = await tools.getSubscriberStats() expect(result.isError).toBeUndefined() - expect(result.structuredContent).toMatchObject({ total: 2 }) + expect(result.structuredContent).toEqual({ data: { total_subscribers: 2 } }) expect(result.content[0].type).toBe('text') - expect(JSON.parse(result.content[0].text)).toMatchObject({ total: 2 }) + expect(result.content[0].text).toBe('Request completed successfully.') }) test('handles errors cleanly in getSubscriberStats tool', async () => { @@ -286,7 +470,7 @@ describe('MCP tools', () => { const result = await tools.getSubscriberStats() expect(result.isError).toBe(true) - expect(result.content[0].text).toBe('Failed to fetch subscriber stats: Network timeout') + expect(result.content[0].text).toBe('Network timeout') }) test('returns growth sources structured content from tool handler with snake_case and camelCase args', async () => {