Skip to content

Commit 49967dd

Browse files
idoshamunidoshamun
authored
fix: ignore channel digest sources in digest and highlight generation (#3732)
1 parent b492b6f commit 49967dd

7 files changed

Lines changed: 381 additions & 15 deletions

File tree

‎__tests__/workers/generateChannelDigest.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,21 @@ const getLockKey = (digestKey: string, scheduledAt: string) =>
9191

9292
describe('generateChannelDigest worker', () => {
9393
afterEach(async () => {
94+
jest.restoreAllMocks();
9495
nock.cleanAll();
9596
await deleteKeysByPattern('channel-digest:*');
97+
await con.getRepository(ChannelDigest).clear();
98+
await con
99+
.createQueryBuilder()
100+
.delete()
101+
.from('post')
102+
.where('"sourceId" IN (:...sourceIds)', {
103+
sourceIds: ['content-source', AGENTS_DIGEST_SOURCE, 'weekly-source'],
104+
})
105+
.execute();
106+
await con
107+
.getRepository(Source)
108+
.delete(['content-source', AGENTS_DIGEST_SOURCE, 'weekly-source']);
96109
});
97110

98111
it('should be registered', () => {
@@ -270,6 +283,70 @@ describe('generateChannelDigest worker', () => {
270283
).toBeGreaterThan(ONE_DAY_IN_SECONDS);
271284
});
272285

286+
it('should ignore posts from all channel digest sources when generating digests', async () => {
287+
const scheduledAt = '2026-03-03T10:00:00.000Z';
288+
await con
289+
.getRepository(Source)
290+
.save([
291+
createSource(
292+
AGENTS_DIGEST_SOURCE,
293+
'Agents Digest',
294+
'https://daily.dev/agents.png',
295+
),
296+
createSource(
297+
'weekly-source',
298+
'Weekly Digest',
299+
'https://daily.dev/weekly.png',
300+
),
301+
]);
302+
await saveDefinition({
303+
key: 'agentic',
304+
sourceId: AGENTS_DIGEST_SOURCE,
305+
channel: 'vibes',
306+
});
307+
await saveDefinition({
308+
key: 'weekly-test',
309+
sourceId: 'weekly-source',
310+
channel: 'weekly',
311+
frequency: 'weekly',
312+
});
313+
await savePost({
314+
id: 'agents-post',
315+
sourceId: AGENTS_DIGEST_SOURCE,
316+
title: 'Agents digest post',
317+
content: 'Agents digest body',
318+
createdAt: new Date('2026-03-03T09:10:00.000Z'),
319+
channel: 'vibes',
320+
});
321+
await savePost({
322+
id: 'weekly-post',
323+
sourceId: 'weekly-source',
324+
title: 'Weekly digest post',
325+
content: 'Weekly digest body',
326+
createdAt: new Date('2026-03-03T09:20:00.000Z'),
327+
channel: 'vibes',
328+
});
329+
const digestCountBefore = await con.getRepository(FreeformPost).countBy({
330+
sourceId: AGENTS_DIGEST_SOURCE,
331+
});
332+
333+
await expectSuccessfulTypedBackground<'api.v1.generate-channel-digest'>(
334+
worker,
335+
{
336+
digestKey: 'agentic',
337+
scheduledAt,
338+
},
339+
);
340+
341+
const digests = await con.getRepository(FreeformPost).findBy({
342+
sourceId: AGENTS_DIGEST_SOURCE,
343+
});
344+
expect(digests).toHaveLength(digestCountBefore);
345+
expect(
346+
digests.find((digest) => digest.title === 'Mock sentiment digest'),
347+
).toBeUndefined();
348+
});
349+
273350
it('should use a weekly done ttl for weekly digests', async () => {
274351
const scheduledAt = '2026-03-02T10:00:00.000Z';
275352
await con

‎__tests__/workers/generateChannelHighlight.ts‎

Lines changed: 173 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
import { IsNull, type DataSource } from 'typeorm';
22
import createOrGetConnection from '../../src/db';
3+
import { ChannelDigest } from '../../src/entity/ChannelDigest';
34
import { ChannelHighlightDefinition } from '../../src/entity/ChannelHighlightDefinition';
45
import { ChannelHighlightRun } from '../../src/entity/ChannelHighlightRun';
6+
import { AGENTS_DIGEST_SOURCE } from '../../src/entity/Source';
57
import {
68
PostHighlight,
79
PostHighlightSignificance,
@@ -123,19 +125,20 @@ describe('generateChannelHighlight worker', () => {
123125
await deleteKeysByPattern('channel-highlight:*');
124126
await con.getRepository(ChannelHighlightRun).clear();
125127
await con.getRepository(ChannelHighlightDefinition).clear();
128+
await con.getRepository(ChannelDigest).clear();
126129
await con.getRepository(PostHighlight).clear();
127130
await con.getRepository(PostRelation).clear();
128131
await con
129132
.createQueryBuilder()
130133
.delete()
131134
.from('post')
132135
.where('"sourceId" IN (:...sourceIds)', {
133-
sourceIds: ['content-source', 'secondary-source'],
136+
sourceIds: ['content-source', 'secondary-source', AGENTS_DIGEST_SOURCE],
134137
})
135138
.execute();
136139
await con
137140
.getRepository(Source)
138-
.delete(['content-source', 'secondary-source']);
141+
.delete(['content-source', 'secondary-source', AGENTS_DIGEST_SOURCE]);
139142
});
140143

141144
it('should be registered', () => {
@@ -379,6 +382,174 @@ describe('generateChannelHighlight worker', () => {
379382
expect(retiredHighlight?.retiredAt).toBeInstanceOf(Date);
380383
});
381384

385+
it('should exclude retired highlights from candidates and keep them retired', async () => {
386+
const now = new Date('2026-03-03T11:45:00.000Z');
387+
await con.getRepository(ChannelHighlightDefinition).save({
388+
channel: 'vibes',
389+
mode: 'publish',
390+
candidateHorizonHours: 72,
391+
maxItems: 3,
392+
});
393+
await saveArticle({
394+
id: 'retired-1',
395+
title: 'Previously highlighted story',
396+
createdAt: new Date('2026-03-03T11:15:00.000Z'),
397+
});
398+
await saveArticle({
399+
id: 'fresh-1',
400+
title: 'Fresh candidate',
401+
createdAt: new Date('2026-03-03T11:20:00.000Z'),
402+
});
403+
await con.getRepository(PostHighlight).save({
404+
channel: 'vibes',
405+
postId: 'retired-1',
406+
highlightedAt: new Date('2026-03-03T11:00:00.000Z'),
407+
headline: 'Previously highlighted headline',
408+
significance: PostHighlightSignificance.Major,
409+
reason: 'previous run',
410+
retiredAt: new Date('2026-03-03T11:10:00.000Z'),
411+
});
412+
413+
const evaluatorSpy = jest
414+
.spyOn(evaluator, 'evaluateChannelHighlights')
415+
.mockResolvedValue({
416+
items: [
417+
{
418+
postId: 'fresh-1',
419+
headline: 'Fresh headline',
420+
significanceLabel: 'breaking',
421+
reason: 'test',
422+
},
423+
],
424+
});
425+
426+
await expectSuccessfulTypedBackground<'api.v1.generate-channel-highlight'>(
427+
worker,
428+
{
429+
channel: 'vibes',
430+
scheduledAt: now.toISOString(),
431+
},
432+
);
433+
434+
expect(evaluatorSpy).toHaveBeenCalledTimes(1);
435+
expect(evaluatorSpy.mock.calls[0][0].newCandidates).toEqual([
436+
expect.objectContaining({
437+
postId: 'fresh-1',
438+
title: 'Fresh candidate',
439+
}),
440+
]);
441+
442+
const liveHighlights = await con.getRepository(PostHighlight).find({
443+
where: { channel: 'vibes', retiredAt: IsNull() },
444+
});
445+
expect(liveHighlights).toEqual([
446+
expect.objectContaining({
447+
postId: 'fresh-1',
448+
headline: 'Fresh headline',
449+
}),
450+
]);
451+
452+
const retiredHighlights = await con.getRepository(PostHighlight).find({
453+
where: { channel: 'vibes', postId: 'retired-1' },
454+
});
455+
expect(retiredHighlights).toHaveLength(1);
456+
expect(retiredHighlights[0].retiredAt).toBeInstanceOf(Date);
457+
});
458+
459+
it('should ignore posts from channel digest sources for highlights', async () => {
460+
const now = new Date('2026-03-03T11:50:00.000Z');
461+
await con
462+
.getRepository(Source)
463+
.save(
464+
createSource(
465+
AGENTS_DIGEST_SOURCE,
466+
'Agents Digest',
467+
'https://daily.dev/agents.png',
468+
),
469+
);
470+
await con.getRepository(ChannelDigest).save({
471+
key: 'agentic',
472+
sourceId: AGENTS_DIGEST_SOURCE,
473+
channel: 'vibes',
474+
targetAudience: 'Digest readers',
475+
frequency: 'daily',
476+
includeSentiment: false,
477+
sentimentGroupIds: [],
478+
enabled: true,
479+
});
480+
await con.getRepository(ChannelHighlightDefinition).save({
481+
channel: 'vibes',
482+
mode: 'publish',
483+
candidateHorizonHours: 72,
484+
maxItems: 3,
485+
});
486+
await saveArticle({
487+
id: 'digest-post',
488+
sourceId: AGENTS_DIGEST_SOURCE,
489+
title: 'Digest source post',
490+
createdAt: new Date('2026-03-03T11:20:00.000Z'),
491+
});
492+
await saveArticle({
493+
id: 'fresh-1',
494+
title: 'Fresh candidate',
495+
createdAt: new Date('2026-03-03T11:25:00.000Z'),
496+
});
497+
await con.getRepository(PostHighlight).save({
498+
channel: 'vibes',
499+
postId: 'digest-post',
500+
highlightedAt: new Date('2026-03-03T11:10:00.000Z'),
501+
headline: 'Digest highlight',
502+
significance: PostHighlightSignificance.Major,
503+
reason: 'existing',
504+
});
505+
506+
const evaluatorSpy = jest
507+
.spyOn(evaluator, 'evaluateChannelHighlights')
508+
.mockResolvedValue({
509+
items: [
510+
{
511+
postId: 'fresh-1',
512+
headline: 'Fresh headline',
513+
significanceLabel: 'major',
514+
reason: 'test',
515+
},
516+
],
517+
});
518+
519+
await expectSuccessfulTypedBackground<'api.v1.generate-channel-highlight'>(
520+
worker,
521+
{
522+
channel: 'vibes',
523+
scheduledAt: now.toISOString(),
524+
},
525+
);
526+
527+
expect(evaluatorSpy).toHaveBeenCalledTimes(1);
528+
expect(evaluatorSpy.mock.calls[0][0].newCandidates).toEqual([
529+
expect.objectContaining({
530+
postId: 'fresh-1',
531+
}),
532+
]);
533+
534+
const liveHighlights = await con.getRepository(PostHighlight).find({
535+
where: { channel: 'vibes', retiredAt: IsNull() },
536+
});
537+
expect(liveHighlights).toEqual([
538+
expect.objectContaining({
539+
postId: 'fresh-1',
540+
headline: 'Fresh headline',
541+
}),
542+
]);
543+
544+
const retiredDigestHighlight = await con
545+
.getRepository(PostHighlight)
546+
.findOneByOrFail({
547+
channel: 'vibes',
548+
postId: 'digest-post',
549+
});
550+
expect(retiredDigestHighlight.retiredAt).toBeInstanceOf(Date);
551+
});
552+
382553
it('should remove highlights that aged past the configured horizon', async () => {
383554
const now = new Date('2026-03-03T12:00:00.000Z');
384555
await con.getRepository(ChannelHighlightDefinition).save({

‎src/common/channelDigest/definitions.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,28 @@ export const getChannelDigestDefinitionByKey = async ({
3535
}),
3636
);
3737

38+
export const getChannelDigestSourceIds = async ({
39+
con,
40+
}: {
41+
con: DataSource;
42+
}): Promise<string[]> => {
43+
const definitions = await queryReadReplica(con, ({ queryRunner }) =>
44+
queryRunner.manager.getRepository(ChannelDigest).find({
45+
select: {
46+
sourceId: true,
47+
},
48+
where: {
49+
enabled: true,
50+
},
51+
order: {
52+
sourceId: 'ASC',
53+
},
54+
}),
55+
);
56+
57+
return definitions.map((definition) => definition.sourceId);
58+
};
59+
3860
export const isChannelDigestScheduledForDate = ({
3961
definition,
4062
now,

‎src/common/channelDigest/generate.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,10 @@ import {
1515
PostRelation,
1616
PostRelationType,
1717
} from '../../entity/posts/PostRelation';
18-
import { getChannelDigestLookbackSeconds } from './definitions';
18+
import {
19+
getChannelDigestLookbackSeconds,
20+
getChannelDigestSourceIds,
21+
} from './definitions';
1922

2023
type DigestPostRow = {
2124
title: string | null;
@@ -76,10 +79,12 @@ const findDigestPosts = async ({
7679
con,
7780
from,
7881
channel,
82+
excludedSourceIds,
7983
}: {
8084
con: DataSource;
8185
from: Date;
8286
channel: string;
87+
excludedSourceIds: string[];
8388
}): Promise<DigestPostRow[]> => {
8489
if (!channel) {
8590
return [];
@@ -104,6 +109,12 @@ const findDigestPosts = async ({
104109
.andWhere(`(post."contentMeta"->'channels') ? :channel`, {
105110
channel,
106111
})
112+
.andWhere(
113+
excludedSourceIds.length
114+
? 'post."sourceId" NOT IN (:...excludedSourceIds)'
115+
: '1=1',
116+
{ excludedSourceIds },
117+
)
107118
.andWhere('relation."relatedPostId" IS NULL')
108119
.orderBy('post.createdAt', 'DESC')
109120
.getRawMany<DigestPostRow>();
@@ -204,6 +215,9 @@ export const generateChannelDigest = async ({
204215
now,
205216
definition,
206217
});
218+
const excludedSourceIds = await getChannelDigestSourceIds({
219+
con,
220+
});
207221
const [sentimentItems, posts] = await Promise.all([
208222
findSentimentItems({
209223
definition,
@@ -214,6 +228,7 @@ export const generateChannelDigest = async ({
214228
con,
215229
from,
216230
channel: definition.channel,
231+
excludedSourceIds,
217232
}),
218233
]);
219234

0 commit comments

Comments
 (0)