Skip to content

fix: ignore channel digest sources in digest and highlight generation #3732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
idoshamun merged 1 commit into from
Mar 23, 2026
Merged

fix: ignore channel digest sources in digest and highlight generation #3732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions __tests__/workers/generateChannelDigest.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,21 @@ const getLockKey = (digestKey: string, scheduledAt: string) =>

describe('generateChannelDigest worker', () => {
afterEach(async () => {
jest.restoreAllMocks();
nock.cleanAll();
await deleteKeysByPattern('channel-digest:*');
await con.getRepository(ChannelDigest).clear();
await con
.createQueryBuilder()
.delete()
.from('post')
.where('"sourceId" IN (:...sourceIds)', {
sourceIds: ['content-source', AGENTS_DIGEST_SOURCE, 'weekly-source'],
})
.execute();
await con
.getRepository(Source)
.delete(['content-source', AGENTS_DIGEST_SOURCE, 'weekly-source']);
});

it('should be registered', () => {
Expand Down Expand Up @@ -270,6 +283,70 @@ describe('generateChannelDigest worker', () => {
).toBeGreaterThan(ONE_DAY_IN_SECONDS);
});

it('should ignore posts from all channel digest sources when generating digests', async () => {
const scheduledAt = '2026-03-03T10:00:00.000Z';
await con
.getRepository(Source)
.save([
createSource(
AGENTS_DIGEST_SOURCE,
'Agents Digest',
'https://daily.dev/agents.png',
),
createSource(
'weekly-source',
'Weekly Digest',
'https://daily.dev/weekly.png',
),
]);
await saveDefinition({
key: 'agentic',
sourceId: AGENTS_DIGEST_SOURCE,
channel: 'vibes',
});
await saveDefinition({
key: 'weekly-test',
sourceId: 'weekly-source',
channel: 'weekly',
frequency: 'weekly',
});
await savePost({
id: 'agents-post',
sourceId: AGENTS_DIGEST_SOURCE,
title: 'Agents digest post',
content: 'Agents digest body',
createdAt: new Date('2026-03-03T09:10:00.000Z'),
channel: 'vibes',
});
await savePost({
id: 'weekly-post',
sourceId: 'weekly-source',
title: 'Weekly digest post',
content: 'Weekly digest body',
createdAt: new Date('2026-03-03T09:20:00.000Z'),
channel: 'vibes',
});
const digestCountBefore = await con.getRepository(FreeformPost).countBy({
sourceId: AGENTS_DIGEST_SOURCE,
});

await expectSuccessfulTypedBackground<'api.v1.generate-channel-digest'>(
worker,
{
digestKey: 'agentic',
scheduledAt,
},
);

const digests = await con.getRepository(FreeformPost).findBy({
sourceId: AGENTS_DIGEST_SOURCE,
});
expect(digests).toHaveLength(digestCountBefore);
expect(
digests.find((digest) => digest.title === 'Mock sentiment digest'),
).toBeUndefined();
});

it('should use a weekly done ttl for weekly digests', async () => {
const scheduledAt = '2026-03-02T10:00:00.000Z';
await con
Expand Down
175 changes: 173 additions & 2 deletions __tests__/workers/generateChannelHighlight.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import { IsNull, type DataSource } from 'typeorm';
import createOrGetConnection from '../../src/db';
import { ChannelDigest } from '../../src/entity/ChannelDigest';
import { ChannelHighlightDefinition } from '../../src/entity/ChannelHighlightDefinition';
import { ChannelHighlightRun } from '../../src/entity/ChannelHighlightRun';
import { AGENTS_DIGEST_SOURCE } from '../../src/entity/Source';
import {
PostHighlight,
PostHighlightSignificance,
Expand Down Expand Up @@ -123,19 +125,20 @@ describe('generateChannelHighlight worker', () => {
await deleteKeysByPattern('channel-highlight:*');
await con.getRepository(ChannelHighlightRun).clear();
await con.getRepository(ChannelHighlightDefinition).clear();
await con.getRepository(ChannelDigest).clear();
await con.getRepository(PostHighlight).clear();
await con.getRepository(PostRelation).clear();
await con
.createQueryBuilder()
.delete()
.from('post')
.where('"sourceId" IN (:...sourceIds)', {
sourceIds: ['content-source', 'secondary-source'],
sourceIds: ['content-source', 'secondary-source', AGENTS_DIGEST_SOURCE],
})
.execute();
await con
.getRepository(Source)
.delete(['content-source', 'secondary-source']);
.delete(['content-source', 'secondary-source', AGENTS_DIGEST_SOURCE]);
});

it('should be registered', () => {
Expand Down Expand Up @@ -379,6 +382,174 @@ describe('generateChannelHighlight worker', () => {
expect(retiredHighlight?.retiredAt).toBeInstanceOf(Date);
});

it('should exclude retired highlights from candidates and keep them retired', async () => {
const now = new Date('2026-03-03T11:45:00.000Z');
await con.getRepository(ChannelHighlightDefinition).save({
channel: 'vibes',
mode: 'publish',
candidateHorizonHours: 72,
maxItems: 3,
});
await saveArticle({
id: 'retired-1',
title: 'Previously highlighted story',
createdAt: new Date('2026-03-03T11:15:00.000Z'),
});
await saveArticle({
id: 'fresh-1',
title: 'Fresh candidate',
createdAt: new Date('2026-03-03T11:20:00.000Z'),
});
await con.getRepository(PostHighlight).save({
channel: 'vibes',
postId: 'retired-1',
highlightedAt: new Date('2026-03-03T11:00:00.000Z'),
headline: 'Previously highlighted headline',
significance: PostHighlightSignificance.Major,
reason: 'previous run',
retiredAt: new Date('2026-03-03T11:10:00.000Z'),
});

const evaluatorSpy = jest
.spyOn(evaluator, 'evaluateChannelHighlights')
.mockResolvedValue({
items: [
{
postId: 'fresh-1',
headline: 'Fresh headline',
significanceLabel: 'breaking',
reason: 'test',
},
],
});

await expectSuccessfulTypedBackground<'api.v1.generate-channel-highlight'>(
worker,
{
channel: 'vibes',
scheduledAt: now.toISOString(),
},
);

expect(evaluatorSpy).toHaveBeenCalledTimes(1);
expect(evaluatorSpy.mock.calls[0][0].newCandidates).toEqual([
expect.objectContaining({
postId: 'fresh-1',
title: 'Fresh candidate',
}),
]);

const liveHighlights = await con.getRepository(PostHighlight).find({
where: { channel: 'vibes', retiredAt: IsNull() },
});
expect(liveHighlights).toEqual([
expect.objectContaining({
postId: 'fresh-1',
headline: 'Fresh headline',
}),
]);

const retiredHighlights = await con.getRepository(PostHighlight).find({
where: { channel: 'vibes', postId: 'retired-1' },
});
expect(retiredHighlights).toHaveLength(1);
expect(retiredHighlights[0].retiredAt).toBeInstanceOf(Date);
});

it('should ignore posts from channel digest sources for highlights', async () => {
const now = new Date('2026-03-03T11:50:00.000Z');
await con
.getRepository(Source)
.save(
createSource(
AGENTS_DIGEST_SOURCE,
'Agents Digest',
'https://daily.dev/agents.png',
),
);
await con.getRepository(ChannelDigest).save({
key: 'agentic',
sourceId: AGENTS_DIGEST_SOURCE,
channel: 'vibes',
targetAudience: 'Digest readers',
frequency: 'daily',
includeSentiment: false,
sentimentGroupIds: [],
enabled: true,
});
await con.getRepository(ChannelHighlightDefinition).save({
channel: 'vibes',
mode: 'publish',
candidateHorizonHours: 72,
maxItems: 3,
});
await saveArticle({
id: 'digest-post',
sourceId: AGENTS_DIGEST_SOURCE,
title: 'Digest source post',
createdAt: new Date('2026-03-03T11:20:00.000Z'),
});
await saveArticle({
id: 'fresh-1',
title: 'Fresh candidate',
createdAt: new Date('2026-03-03T11:25:00.000Z'),
});
await con.getRepository(PostHighlight).save({
channel: 'vibes',
postId: 'digest-post',
highlightedAt: new Date('2026-03-03T11:10:00.000Z'),
headline: 'Digest highlight',
significance: PostHighlightSignificance.Major,
reason: 'existing',
});

const evaluatorSpy = jest
.spyOn(evaluator, 'evaluateChannelHighlights')
.mockResolvedValue({
items: [
{
postId: 'fresh-1',
headline: 'Fresh headline',
significanceLabel: 'major',
reason: 'test',
},
],
});

await expectSuccessfulTypedBackground<'api.v1.generate-channel-highlight'>(
worker,
{
channel: 'vibes',
scheduledAt: now.toISOString(),
},
);

expect(evaluatorSpy).toHaveBeenCalledTimes(1);
expect(evaluatorSpy.mock.calls[0][0].newCandidates).toEqual([
expect.objectContaining({
postId: 'fresh-1',
}),
]);

const liveHighlights = await con.getRepository(PostHighlight).find({
where: { channel: 'vibes', retiredAt: IsNull() },
});
expect(liveHighlights).toEqual([
expect.objectContaining({
postId: 'fresh-1',
headline: 'Fresh headline',
}),
]);

const retiredDigestHighlight = await con
.getRepository(PostHighlight)
.findOneByOrFail({
channel: 'vibes',
postId: 'digest-post',
});
expect(retiredDigestHighlight.retiredAt).toBeInstanceOf(Date);
});

it('should remove highlights that aged past the configured horizon', async () => {
const now = new Date('2026-03-03T12:00:00.000Z');
await con.getRepository(ChannelHighlightDefinition).save({
Expand Down
22 changes: 22 additions & 0 deletions src/common/channelDigest/definitions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,28 @@ export const getChannelDigestDefinitionByKey = async ({
}),
);

export const getChannelDigestSourceIds = async ({
con,
}: {
con: DataSource;
}): Promise<string[]> => {
const definitions = await queryReadReplica(con, ({ queryRunner }) =>
queryRunner.manager.getRepository(ChannelDigest).find({
select: {
sourceId: true,
},
where: {
enabled: true,
},
order: {
sourceId: 'ASC',
},
}),
);

return definitions.map((definition) => definition.sourceId);
};

export const isChannelDigestScheduledForDate = ({
definition,
now,
Expand Down
17 changes: 16 additions & 1 deletion src/common/channelDigest/generate.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@ import {
PostRelation,
PostRelationType,
} from '../../entity/posts/PostRelation';
import { getChannelDigestLookbackSeconds } from './definitions';
import {
getChannelDigestLookbackSeconds,
getChannelDigestSourceIds,
} from './definitions';

type DigestPostRow = {
title: string | null;
Expand Down Expand Up @@ -76,10 +79,12 @@ const findDigestPosts = async ({
con,
from,
channel,
excludedSourceIds,
}: {
con: DataSource;
from: Date;
channel: string;
excludedSourceIds: string[];
}): Promise<DigestPostRow[]> => {
if (!channel) {
return [];
Expand All @@ -104,6 +109,12 @@ const findDigestPosts = async ({
.andWhere(`(post."contentMeta"->'channels') ? :channel`, {
channel,
})
.andWhere(
excludedSourceIds.length
? 'post."sourceId" NOT IN (:...excludedSourceIds)'
: '1=1',
{ excludedSourceIds },
)
.andWhere('relation."relatedPostId" IS NULL')
.orderBy('post.createdAt', 'DESC')
.getRawMany<DigestPostRow>();
Expand Down Expand Up @@ -204,6 +215,9 @@ export const generateChannelDigest = async ({
now,
definition,
});
const excludedSourceIds = await getChannelDigestSourceIds({
con,
});
const [sentimentItems, posts] = await Promise.all([
findSentimentItems({
definition,
Expand All @@ -214,6 +228,7 @@ export const generateChannelDigest = async ({
con,
from,
channel: definition.channel,
excludedSourceIds,
}),
]);

Expand Down
Loading
Loading