Skip to content

Commit bbda8d7

Browse files
authored
feat: publish canonical highlight events (#3953)
1 parent f1668d7 commit bbda8d7

11 files changed

Lines changed: 390 additions & 8 deletions

File tree

.infra/application.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ debezium.source.database.user=%database_user%
88
debezium.source.database.password=%database_pass%
99
debezium.source.database.dbname=%database_dbname%
1010
debezium.source.database.server.name=api
11-
debezium.source.table.include.list=public.comment,public.user_comment,public.comment_mention,public.source_request,public.post,public.user,public.post_report,public.source_feed,public.settings,public.reputation_event,public.submission,public.user_state,public.notification_v2,public.source_member,public.feature,public.source,public.post_mention,public.content_image,public.comment_report,public.user_post,public.banner,public.post_relation,public.marketing_cta,public.squad_public_request,public.user_streak,public.bookmark,public.bookmark_list,public.user_company,public.source_report,public.user_top_reader,public.source_post_moderation,public.user_report,public.user_transaction,public.content_preference,public.campaign,public.opportunity_match,public.opportunity,public.organization,public.user_candidate_preference,public.user_experience,public.feedback,public.hot_take,public.user_stack,public.quest,public.quest_reward,public.quest_rotation,public.user_quest,public.user_quest_profile,public.feed,public.opportunity_user,public.user_marketing_cta
11+
debezium.source.table.include.list=public.comment,public.user_comment,public.comment_mention,public.source_request,public.post,public.user,public.post_report,public.source_feed,public.settings,public.reputation_event,public.submission,public.user_state,public.notification_v2,public.source_member,public.feature,public.source,public.post_mention,public.content_image,public.comment_report,public.user_post,public.banner,public.post_relation,public.marketing_cta,public.squad_public_request,public.user_streak,public.bookmark,public.bookmark_list,public.user_company,public.source_report,public.user_top_reader,public.source_post_moderation,public.user_report,public.user_transaction,public.content_preference,public.campaign,public.opportunity_match,public.opportunity,public.organization,public.user_candidate_preference,public.user_experience,public.feedback,public.hot_take,public.user_stack,public.quest,public.quest_reward,public.quest_rotation,public.user_quest,public.user_quest_profile,public.highlights_canonical,public.feed,public.opportunity_user,public.user_marketing_cta
1212
debezium.source.column.exclude.list=public.post.tsv,public.post.placeholder,public.source.flags,public.user_top_reader.image
1313
debezium.source.skip.messages.without.change=true
1414
debezium.source.plugin.name=pgoutput

.infra/common.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,10 @@ export const workers: Worker[] = [
154154
topic: 'api.v1.new-notification',
155155
subscription: 'api.new-notification-push',
156156
},
157+
{
158+
topic: 'api.v1.post-highlighted',
159+
subscription: 'api.new-highlight-real-time',
160+
},
157161
{
158162
topic: 'api.v1.source-privacy-updated',
159163
subscription: 'api.source-privacy-updated',

__tests__/workers/cdc/primary.ts

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import nock from 'nock';
22
import {
33
CandidateStatus,
4+
HighlightsCanonicalPublishedMessage,
45
OpportunityState,
56
OpportunityType,
67
} from '@dailydotdev/schema';
@@ -29,6 +30,7 @@ import {
2930
FREEFORM_POST_MINIMUM_CHANGE_LENGTH,
3031
FREEFORM_POST_MINIMUM_CONTENT_LENGTH,
3132
FreeformPost,
33+
HighlightsCanonical,
3234
Keyword,
3335
KeywordStatus,
3436
MarketingCta,
@@ -138,6 +140,7 @@ import { SourceMemberRoles } from '../../../src/roles';
138140
import { CommentReport } from '../../../src/entity/CommentReport';
139141
import { badUsersFixture, usersFixture } from '../../fixture/user';
140142
import { DEFAULT_DEV_CARD_UNLOCKED_THRESHOLD } from '../../../src/workers/notifications/devCardUnlocked';
143+
import { HighlightSignificance } from '../../../src/common/channelHighlight/significance';
141144
import { UserComment } from '../../../src/entity/user/UserComment';
142145
import { Product, ProductType } from '../../../src/entity/Product';
143146
import {
@@ -3215,6 +3218,104 @@ describe('marketing cta', () => {
32153218
});
32163219
});
32173220

3221+
describe('highlights canonical', () => {
3222+
type ObjectType = HighlightsCanonical;
3223+
3224+
const base: ChangeObject<ObjectType> = {
3225+
id: 'hc_1',
3226+
postId: 'p1',
3227+
channels: ['javascript'],
3228+
highlightedAt: 1_770_000_000_000_000,
3229+
headline: 'JavaScript in 2026',
3230+
significance: HighlightSignificance.Major,
3231+
reason: 'Fast ecosystem update',
3232+
createdAt: 1_770_000_000_000_000,
3233+
updatedAt: 1_770_000_000_000_000,
3234+
};
3235+
3236+
beforeEach(() => {
3237+
jest.clearAllMocks();
3238+
});
3239+
3240+
it('should publish canonical highlight event on create', async () => {
3241+
await expectSuccessfulBackground(
3242+
worker,
3243+
mockChangeMessage<ObjectType>({
3244+
after: base,
3245+
before: null,
3246+
op: 'c',
3247+
table: 'highlights_canonical',
3248+
}),
3249+
);
3250+
3251+
expectTypedEvent(
3252+
'api.v1.post-highlighted',
3253+
new HighlightsCanonicalPublishedMessage({
3254+
highlightId: base.id,
3255+
channels: base.channels,
3256+
publishedChannels: base.channels,
3257+
postId: base.postId,
3258+
headline: base.headline,
3259+
significance: base.significance,
3260+
reason: base.reason ?? undefined,
3261+
highlightedAt: base.highlightedAt,
3262+
}),
3263+
);
3264+
});
3265+
3266+
it('should publish only newly added channels on update', async () => {
3267+
const after: ChangeObject<ObjectType> = {
3268+
...base,
3269+
channels: ['javascript', 'backend'],
3270+
updatedAt: 1_770_000_100_000_000,
3271+
};
3272+
3273+
await expectSuccessfulBackground(
3274+
worker,
3275+
mockChangeMessage<ObjectType>({
3276+
after,
3277+
before: base,
3278+
op: 'u',
3279+
table: 'highlights_canonical',
3280+
}),
3281+
);
3282+
3283+
expectTypedEvent(
3284+
'api.v1.post-highlighted',
3285+
new HighlightsCanonicalPublishedMessage({
3286+
highlightId: after.id,
3287+
channels: after.channels,
3288+
publishedChannels: ['backend'],
3289+
postId: after.postId,
3290+
headline: after.headline,
3291+
significance: after.significance,
3292+
reason: after.reason ?? undefined,
3293+
highlightedAt: after.highlightedAt,
3294+
}),
3295+
);
3296+
});
3297+
3298+
it('should not publish on update without a new channel', async () => {
3299+
const after: ChangeObject<ObjectType> = {
3300+
...base,
3301+
headline: 'Updated headline',
3302+
updatedAt: 1_770_000_100_000_000,
3303+
};
3304+
3305+
await expectSuccessfulBackground(
3306+
worker,
3307+
mockChangeMessage<ObjectType>({
3308+
after,
3309+
before: base,
3310+
op: 'u',
3311+
table: 'highlights_canonical',
3312+
}),
3313+
);
3314+
3315+
expect(triggerTypedEvent).not.toHaveBeenCalled();
3316+
});
3317+
});
3318+
32183319
describe('squad public request', () => {
32193320
type ObjectType = SquadPublicRequest;
32203321
const base: ChangeObject<ObjectType> = {
Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
import { HighlightsCanonicalPublishedMessage } from '@dailydotdev/schema';
2+
import { DataSource } from 'typeorm';
3+
import { HighlightSignificance } from '../../src/common/channelHighlight/significance';
4+
import { NEW_HIGHLIGHT_CHANNEL } from '../../src/common/highlights';
5+
import { HighlightsCanonical } from '../../src/entity/HighlightsCanonical';
6+
import { Source, SourceType } from '../../src/entity/Source';
7+
import { ArticlePost } from '../../src/entity/posts/ArticlePost';
8+
import { PostType } from '../../src/entity/posts/Post';
9+
import { redisPubSub } from '../../src/redis';
10+
import worker from '../../src/workers/newHighlightRealTime';
11+
import { typedWorkers } from '../../src/workers';
12+
import createOrGetConnection from '../../src/db';
13+
import { expectSuccessfulTypedBackground } from '../helpers';
14+
15+
let con: DataSource;
16+
17+
beforeAll(async () => {
18+
con = await createOrGetConnection();
19+
});
20+
21+
describe('newHighlightRealTime worker', () => {
22+
beforeEach(async () => {
23+
jest.resetAllMocks();
24+
25+
await con.getRepository(Source).save({
26+
id: 'highlight-source',
27+
name: 'Highlight Source',
28+
image: 'https://example.com/highlight.png',
29+
handle: 'highlight-source',
30+
type: SourceType.Machine,
31+
active: true,
32+
private: false,
33+
});
34+
});
35+
36+
afterEach(async () => {
37+
if (!con) {
38+
return;
39+
}
40+
41+
await con.getRepository(HighlightsCanonical).clear();
42+
await con.getRepository(ArticlePost).delete(['highlight-post']);
43+
await con.getRepository(Source).delete(['highlight-source']);
44+
});
45+
46+
it('should be registered', () => {
47+
const registeredWorker = typedWorkers.find(
48+
(item) => item.subscription === worker.subscription,
49+
);
50+
51+
expect(registeredWorker).toBeDefined();
52+
});
53+
54+
it('should publish full highlights to the broadcast redis channel', async () => {
55+
await con.getRepository(ArticlePost).save({
56+
id: 'highlight-post',
57+
shortId: 'highlight-post',
58+
title: 'Highlight post',
59+
url: 'https://example.com/highlight-post',
60+
canonicalUrl: 'https://example.com/highlight-post',
61+
score: 0,
62+
sourceId: 'highlight-source',
63+
visible: true,
64+
deleted: false,
65+
banned: false,
66+
showOnFeed: true,
67+
createdAt: new Date('2026-04-26T10:00:00.000Z'),
68+
metadataChangedAt: new Date('2026-04-26T10:00:00.000Z'),
69+
type: PostType.Article,
70+
});
71+
72+
const highlight = await con.getRepository(HighlightsCanonical).save({
73+
postId: 'highlight-post',
74+
channels: ['backend', 'javascript', 'vibes'],
75+
highlightedAt: new Date('2026-04-26T10:05:00.000Z'),
76+
headline: 'New canonical highlight',
77+
significance: HighlightSignificance.Major,
78+
});
79+
const publishSpy = jest.spyOn(redisPubSub, 'publish');
80+
81+
await expectSuccessfulTypedBackground<'api.v1.post-highlighted'>(
82+
worker,
83+
new HighlightsCanonicalPublishedMessage({
84+
highlightId: highlight.id,
85+
channels: highlight.channels,
86+
publishedChannels: ['backend', 'javascript'],
87+
}),
88+
);
89+
90+
expect(publishSpy).toHaveBeenCalledTimes(2);
91+
for (const channel of ['backend', 'javascript']) {
92+
expect(publishSpy).toHaveBeenCalledWith(
93+
NEW_HIGHLIGHT_CHANNEL,
94+
expect.objectContaining({
95+
id: highlight.id,
96+
postId: 'highlight-post',
97+
channel,
98+
channels: ['backend', 'javascript', 'vibes'],
99+
headline: 'New canonical highlight',
100+
significance: 'major',
101+
highlightedAt: highlight.highlightedAt,
102+
createdAt: highlight.createdAt,
103+
updatedAt: highlight.updatedAt,
104+
post: expect.objectContaining({
105+
id: 'highlight-post',
106+
title: 'Highlight post',
107+
source: expect.objectContaining({
108+
id: 'highlight-source',
109+
handle: 'highlight-source',
110+
}),
111+
}),
112+
}),
113+
);
114+
}
115+
});
116+
117+
it('should fallback to canonical channels when published channels are missing', async () => {
118+
await con.getRepository(ArticlePost).save({
119+
id: 'highlight-post',
120+
shortId: 'highlight-post',
121+
title: 'Highlight post',
122+
url: 'https://example.com/highlight-post',
123+
canonicalUrl: 'https://example.com/highlight-post',
124+
score: 0,
125+
sourceId: 'highlight-source',
126+
visible: true,
127+
deleted: false,
128+
banned: false,
129+
showOnFeed: true,
130+
createdAt: new Date('2026-04-26T10:00:00.000Z'),
131+
metadataChangedAt: new Date('2026-04-26T10:00:00.000Z'),
132+
type: PostType.Article,
133+
});
134+
135+
const highlight = await con.getRepository(HighlightsCanonical).save({
136+
postId: 'highlight-post',
137+
channels: ['backend'],
138+
highlightedAt: new Date('2026-04-26T10:05:00.000Z'),
139+
headline: 'New canonical highlight',
140+
significance: HighlightSignificance.Major,
141+
});
142+
const publishSpy = jest.spyOn(redisPubSub, 'publish');
143+
144+
await expectSuccessfulTypedBackground<'api.v1.post-highlighted'>(
145+
worker,
146+
new HighlightsCanonicalPublishedMessage({
147+
highlightId: highlight.id,
148+
channels: ['backend'],
149+
}),
150+
);
151+
152+
expect(publishSpy).toHaveBeenCalledWith(
153+
NEW_HIGHLIGHT_CHANNEL,
154+
expect.objectContaining({
155+
id: highlight.id,
156+
channel: 'backend',
157+
}),
158+
);
159+
});
160+
});

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
"@connectrpc/connect-fastify": "1.6.1",
3939
"@connectrpc/connect-node": "1.6.1",
4040
"@dailydotdev/graphql-redis-subscriptions": "2.4.3",
41-
"@dailydotdev/schema": "0.3.10",
41+
"@dailydotdev/schema": "0.3.11",
4242
"@dailydotdev/ts-ioredis-pool": "1.0.2",
4343
"@fastify/cookie": "11.0.2",
4444
"@fastify/cors": "11.2.0",

pnpm-lock.yaml

Lines changed: 5 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pnpm-workspace.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# Delay new package versions for 7 days before pnpm may resolve them.
22
minimumReleaseAge: 10080
3+
minimumReleaseAgeExclude:
4+
- '@dailydotdev/schema'
35
blockExoticSubdeps: true
46

57
patchedDependencies:

src/common/typedPubsub.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import {
3434
CandidatePreferenceUpdated,
3535
CandidateRejectedOpportunityMessage,
3636
ContentUpdatedMessage,
37+
type HighlightsCanonicalPublishedMessage,
3738
MatchedCandidate,
3839
type OpportunityMessage,
3940
type OpportunityPreviewResult,
@@ -209,6 +210,7 @@ export type PubSubSchema = {
209210
};
210211
'skadi.v2.campaign-updated': CampaignUpdateEventArgs;
211212
'api.v1.post-metrics-updated': z.infer<typeof postMetricsUpdatedTopic>;
213+
'api.v1.post-highlighted': HighlightsCanonicalPublishedMessage;
212214
'api.v1.reputation-event': {
213215
op: ChangeMessage<unknown>['payload']['op'];
214216
payload: ChangeObject<ReputationEvent>;

0 commit comments

Comments
 (0)