feat: add highlights sitemap (#3795)

Author: idoshamun

__tests__/sitemaps.ts

@@ -33,6 +33,8 @@ import { updateFlagsStatement } from '../src/common/utils';
 import { sourcesFixture } from './fixture/source';
 import { keywordsFixture } from './fixture/keywords';
 import { ONE_DAY_IN_SECONDS } from '../src/common/constants';
+import { ChannelHighlightDefinition } from '../src/entity/ChannelHighlightDefinition';
+import { PostHighlight } from '../src/entity/PostHighlight';
 let app: FastifyInstance;
 let con: DataSource;
 const previousSitemapLimit = process.env.SITEMAP_LIMIT;
@@ -150,6 +152,7 @@ beforeAll(async () => {
 
 beforeEach(async () => {
   nock.cleanAll();
+  await con.getRepository(ChannelHighlightDefinition).clear();
   await saveFixtures(con, SentimentGroup, sentimentGroupsFixture);
   await saveFixtures(con, SentimentEntity, sentimentEntitiesFixture);
   await saveFixtures(con, Keyword, keywordsFixture);
@@ -524,6 +527,9 @@ describe('GET /sitemaps/index.xml', () => {
     expect(res.text).toContain(
       '<loc>http://localhost:5002/api/sitemaps/collections.xml</loc>',
     );
+    expect(res.text).toContain(
+      '<loc>http://localhost:5002/api/sitemaps/highlights.xml</loc>',
+    );
     expect(res.text).toContain(
       '<loc>http://localhost:5002/api/sitemaps/agents.xml</loc>',
     );
@@ -545,6 +551,86 @@ describe('GET /sitemaps/index.xml', () => {
   });
 });
 
+describe('GET /sitemaps/highlights.xml', () => {
+  it('should return the highlights sitemap with latest live highlight lastmod per channel', async () => {
+    await con.getRepository(ChannelHighlightDefinition).save([
+      {
+        channel: 'career',
+        displayName: 'Career',
+        mode: 'shadow',
+        order: 1,
+      },
+      {
+        channel: 'backend',
+        displayName: 'Backend',
+        mode: 'publish',
+        order: 2,
+      },
+      {
+        channel: 'disabled',
+        displayName: 'Disabled',
+        mode: 'disabled',
+        order: 0,
+      },
+    ]);
+    await con.getRepository(PostHighlight).save([
+      {
+        postId: 'p1',
+        channel: 'career',
+        highlightedAt: new Date('2026-04-10T10:00:00.000Z'),
+        headline: 'Career early',
+      },
+      {
+        postId: 'p4',
+        channel: 'career',
+        highlightedAt: new Date('2026-04-12T09:00:00.000Z'),
+        headline: 'Career latest',
+      },
+      {
+        postId: 'p1',
+        channel: 'backend',
+        highlightedAt: new Date('2026-04-09T08:00:00.000Z'),
+        headline: 'Backend live',
+      },
+      {
+        postId: 'p4',
+        channel: 'backend',
+        highlightedAt: new Date('2026-04-13T08:00:00.000Z'),
+        headline: 'Backend retired',
+        retiredAt: new Date('2026-04-13T08:30:00.000Z'),
+      },
+    ]);
+
+    const res = await request(app.server)
+      .get('/sitemaps/highlights.xml')
+      .expect(200);
+
+    expect(res.header['content-type']).toContain('application/xml');
+    expect(res.header['cache-control']).toBeTruthy();
+    expect(res.text).toContain(
+      '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+    );
+    expect(res.text).toContain(
+      '<loc>http://localhost:5002/highlights</loc><lastmod>2026-04-12T09:00:00.000Z</lastmod>',
+    );
+    expect(res.text).toContain(
+      '<loc>http://localhost:5002/highlights/career</loc><lastmod>2026-04-12T09:00:00.000Z</lastmod>',
+    );
+    expect(res.text).toContain(
+      '<loc>http://localhost:5002/highlights/backend</loc><lastmod>2026-04-09T08:00:00.000Z</lastmod>',
+    );
+    expect(res.text).not.toContain('/highlights/disabled');
+    expect(res.text).not.toContain('2026-04-13T08:00:00.000Z');
+
+    expect(res.text.indexOf('/highlights</loc>')).toBeLessThan(
+      res.text.indexOf('/highlights/career</loc>'),
+    );
+    expect(res.text.indexOf('/highlights/career</loc>')).toBeLessThan(
+      res.text.indexOf('/highlights/backend</loc>'),
+    );
+  });
+});
+
 describe('GET /sitemaps/sources.xml', () => {
   it('should include only qualified public machine sources', async () => {
     const sourceCreatedAt = new Date('2023-10-01T10:00:00.000Z');

src/routes/sitemaps.ts

@@ -11,6 +11,8 @@ import {
   User,
 } from '../entity';
 import { AGENTS_DIGEST_SOURCE } from '../entity/Source';
+import { ChannelHighlightDefinition } from '../entity/ChannelHighlightDefinition';
+import { PostHighlight } from '../entity/PostHighlight';
 import { ArchivePeriodType, ArchiveScopeType } from '../common/archive';
 import { getUserProfileUrl } from '../common/users';
 import createOrGetConnection from '../db';
@@ -98,6 +100,29 @@ const getSourceSitemapUrl = (prefix: string, handle: string): string =>
 const getSquadSitemapUrl = (prefix: string, handle: string): string =>
   `${prefix}/squads/${encodeURIComponent(handle)}`;
 
+const getHighlightsSitemapUrl = (prefix: string, channel?: string): string =>
+  channel
+    ? `${prefix}/highlights/${encodeURIComponent(channel)}`
+    : `${prefix}/highlights`;
+
+type SitemapUrlEntry = {
+  url: string;
+  lastmod?: string;
+};
+
+const getSitemapUrlSetXml = (
+  entries: SitemapUrlEntry[],
+): string => `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${entries
+  .map(({ url, lastmod }) =>
+    lastmod
+      ? `  <url><loc>${escapeXml(url)}</loc><lastmod>${escapeXml(lastmod)}</lastmod></url>`
+      : `  <url><loc>${escapeXml(url)}</loc></url>`,
+  )
+  .join('\n')}
+</urlset>`;
+
 const streamReplicaQuery = async <T extends ObjectLiteral>(
   con: DataSource,
   buildQuery: (source: EntityManager) => SelectQueryBuilder<T>,
@@ -299,6 +324,26 @@ const buildCollectionsSitemapQuery = (
       .limit(DEFAULT_SITEMAP_LIMIT),
   );
 
+const buildHighlightsSitemapQuery = (
+  source: DataSource | EntityManager,
+): SelectQueryBuilder<ChannelHighlightDefinition> =>
+  source
+    .createQueryBuilder()
+    .select('chd.channel', 'channel')
+    .addSelect('MAX(ph."highlightedAt")', 'lastmod')
+    .from(ChannelHighlightDefinition, 'chd')
+    .leftJoin(
+      PostHighlight,
+      'ph',
+      'ph.channel = chd.channel AND ph."retiredAt" IS NULL',
+    )
+    .where('chd.mode != :disabledMode', { disabledMode: 'disabled' })
+    .groupBy('chd.channel')
+    .addGroupBy('chd."order"')
+    .orderBy('chd."order"', 'ASC')
+    .addOrderBy('chd.channel', 'ASC')
+    .limit(DEFAULT_SITEMAP_LIMIT);
+
 const buildTagsSitemapQuery = (
   source: DataSource | EntityManager,
 ): SelectQueryBuilder<Keyword> =>
@@ -617,6 +662,42 @@ const buildArchivePagesIndexEntries = (
     })
     .join('\n');
 
+const buildHighlightsSitemapXml = async (con: DataSource): Promise<string> => {
+  const prefix = getSitemapUrlPrefix();
+  const queryRunner = con.createQueryRunner('slave');
+
+  try {
+    const rows = await buildHighlightsSitemapQuery(
+      queryRunner.manager,
+    ).getRawMany<{ channel: string; lastmod?: string | Date | null }>();
+
+    const channelEntries = rows.map((row) => ({
+      url: getHighlightsSitemapUrl(prefix, row.channel),
+      lastmod: getSitemapRowLastmod(row),
+    }));
+    const rootLastmod = channelEntries.reduce<string | undefined>(
+      (latest, entry) => {
+        if (!entry.lastmod) {
+          return latest;
+        }
+
+        return !latest || entry.lastmod > latest ? entry.lastmod : latest;
+      },
+      undefined,
+    );
+
+    return getSitemapUrlSetXml([
+      {
+        url: getHighlightsSitemapUrl(prefix),
+        lastmod: rootLastmod,
+      },
+      ...channelEntries,
+    ]);
+  } finally {
+    await queryRunner.release();
+  }
+};
+
 const getSitemapIndexXml = (
   postsSitemapCount: number,
   evergreenSitemapCount: number,
@@ -645,6 +726,9 @@ ${evergreenSitemaps}
   <sitemap>
     <loc>${escapeXml(`${prefix}/api/sitemaps/collections.xml`)}</loc>
   </sitemap>
+  <sitemap>
+    <loc>${escapeXml(`${prefix}/api/sitemaps/highlights.xml`)}</loc>
+  </sitemap>
   <sitemap>
     <loc>${escapeXml(`${prefix}/api/sitemaps/tags.xml`)}</loc>
   </sitemap>
@@ -785,6 +869,15 @@ export default async function (fastify: FastifyInstance): Promise<void> {
       );
   });
 
+  fastify.get('/highlights.xml', async (_, res) => {
+    const con = await createOrGetConnection();
+
+    return res
+      .type('application/xml')
+      .header('cache-control', SITEMAP_CACHE_CONTROL)
+      .send(await buildHighlightsSitemapXml(con));
+  });
+
   fastify.get('/tags.txt', async (_, res) => {
     const con = await createOrGetConnection();
     const prefix = getSitemapUrlPrefix();