fix: filter noindex posts from sitemaps and URL-encode tag slugs

__tests__/sitemaps.ts

@@ -15,10 +15,13 @@ import {
   PostType,
   SentimentEntity,
   SentimentGroup,
+  SharePost,
+  SocialTwitterPost,
   Source,
   SourceType,
   User,
 } from '../src/entity';
+import { BriefPost } from '../src/entity/posts/BriefPost';
 import {
   ArchivePeriodType,
   ArchiveRankingType,
@@ -254,6 +257,68 @@ describe('GET /sitemaps/posts-:page.xml', () => {
   });
 });
 
+describe('GET /sitemaps/posts.xml (type filtering)', () => {
+  it('should exclude noindex post types and collection posts', async () => {
+    const postBase = { sourceId: 'a', createdAt: now };
+
+    await con.getRepository(BriefPost).insert({
+      ...postBase,
+      id: 'brief-post',
+      shortId: 'bp1',
+      title: 'Brief Post',
+    });
+    await con.getRepository(SocialTwitterPost).insert({
+      ...postBase,
+      id: 'social-post',
+      shortId: 'sp1',
+      title: 'Social Post',
+    });
+    await con.getRepository(SharePost).insert({
+      ...postBase,
+      id: 'share-post',
+      shortId: 'shp',
+      title: 'Share Post',
+      sharedPostId: 'p1',
+    });
+    await con.getRepository(CollectionPost).insert({
+      ...postBase,
+      id: 'collection-sitemap',
+      shortId: 'csm',
+      title: 'Collection Sitemap',
+      visible: true,
+      upvotes: 5,
+      collectionSources: ['a', 'b', 'c'],
+    });
+
+    const res = await request(app.server)
+      .get('/sitemaps/posts.xml')
+      .expect(200);
+
+    expect(res.text).not.toContain('/posts/brief-post');
+    expect(res.text).not.toContain('/posts/social-post');
+    expect(res.text).not.toContain('/posts/share-post');
+    expect(res.text).not.toContain('/posts/collection-sitemap');
+    expect(res.text).toContain('/posts/p4-p4');
+  });
+
+  it('should exclude non-visible posts', async () => {
+    await con.getRepository(Post).insert({
+      id: 'hidden-post',
+      shortId: 'hp1',
+      title: 'Hidden Post',
+      sourceId: 'a',
+      createdAt: now,
+      visible: false,
+    });
+
+    const res = await request(app.server)
+      .get('/sitemaps/posts.xml')
+      .expect(200);
+
+    expect(res.text).not.toContain('/posts/hidden-post');
+  });
+});
+
 describe('GET /sitemaps/collections.xml', () => {
   it('should return only qualified public collections ordered by time as xml', async () => {
     const updatedAt = new Date('2024-02-01T12:00:00.123Z');
@@ -393,7 +458,7 @@ describe('GET /sitemaps/tags.xml', () => {
     );
     expect(res.text).toContain('<loc>http://localhost:5002/tags/webdev</loc>');
     expect(res.text).toContain(
-      '<loc>http://localhost:5002/tags/web&amp;ai</loc>',
+      '<loc>http://localhost:5002/tags/web%26ai</loc>',
     );
     expect(res.text).not.toContain('/tags/web-development');
     expect(res.text).not.toContain('/tags/politics');

src/routes/sitemaps.ts

@@ -87,7 +87,7 @@ const getPostSitemapUrl = (prefix: string, slug: string): string =>
   `${prefix}/posts/${slug}`;
 
 const getTagSitemapUrl = (prefix: string, value: string): string =>
-  `${prefix}/tags/${value}`;
+  `${prefix}/tags/${encodeURIComponent(value)}`;
 
 const getAgentSitemapUrl = (prefix: string, entity: string): string =>
   `${prefix}/agents/${encodeURIComponent(entity)}`;
@@ -128,17 +128,26 @@ const streamReplicaQuery = async <T extends ObjectLiteral>(
   }
 };
 
+const POSTS_SITEMAP_EXCLUDED_TYPES = [
+  PostType.Welcome,
+  PostType.Collection,
+  PostType.Share,
+  PostType.Brief,
+  PostType.SocialTwitter,
+];
+
 const buildPostsSitemapBaseQuery = (
   source: DataSource | EntityManager,
 ): SelectQueryBuilder<Post> =>
   source
     .createQueryBuilder()
     .from(Post, 'p')
     .leftJoin(User, 'u', 'p."authorId" = u.id')
-    .where('p.type NOT IN (:...types)', { types: [PostType.Welcome] })
+    .where('p.type NOT IN (:...types)', { types: POSTS_SITEMAP_EXCLUDED_TYPES })
     .andWhere('NOT p.private')
     .andWhere('NOT p.banned')
     .andWhere('NOT p.deleted')
+    .andWhere('p.visible = true')
     .andWhere('p."createdAt" > current_timestamp - interval \'90 day\'')
     .andWhere('(u.id is null or u.reputation > 10)');
 
@@ -244,10 +253,11 @@ const buildEvergreenSitemapBaseQuery = (
     .createQueryBuilder()
     .from(Post, 'p')
     .leftJoin(User, 'u', 'p."authorId" = u.id')
-    .where('p.type NOT IN (:...types)', { types: [PostType.Welcome] })
+    .where('p.type NOT IN (:...types)', { types: POSTS_SITEMAP_EXCLUDED_TYPES })
     .andWhere('NOT p.private')
     .andWhere('NOT p.banned')
     .andWhere('NOT p.deleted')
+    .andWhere('p.visible = true')
     .andWhere('p."createdAt" <= current_timestamp - interval \'90 day\'')
     .andWhere('p.upvotes >= :minUpvotes', { minUpvotes: 10 })
     .andWhere('(u.id is null or u.reputation > 10)');