1- import { libraries } from '~/libraries'
1+ import { getBranch , libraries } from '~/libraries'
2+ import type { LibrarySlim } from '~/libraries/types'
23import { getPublishedPosts } from '~/utils/blog'
4+ import { fetchRepoDirectoryContents } from '~/utils/docs'
5+ import type { GitHubFileNode } from '~/utils/documents.server'
36import { env } from '~/utils/env'
47
8+ const TOP_LEVEL_ROUTE_MODULES = Object . keys (
9+ import . meta. glob ( '../routes/*.{ts,tsx}' ) ,
10+ )
11+
12+ const TOP_LEVEL_INDEX_ROUTE_MODULES = Object . keys (
13+ import . meta. glob ( '../routes/*/index.tsx' ) ,
14+ )
15+
516export type SitemapEntry = {
617 path : string
718 lastModified ?: string
819}
920
10- const HIGH_VALUE_STATIC_SITEMAP_PATHS = [
11- '/' ,
12- '/blog' ,
13- '/libraries' ,
14- '/learn' ,
15- '/showcase' ,
16- '/support' ,
17- '/partners' ,
18- '/workshops' ,
19- '/maintainers' ,
20- '/builder' ,
21- '/explore' ,
22- '/ethos' ,
23- '/tenets' ,
24- ] as const satisfies ReadonlyArray < string >
21+ const MAX_DOCS_SITEMAP_DEPTH = 3
22+
23+ const EXCLUDED_TOP_LEVEL_ROUTE_NAMES = new Set ( [
24+ '__root' ,
25+ 'account' ,
26+ 'ads' ,
27+ 'blog.$' ,
28+ 'brand-guide' ,
29+ 'builder.docs' ,
30+ 'dashboard' ,
31+ 'feed' ,
32+ 'feedback-leaderboard' ,
33+ 'llms.txt' ,
34+ 'login' ,
35+ 'merch' ,
36+ 'partners-embed' ,
37+ 'privacy' ,
38+ 'terms' ,
39+ 'robots.txt' ,
40+ 'rss.xml' ,
41+ 'sitemap.xml' ,
42+ 'sponsors-embed' ,
43+ ] )
44+
45+ const EXCLUDED_TOP_LEVEL_ROUTE_DIRECTORIES = new Set ( [
46+ '$libraryId' ,
47+ '[.]well-known' ,
48+ 'account' ,
49+ 'admin' ,
50+ 'stats' ,
51+ ] )
2552
2653function trimTrailingSlash ( url : string ) {
2754 return url . replace ( / \/ $ / , '' )
@@ -40,23 +67,133 @@ function asLastModified(value: string) {
4067 return new Date ( `${ value } ) . toISOString ( )
4168}
4269
70+ function normalizeRouteName ( routeName : string ) {
71+ return routeName . replace ( / \[ \. \] / g, '.' )
72+ }
73+
74+ function getTopLevelRoutePath ( routeName : string ) {
75+ if ( routeName === 'index' ) {
76+ return '/'
77+ }
78+
79+ if ( routeName . endsWith ( '.index' ) ) {
80+ return `/${ routeName . slice ( 0 , - '.index' . length ) }
81+ }
82+
83+ return `/${ routeName }
84+ }
85+
86+ function getTopLevelEntries ( ) : Array < SitemapEntry > {
87+ const fileEntries = TOP_LEVEL_ROUTE_MODULES . flatMap ( ( modulePath ) => {
88+ const routeName = normalizeRouteName (
89+ modulePath
90+ . split ( '/' )
91+ . at ( - 1 )
92+ ?. replace ( / \. ( t s | t s x ) $ / , '' ) ?? '' ,
93+ )
94+
95+ if ( ! routeName || EXCLUDED_TOP_LEVEL_ROUTE_NAMES . has ( routeName ) ) {
96+ return [ ]
97+ }
98+
99+ return [ { path : getTopLevelRoutePath ( routeName ) } ]
100+ } )
101+
102+ const directoryEntries = TOP_LEVEL_INDEX_ROUTE_MODULES . flatMap (
103+ ( modulePath ) => {
104+ const routeDirectory = modulePath . split ( '/' ) . at ( - 2 )
105+
106+ if (
107+ ! routeDirectory ||
108+ EXCLUDED_TOP_LEVEL_ROUTE_DIRECTORIES . has ( routeDirectory )
109+ ) {
110+ return [ ]
111+ }
112+
113+ return [ { path : `/${ normalizeRouteName ( routeDirectory ) } } ]
114+ } ,
115+ )
116+
117+ return [ ...fileEntries , ...directoryEntries ]
118+ }
119+
43120function getLibraryEntries ( ) : Array < SitemapEntry > {
44121 return libraries . flatMap ( ( library ) => {
45- if ( library . visible === false || ! library . latestVersion ) {
122+ if (
123+ library . visible === false ||
124+ ! library . latestVersion ||
125+ library . sitemap ?. includeLandingPage !== true
126+ ) {
46127 return [ ]
47128 }
48129
49130 const basePath = `/${ library . id }
50- const entries : Array < SitemapEntry > = [ { path : basePath } ]
131+ return [ { path : basePath } ]
132+ } )
133+ }
51134
52- if ( library . defaultDocs ) {
53- entries . push ( {
54- path : `${ basePath } ${ library . defaultDocs } ,
55- } )
56- }
135+ function flattenDocsTree ( nodes : Array < GitHubFileNode > ) : Array < GitHubFileNode > {
136+ return nodes . flatMap ( ( node ) => [
137+ node ,
138+ ...( node . children ? flattenDocsTree ( node . children ) : [ ] ) ,
139+ ] )
140+ }
57141
58- return entries
59- } )
142+ function toDocsSlug ( filePath : string , docsRoot : string ) {
143+ const docsPrefix = `${ docsRoot }
144+
145+ if ( ! filePath . startsWith ( docsPrefix ) || ! filePath . endsWith ( '.md' ) ) {
146+ return null
147+ }
148+
149+ const slug = filePath . slice ( docsPrefix . length , - '.md' . length )
150+
151+ if ( ! slug || slug . endsWith ( '/index' ) ) {
152+ return null
153+ }
154+
155+ return slug
156+ }
157+
158+ function isTopLevelDocsSlug ( slug : string ) {
159+ const segments = slug . split ( '/' )
160+
161+ return segments . length <= MAX_DOCS_SITEMAP_DEPTH
162+ }
163+
164+ function isDefined < T > ( value : T | null ) : value is T {
165+ return value !== null
166+ }
167+
168+ async function getLibraryDocsEntries (
169+ library : LibrarySlim ,
170+ ) : Promise < Array < SitemapEntry > > {
171+ if (
172+ library . visible === false ||
173+ ! library . latestVersion ||
174+ library . sitemap ?. includeTopLevelDocsPages !== true
175+ ) {
176+ return [ ]
177+ }
178+
179+ const docsRoot = library . docsRoot || 'docs'
180+ const branch = getBranch ( library , 'latest' )
181+ const docsTree = await fetchRepoDirectoryContents ( {
182+ data : {
183+ repo : library . repo ,
184+ branch,
185+ startingPath : docsRoot ,
186+ } ,
187+ } ) . catch ( ( ) => [ ] )
188+
189+ return flattenDocsTree ( docsTree )
190+ . filter ( ( node ) => node . type === 'file' )
191+ . map ( ( node ) => toDocsSlug ( node . path , docsRoot ) )
192+ . filter ( isDefined )
193+ . filter ( isTopLevelDocsSlug )
194+ . map ( ( slug ) => ( {
195+ path : `/${ library . id } ${ slug } ,
196+ } ) )
60197}
61198
62199function getBlogEntries ( ) : Array < SitemapEntry > {
@@ -70,10 +207,15 @@ export function getSiteOrigin(request: Request) {
70207 return trimTrailingSlash ( env . SITE_URL || new URL ( request . url ) . origin )
71208}
72209
73- export function getSitemapEntries ( ) : Array < SitemapEntry > {
210+ export async function getSitemapEntries ( ) : Promise < Array < SitemapEntry > > {
211+ const docsEntries = await Promise . all (
212+ libraries . map ( ( library ) => getLibraryDocsEntries ( library ) ) ,
213+ )
214+
74215 const entries = [
75- ...HIGH_VALUE_STATIC_SITEMAP_PATHS . map ( ( path ) => ( { path } ) ) ,
216+ ...getTopLevelEntries ( ) ,
76217 ...getLibraryEntries ( ) ,
218+ ...docsEntries . flat ( ) ,
77219 ...getBlogEntries ( ) ,
78220 ]
79221
@@ -82,8 +224,8 @@ export function getSitemapEntries(): Array<SitemapEntry> {
82224 )
83225}
84226
85- export function generateSitemapXml ( origin : string ) {
86- const urls = getSitemapEntries ( )
227+ export async function generateSitemapXml ( origin : string ) {
228+ const urls = ( await getSitemapEntries ( ) )
87229 . map ( ( entry ) => {
88230 const loc = `${ origin } ${ entry . path }
89231
0 commit comments