Skip to content

Commit 6b0d4b0

Browse files
hhvrchhvrc
authored
feat: Dynamic SEO/crawler endpoints (#191)
1 parent f16a3c8 commit 6b0d4b0

10 files changed

Lines changed: 354 additions & 1 deletion

File tree

‎src/lib/utils/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ export * from './debounce';
22
export * from './encoding';
33
export * from './entropy';
44
export * from './math';
5+
export * from './parse';
56
export * from './rand';
67
export * from './shadcn';
78
export * from './shockerPause';

‎src/lib/utils/parse.test.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import { describe, expect, it } from 'vitest';
2+
import { isTruthy } from './parse';
3+
4+
describe('isTruthy', () => {
5+
it.each([true, 1, '1', 'true', 'TRUE', 'True', 'yes', 'YES', 'y', 'Y', 'on', 'ON'])(
6+
'returns true for %j',
7+
(value) => {
8+
expect(isTruthy(value)).toBe(true);
9+
}
10+
);
11+
12+
it.each([false, 0, 2, -1, '0', 'false', 'no', 'off', 'random', '', null, undefined])(
13+
'returns false for %j',
14+
(value) => {
15+
expect(isTruthy(value)).toBe(false);
16+
}
17+
);
18+
});

‎src/lib/utils/parse.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/**
2+
* Coerces an env-var / URL-param style value into a boolean.
3+
*
4+
* Truthy: `true`, the number `1`, or a case-insensitive `'1'`, `'true'`, `'yes'`, `'y'`, or `'on'`.
5+
*
6+
* Everything else - including `null`, `undefined`, other numbers, and unrecognized strings - is falsy.
7+
*
8+
* @param value - The value to coerce (typically an environment variable or query parameter)
9+
* @returns `true` when the value matches a recognized truthy form, `false` otherwise
10+
*
11+
* @example
12+
* ```ts
13+
* isTruthy(env.PUBLIC_DISABLE_SITEMAP) // false when unset, true when "1" / "true"
14+
* ```
15+
*/
16+
export function isTruthy(value: boolean | number | string | null | undefined): boolean {
17+
if (typeof value === 'boolean') return value;
18+
if (typeof value === 'number') return value === 1;
19+
if (typeof value !== 'string') return false;
20+
21+
switch (value.toLowerCase()) {
22+
case '1':
23+
case 'true':
24+
case 'yes':
25+
case 'y':
26+
case 'on':
27+
return true;
28+
default:
29+
return false;
30+
}
31+
}

‎src/lib/utils/public-routes.test.ts‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import { describe, expect, it } from 'vitest';
2+
import { fileToPath } from './public-routes';
3+
4+
describe('fileToPath', () => {
5+
it('parses the root route', () => {
6+
const result = fileToPath('/src/routes/+page.svelte');
7+
expect(result).toEqual({
8+
path: '/',
9+
original: '/',
10+
categories: [],
11+
parameters: [],
12+
});
13+
});
14+
15+
it('parses a simple path', () => {
16+
const result = fileToPath('/src/routes/about/+page.svelte');
17+
expect(result).toEqual({
18+
path: '/about',
19+
original: '/about',
20+
categories: [],
21+
parameters: [],
22+
});
23+
});
24+
25+
it('parses nested segments', () => {
26+
const result = fileToPath('/src/routes/settings/account/+page.svelte');
27+
expect(result).toEqual({
28+
path: '/settings/account',
29+
original: '/settings/account',
30+
categories: [],
31+
parameters: [],
32+
});
33+
});
34+
35+
it('extracts route groups as categories', () => {
36+
const result = fileToPath('/src/routes/(auth)/login/+page.svelte');
37+
expect(result).toEqual({
38+
path: '/login',
39+
original: '/(auth)/login',
40+
categories: ['auth'],
41+
parameters: [],
42+
});
43+
});
44+
45+
it('extracts multiple route groups', () => {
46+
const result = fileToPath('/src/routes/(app)/(settings)/profile/+page.svelte');
47+
expect(result).toEqual({
48+
path: '/profile',
49+
original: '/(app)/(settings)/profile',
50+
categories: ['app', 'settings'],
51+
parameters: [],
52+
});
53+
});
54+
55+
it('extracts untyped parameters', () => {
56+
const result = fileToPath('/src/routes/user/[id]/+page.svelte');
57+
expect(result).toEqual({
58+
path: '/user/[id]',
59+
original: '/user/[id]',
60+
categories: [],
61+
parameters: [{ name: 'id', type: 'unknown' }],
62+
});
63+
});
64+
65+
it('extracts typed parameters', () => {
66+
const result = fileToPath('/src/routes/user/[id=integer]/+page.svelte');
67+
expect(result).toEqual({
68+
path: '/user/[id]',
69+
original: '/user/[id=integer]',
70+
categories: [],
71+
parameters: [{ name: 'id', type: 'integer' }],
72+
});
73+
});
74+
75+
it('handles groups, params, and segments together', () => {
76+
const result = fileToPath('/src/routes/(app)/org/[slug=string]/members/+page.svelte');
77+
expect(result).toEqual({
78+
path: '/org/[slug]/members',
79+
original: '/(app)/org/[slug=string]/members',
80+
categories: ['app'],
81+
parameters: [{ name: 'slug', type: 'string' }],
82+
});
83+
});
84+
});

‎src/lib/utils/public-routes.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import type { Pathname } from '$app/types';
2+
3+
export type RouteParam = { name: string; type: string };
4+
5+
export type RouteInfo = {
6+
categories: readonly string[];
7+
parameters: readonly RouteParam[];
8+
path: Pathname;
9+
original: string;
10+
};
11+
12+
export function fileToPath(file: string): RouteInfo {
13+
const original = file.replace(/^\/src\/routes/, '').replace(/\/\+page\.svelte$/, '') || '/';
14+
15+
const categories: string[] = [];
16+
const parameters: RouteParam[] = [];
17+
const segments: string[] = [];
18+
19+
for (const part of original.split('/')) {
20+
if (!part) continue;
21+
22+
// (group)
23+
if (part[0] === '(' && part.at(-1) === ')') {
24+
categories.push(part.slice(1, -1));
25+
continue;
26+
}
27+
28+
// [param] or [param=type]
29+
if (part[0] === '[' && part.at(-1) === ']') {
30+
const [name, type = 'unknown'] = part.slice(1, -1).split('=');
31+
parameters.push({ name, type });
32+
segments.push(`[${name}]`);
33+
continue;
34+
}
35+
36+
segments.push(part);
37+
}
38+
39+
const path = (segments.length ? '/' + segments.join('/') : '/') as Pathname;
40+
return { categories, parameters, path, original };
41+
}
42+
43+
export const paths: readonly RouteInfo[] = Object.keys(
44+
import.meta.glob('/src/routes/**/+page.svelte')
45+
).map(fileToPath);
46+
47+
const byPath = (a: RouteInfo, b: RouteInfo) => a.path.localeCompare(b.path);
48+
49+
export const publicRoutes: readonly Pathname[] = paths
50+
.filter((p) => !p.categories.includes('app') && p.parameters.length === 0)
51+
.toSorted(byPath)
52+
.map((p) => p.path);
53+
54+
export const authenticatedRoutes: readonly Pathname[] = paths
55+
.filter((p) => p.categories.includes('app'))
56+
.toSorted(byPath)
57+
.map((p) => p.path);

‎src/routes/(meta)/+layout.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export const prerender = false;

‎src/routes/(meta)/llms.txt/+server.ts‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import { env } from '$env/dynamic/public';
2+
import {
3+
PUBLIC_DISCORD_INVITE_URL,
4+
PUBLIC_GITHUB_PROJECT_URL,
5+
PUBLIC_SITE_DESCRIPTION,
6+
PUBLIC_SITE_NAME,
7+
} from '$env/static/public';
8+
import { isTruthy } from '$lib/utils/parse';
9+
import { paths } from '$lib/utils/public-routes';
10+
import { getBackendURL, getSiteURL } from '$lib/utils/url';
11+
import { error } from '@sveltejs/kit';
12+
import type { RequestHandler } from './$types';
13+
14+
// Internal-only top-level segments we don't want to disclose to crawlers.
15+
const HIDDEN_SEGMENTS: ReadonlySet<string> = new Set(['admin', 'hangfire']);
16+
17+
const SWAGGER_VERSIONS = [1, 2] as const;
18+
19+
export const GET: RequestHandler = ({ setHeaders }) => {
20+
if (isTruthy(env.PUBLIC_DISABLE_LLMS_TXT)) error(404);
21+
22+
setHeaders({
23+
'content-type': 'text/plain; charset=utf-8',
24+
'cache-control': 'public, max-age=3600',
25+
});
26+
27+
const name = PUBLIC_SITE_NAME.trim();
28+
const description = PUBLIC_SITE_DESCRIPTION.trim();
29+
const isOpenShock = name.toLowerCase() === 'openshock';
30+
31+
const summary = isOpenShock
32+
? `OpenShock — ${description}`
33+
: `${name} — an independent instance of OpenShock — ${description}`;
34+
35+
// Public, linkable routes grouped by role.
36+
const publicPaths = paths
37+
.filter((p) => !p.categories.includes('app') && p.parameters.length === 0)
38+
.toSorted((a, b) => a.path.localeCompare(b.path));
39+
40+
const generalPages = publicPaths.filter((p) => p.categories.length === 0);
41+
const authPages = publicPaths.filter((p) => p.categories.includes('auth'));
42+
43+
const renderLinks = (list: typeof publicPaths) =>
44+
list.map((p) => `- [${p.path}](${getSiteURL(p.path).href})`).join('\n');
45+
46+
const generalList = renderLinks(generalPages);
47+
const authList = renderLinks(authPages);
48+
49+
// OpenAPI specs, one per backend API version.
50+
const apiURL = getBackendURL();
51+
const swaggerList = SWAGGER_VERSIONS.map(
52+
(v) => `- [v${v} OpenAPI spec](${new URL(`/swagger/${v}/swagger.json`, apiURL).href})`
53+
).join('\n');
54+
55+
// Authenticated app routes — listed for crawlers with a "not indexable" notice.
56+
const appPaths = paths
57+
.filter((p) => p.categories.includes('app'))
58+
.filter((p) => !HIDDEN_SEGMENTS.has(p.path.split('/')[1] ?? ''));
59+
60+
const appSegments = [
61+
...new Set(
62+
appPaths.map((p) => p.path.split('/')[1]).filter((seg): seg is string => Boolean(seg))
63+
),
64+
]
65+
.sort()
66+
.map((seg) => `\`/${seg}\``)
67+
.join(', ');
68+
69+
const appList = appPaths
70+
.map((p) => p.path)
71+
.toSorted()
72+
.map((path) => `- ${path}`)
73+
.join('\n');
74+
75+
const body = `# ${name}
76+
77+
> ${summary}
78+
79+
${name} is the web frontend of the OpenShock platform. Authenticated users manage \
80+
hubs (ESP32-based bridges), shockers, share permissions, API tokens, and live control \
81+
sessions. Unauthenticated visitors can flash firmware to a device over WebSerial or sign \
82+
up for an account.
83+
84+
## Public pages
85+
86+
${generalList}
87+
88+
## Authentication pages
89+
90+
${authList}
91+
92+
## External resources
93+
94+
- [openshock.org](https://openshock.org): project website
95+
- [wiki.openshock.org](https://wiki.openshock.org): user and developer documentation
96+
- [GitHub](${PUBLIC_GITHUB_PROJECT_URL}): source code and issues
97+
- [Discord](${PUBLIC_DISCORD_INVITE_URL}): community chat and support
98+
99+
## Backend API
100+
101+
${swaggerList}
102+
103+
## Notes for crawlers
104+
105+
Routes under ${appSegments} require authentication and contain user-specific \
106+
data — not indexable:
107+
108+
${appList}
109+
`;
110+
111+
return new Response(body);
112+
};

‎src/routes/(meta)/robots.txt/+server.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import { env } from '$env/dynamic/public';
2+
import { isTruthy } from '$lib/utils/parse';
3+
import { getSiteURL } from '$lib/utils/url';
4+
import type { RequestHandler } from './$types';
5+
6+
export const GET: RequestHandler = ({ setHeaders }) => {
7+
setHeaders({
8+
'content-type': 'text/plain; charset=utf-8',
9+
'cache-control': 'public, max-age=3600',
10+
});
11+
12+
if (isTruthy(env.PUBLIC_DENY_ROBOTS)) {
13+
return new Response('User-agent: *\nDisallow: /\n');
14+
}
15+
16+
const lines = ['User-agent: *', 'Allow: /'];
17+
if (!isTruthy(env.PUBLIC_DISABLE_SITEMAP)) {
18+
lines.push(`Sitemap: ${getSiteURL('/sitemap.xml').href}`);
19+
}
20+
21+
return new Response(lines.join('\n') + '\n');
22+
};

‎src/routes/(meta)/sitemap.xml/+server.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import { env } from '$env/dynamic/public';
2+
import { isTruthy } from '$lib/utils/parse';
3+
import { publicRoutes } from '$lib/utils/public-routes';
4+
import { getSiteURL } from '$lib/utils/url';
5+
import { error } from '@sveltejs/kit';
6+
import type { RequestHandler } from './$types';
7+
8+
export const GET: RequestHandler = ({ setHeaders }) => {
9+
if (isTruthy(env.PUBLIC_DISABLE_SITEMAP)) error(404);
10+
11+
setHeaders({
12+
'content-type': 'application/xml; charset=utf-8',
13+
'cache-control': 'public, max-age=3600',
14+
});
15+
16+
const lastmod = new Date().toISOString().slice(0, 10);
17+
const urls = publicRoutes
18+
.map((path) => ` <url><loc>${getSiteURL(path).href}</loc><lastmod>${lastmod}</lastmod></url>`)
19+
.join('\n');
20+
21+
return new Response(
22+
`<?xml version="1.0" encoding="UTF-8"?>
23+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
24+
${urls}
25+
</urlset>
26+
`
27+
);
28+
};

‎static/robots.txt‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

0 commit comments

Comments
 (0)