Skip to content

Commit e6de70a

Browse files
Benanna2019Benanna2019
committed
added llms.txt, llms-full.txt, copy markdown button
1 parent 60f095f commit e6de70a

9 files changed

Lines changed: 324 additions & 9 deletions

File tree

‎app.arc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ fingerprint true
1212
get /
1313
get /docs/:lang/*
1414
get /api/package
15+
get /llms.txt
16+
get /llms-full.txt
1517
any /*
1618

1719
@plugins

‎eslint.config.mjs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,20 @@ export default [
77
languageOptions: {
88
parserOptions: {
99
ecmaVersion: 2022,
10-
sourceType: 'module'
11-
}
10+
sourceType: 'module',
11+
},
1212
},
1313
plugins: {
14-
import: importPlugin.flatConfigs.recommended.plugins.import
14+
import: importPlugin.flatConfigs.recommended.plugins.import,
1515
},
1616
rules: {
1717
'import/no-commonjs': 'error',
1818
'import/extensions': [
1919
'error',
20-
'ignorePackages'
20+
'ignorePackages',
2121
],
2222
// Additive to our old `import` config, but everything seems quite sane!
2323
...importPlugin.flatConfigs.recommended.rules,
24-
}
24+
},
2525
},
2626
]

‎public/index.js‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,30 @@
3030
localStorage.setItem('theme', targetTheme)
3131
}
3232

33+
// Copy Markdown button for LLM use
34+
const copyMarkdownBtn = document.getElementById('copy-markdown-btn')
35+
if (copyMarkdownBtn) {
36+
copyMarkdownBtn.onclick = () => {
37+
const markdown = copyMarkdownBtn.getAttribute('data-markdown')
38+
.replace(/"/g, '"')
39+
.replace(/&lt;/g, '<')
40+
.replace(/&gt;/g, '>')
41+
.replace(/&amp;/g, '&')
42+
const textSpan = document.getElementById('copy-markdown-text')
43+
44+
navigator.clipboard.writeText(markdown).then(
45+
() => {
46+
textSpan.textContent = 'Copied!'
47+
setTimeout(() => textSpan.textContent = 'Copy for LLM', 2000)
48+
},
49+
() => {
50+
textSpan.textContent = 'Error!'
51+
setTimeout(() => textSpan.textContent = 'Copy for LLM', 2000)
52+
}
53+
)
54+
}
55+
}
56+
3357
// Copy-Paste function for code blocks
3458
const buttonClassList = [
3559
'icon',

‎src/http/get-docs-000lang-catchall/index.mjs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ async function handler (req) {
7474
active,
7575
editURL,
7676
lang,
77+
markdown: md,
7778
path,
7879
scripts: [
7980
'/index.js',

‎src/http/get-llms_full_txt/index.mjs‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
import { readFileSync, readdirSync, existsSync } from 'fs'
2+
import { join, relative, dirname } from 'path'
3+
import { fileURLToPath } from 'url'
4+
import arc from '@architect/functions'
5+
6+
const __dirname = dirname(fileURLToPath(import.meta.url))
7+
const BASE_URL = 'https://arc.codes'
8+
9+
// Configuration
10+
const config = {
11+
// Files or directories to skip
12+
excludes: [ '.DS_Store', 'node_modules', 'table-of-contents.mjs' ],
13+
}
14+
15+
/**
16+
* Cleans markdown content for LLM consumption
17+
* @param {string} content - Raw markdown content
18+
* @returns {string} Cleaned content
19+
*/
20+
function cleanMarkdownContent (content) {
21+
return content
22+
// Remove frontmatter
23+
.replace(/^---[\s\S]*?---\n*/m, '')
24+
// Remove custom HTML components but keep content
25+
.replace(/<arc-viewer[^>]*>/g, '')
26+
.replace(/<\/arc-viewer>/g, '')
27+
.replace(/<arc-tab[^>]*label="([^"]*)"[^>]*>/g, '**$1:**\n')
28+
.replace(/<\/arc-tab>/g, '')
29+
.replace(/<div[^>]*slot[^>]*>/g, '')
30+
.replace(/<\/div>/g, '')
31+
.replace(/<h5>/g, '')
32+
.replace(/<\/h5>/g, '')
33+
// Remove multiple newlines
34+
.replace(/\n{3,}/g, '\n\n')
35+
.trim()
36+
}
37+
38+
/**
39+
* Extracts frontmatter from markdown content
40+
* @param {string} content - Raw markdown content
41+
* @returns {Object} Frontmatter data
42+
*/
43+
function extractFrontmatter (content) {
44+
const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/)
45+
if (!frontmatterMatch) return {}
46+
47+
const frontmatter = {}
48+
const lines = frontmatterMatch[1].split('\n')
49+
for (const line of lines) {
50+
const [ key, ...valueParts ] = line.split(':')
51+
if (key && valueParts.length) {
52+
frontmatter[key.trim()] = valueParts.join(':').trim()
53+
}
54+
}
55+
return frontmatter
56+
}
57+
58+
/**
59+
* Generates a URL from a relative file path
60+
* @param {string} relativePath - Relative path to the markdown file
61+
* @returns {string} Full URL
62+
*/
63+
function filePathToUrl (relativePath) {
64+
const urlPath = relativePath
65+
.replace(/\.md$/, '')
66+
.replace(/:/g, '') // Remove colons from path (e.g., :tutorials)
67+
return `${BASE_URL}/docs/en/${urlPath}`
68+
}
69+
70+
/**
71+
* Processes a markdown file and extracts its content
72+
* @param {string} filePath - Path to the markdown file
73+
* @param {string} docsDir - Base docs directory
74+
* @returns {string} Processed content with metadata
75+
*/
76+
function processMarkdownFile (filePath, docsDir) {
77+
const content = readFileSync(filePath, 'utf-8')
78+
const relativePath = relative(docsDir, filePath)
79+
const frontmatter = extractFrontmatter(content)
80+
const cleanContent = cleanMarkdownContent(content)
81+
82+
const metadata = [
83+
frontmatter.title ? `# ${frontmatter.title}` : null,
84+
`Source: ${filePathToUrl(relativePath)}`,
85+
frontmatter.description ? `Description: ${frontmatter.description}` : null,
86+
frontmatter.category ? `Category: ${frontmatter.category}` : null,
87+
]
88+
.filter(Boolean)
89+
.join('\n')
90+
91+
return `${metadata}\n\n${cleanContent}\n`
92+
}
93+
94+
/**
95+
* Recursively processes all markdown files in a directory
96+
* @param {string} dir - Directory to process
97+
* @param {string} docsDir - Base docs directory for relative paths
98+
* @returns {string[]} Array of processed file contents
99+
*/
100+
function processDirectory (dir, docsDir) {
101+
const results = []
102+
103+
if (!existsSync(dir)) {
104+
console.error(`Directory does not exist: ${dir}`)
105+
return results
106+
}
107+
108+
let files
109+
try {
110+
files = readdirSync(dir, { withFileTypes: true })
111+
}
112+
catch (err) {
113+
console.error(`Error reading directory ${dir}:`, err.message)
114+
return results
115+
}
116+
117+
for (const file of files) {
118+
if (config.excludes.includes(file.name)) continue
119+
120+
const fullPath = join(dir, file.name)
121+
122+
if (file.isDirectory()) {
123+
results.push(...processDirectory(fullPath, docsDir))
124+
}
125+
else if (file.name.endsWith('.md')) {
126+
try {
127+
results.push(processMarkdownFile(fullPath, docsDir))
128+
}
129+
catch (err) {
130+
console.error(`Error processing ${fullPath}:`, err.message)
131+
}
132+
}
133+
}
134+
135+
return results
136+
}
137+
138+
async function _handler () {
139+
// Try local dev path first (src/views), then fall back to production symlink (node_modules/@architect/views)
140+
let docsDir = join(__dirname, '..', '..', 'views', 'docs', 'en')
141+
142+
if (!existsSync(docsDir)) {
143+
docsDir = join(__dirname, 'node_modules', '@architect', 'views', 'docs', 'en')
144+
}
145+
146+
console.log('Attempting to read docs from:', docsDir)
147+
148+
const header = `# Architect (arc.codes) - Complete Documentation
149+
150+
> This is the complete documentation for Architect, a simple framework for building and delivering powerful Functional Web Apps (FWAs) on AWS.
151+
152+
> For a high-level overview, see: ${BASE_URL}/llms.txt
153+
154+
---
155+
156+
`
157+
158+
const content = processDirectory(docsDir, docsDir)
159+
const separator = '\n\n---\n\n'
160+
const body = header + content.join(separator)
161+
162+
return {
163+
statusCode: 200,
164+
headers: {
165+
'content-type': 'text/plain; charset=utf-8',
166+
'cache-control': 'no-cache, no-store, must-revalidate',
167+
},
168+
body,
169+
}
170+
}
171+
172+
export const handler = arc.http.async(_handler)

‎src/http/get-llms_txt/index.mjs‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import toc from '../../views/docs/table-of-contents.mjs'
2+
3+
const BASE_URL = 'https://arc.codes'
4+
5+
/**
6+
* Generates a URL for a documentation page
7+
* @param {string[]} pathParts - Path segments
8+
* @returns {string} Full URL
9+
*/
10+
function docUrl (pathParts) {
11+
const slug = pathParts
12+
.map(part => part.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, ''))
13+
.join('/')
14+
return `${BASE_URL}/docs/en/${slug}`
15+
}
16+
17+
/**
18+
* Recursively generates markdown links from the table of contents structure
19+
* @param {Array} items - TOC items (strings or objects)
20+
* @param {string[]} parentPath - Parent path segments
21+
* @param {number} depth - Current nesting depth
22+
* @returns {string} Markdown formatted links
23+
*/
24+
function generateLinks (items, parentPath = [], depth = 0) {
25+
const indent = ' '.repeat(depth)
26+
const lines = []
27+
28+
for (const item of items) {
29+
if (typeof item === 'string') {
30+
// Simple string item - it's a doc page
31+
const path = [ ...parentPath, item ]
32+
lines.push(`${indent}- [${item}](${docUrl(path)})`)
33+
}
34+
else if (typeof item === 'object' && !Array.isArray(item)) {
35+
// Object with nested structure
36+
for (const [ key, value ] of Object.entries(item)) {
37+
if (Array.isArray(value)) {
38+
// Category with sub-items
39+
lines.push(`${indent}- ${key}`)
40+
lines.push(generateLinks(value, [ ...parentPath, key ], depth + 1))
41+
}
42+
}
43+
}
44+
}
45+
46+
return lines.join('\n')
47+
}
48+
49+
export async function handler () {
50+
const sections = []
51+
52+
sections.push('# Architect (arc.codes)')
53+
sections.push('')
54+
sections.push('> Architect is a simple framework for building and delivering powerful Functional Web Apps (FWAs) on AWS')
55+
sections.push('')
56+
sections.push('## Documentation')
57+
sections.push('')
58+
59+
for (const [ sectionName, items ] of Object.entries(toc)) {
60+
sections.push(`### ${sectionName}`)
61+
sections.push('')
62+
sections.push(generateLinks(items, [ sectionName ]))
63+
sections.push('')
64+
}
65+
66+
// Add quick links section
67+
sections.push('## Quick Links')
68+
sections.push('')
69+
sections.push(`- [GitHub Repository](https://github.com/architect/architect)`)
70+
sections.push(`- [Full Documentation for LLMs](${BASE_URL}/llms-full.txt)`)
71+
sections.push(`- [Discord Community](https://discord.gg/y5A2eTsCRX)`)
72+
sections.push('')
73+
74+
const content = sections.join('\n')
75+
76+
return {
77+
statusCode: 200,
78+
headers: {
79+
'content-type': 'text/plain; charset=utf-8',
80+
'cache-control': 'max-age=3600, s-maxage=3600, stale-while-revalidate=86400',
81+
},
82+
body: content,
83+
}
84+
}

‎src/views/modules/components/copy-markdown.mjs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
export default function CopyMarkdown (state = {}) {
2+
const { markdown } = state
3+
4+
if (!markdown) return ''
5+
6+
// Escape the markdown for safe embedding in a data attribute
7+
const escapedMarkdown = markdown
8+
.replace(/&/g, '&amp;')
9+
.replace(/"/g, '&quot;')
10+
.replace(/</g, '&lt;')
11+
.replace(/>/g, '&gt;')
12+
13+
return `
14+
<button
15+
id="copy-markdown-btn"
16+
class="text1 text-p1 text-h1 text-a2 bg-unset cursor-pointer font-semibold inline-flex align-items-center gap-4"
17+
data-markdown="${escapedMarkdown}"
18+
title="Copy page markdown for use with LLMs"
19+
>
20+
<span class="icon fill-current">
21+
<svg><use xlink:href="#copy"></use></svg>
22+
</span>
23+
<span id="copy-markdown-text">Copy for LLM</span>
24+
</button>
25+
`
26+
}
27+

‎src/views/modules/components/edit-link.mjs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
export default function EditLink (state = {}) {
22
const { editURL } = state
33
return editURL ? `
4-
<div class="flex justify-end mt4">
5-
<a href="${editURL}" target="_blank" rel="noreferrer" class="text1 text-p1 text-h1 text-a2 no-underline font-semibold">Edit this doc on Github →</a>
6-
</div>
4+
<a href="${editURL}" target="_blank" rel="noreferrer" class="text1 text-p1 text-h1 text-a2 no-underline font-semibold">Edit this doc on GitHub →</a>
75
` : ''
86
}

‎src/views/modules/document/html.mjs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import Banner from '../components/banner.mjs'
2+
import CopyMarkdown from '../components/copy-markdown.mjs'
23
import DocumentOutline from '../components/document-outline.mjs'
34
import EditLink from '../components/edit-link.mjs'
45
import GoogleAnalytics from './ga.mjs'
@@ -14,6 +15,7 @@ export default function HTML (props = {}) {
1415
html = '',
1516
editURL = '',
1617
lang = 'en',
18+
markdown = '',
1719
scripts = '',
1820
slug = '',
1921
state = {},
@@ -75,9 +77,14 @@ ${Symbols}
7577
>
7678
${title}
7779
</h1>
80+
<div class="mb2">
81+
${CopyMarkdown({ markdown })}
82+
</div>
7883
<div class="pb4 docs">
7984
${html}
80-
${EditLink({ editURL })}
85+
<div class="flex justify-end align-items-center gap0 mt4 flex-wrap">
86+
${EditLink({ editURL })}
87+
</div>
8188
</div>
8289
</div>
8390
${DocumentOutline(props)}

0 commit comments

Comments
 (0)