Skip to content

Commit 33b9060

Browse files
johndmulhausenjohndmulhausen
authored
Create llms.txt and llms-full.txt (#1328)
Enhance LLMs' understanding of W&B docs by generating llms.txt and llms-full.txt at the root of the docs site. Previews: https://llms-full.docodile.pages.dev/llms.txt https://llms-full.docodile.pages.dev/llms-full.txt
1 parent 0f7cac3 commit 33b9060

3 files changed

Lines changed: 148 additions & 0 deletions

File tree

‎hugo.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,20 @@ outputs:
142142
- HTML
143143
- RSS
144144
- print
145+
home:
146+
- HTML
147+
- llms
148+
- llms-full
149+
150+
outputFormats:
151+
llms:
152+
mediaType: "text/plain"
153+
baseName: "llms"
154+
isPlainText: true
155+
llms-full:
156+
mediaType: "text/plain"
157+
baseName: "llms-full"
158+
isPlainText: true
145159

146160
services:
147161
googleAnalytics:

‎layouts/_default/index.llms-full.txt‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
{{ with .Site.Title }}
2+
# {{ . }}
3+
{{ end }}
4+
5+
{{ with .Site.Params.Description }}
6+
> {{ . }}
7+
{{ end }}
8+
9+
{{/* Top-level pages in English */}}
10+
{{ range (where ((.Site.GetPage "/").Pages) "Params.sitemap_exclude" "ne" true) }}
11+
{{ $page := . }}
12+
{{ if (not (or (in .RelPermalink "/_includes/") (in $page.RelPermalink "/ja/") (in $page.RelPermalink "/ko/"))) }}
13+
# {{ $page.Title }}
14+
{{ with $page.Description }}
15+
> {{ . }}
16+
{{ end }}
17+
{{ $page.RawContent }}
18+
19+
{{ end }}
20+
{{ end }}
21+
22+
{{/* Sections in English */}}
23+
{{ range (where ((.Site.GetPage "/").Sections) "Params.sitemap_exclude" "ne" true) }}
24+
{{ $section := . }}
25+
{{ if (not (or (in .RelPermalink "/_includes/") (in $section.RelPermalink "/ja/") (in $section.RelPermalink "/ko/"))) }}
26+
{{ with $section.Title }}
27+
# {{ . }}
28+
{{ end }}
29+
30+
{{ with $section.Description }}
31+
> {{ . }}
32+
{{ end }}
33+
34+
{{ $section.RawContent }}
35+
36+
{{/* Pages inside section */}}
37+
{{ range (where $section.Pages "Params.sitemap_exclude" "ne" true) }}
38+
{{ $page := . }}
39+
{{ if (not (or (in .RelPermalink "/_includes/") (in $page.RelPermalink "/ja/") (in $page.RelPermalink "/ko/"))) }}
40+
# {{ $page.Title }}
41+
{{ with $page.Description }}
42+
> {{ . }}
43+
{{ end }}
44+
{{ $page.RawContent }}
45+
46+
{{ end }}
47+
{{ end }}
48+
49+
{{/* Sub-sections inside section */}}
50+
{{ range (where $section.Sections "Params.sitemap_exclude" "ne" true) }}
51+
{{ $subsection := . }}
52+
{{ if (not (or (in .RelPermalink "/_includes/") (in $subsection.RelPermalink "/ja/") (in $subsection.RelPermalink "/ko/"))) }}
53+
{{ with $subsection.Title }}
54+
# {{ . }}
55+
{{ end }}
56+
57+
{{ with $subsection.Description }}
58+
> {{ . }}
59+
{{ end }}
60+
61+
{{ $subsection.RawContent }}
62+
63+
{{ range (where $subsection.Pages "Params.sitemap_exclude" "ne" true) }}
64+
{{ $page := . }}
65+
{{ if (not (or (in .RelPermalink "/_includes/") (in $page.RelPermalink "/ja/") (in $page.RelPermalink "/ko/"))) }}
66+
# {{ $page.Title }}
67+
{{ with $page.Description }}
68+
> {{ . }}
69+
{{ end }}
70+
{{ $page.RawContent }}
71+
72+
{{ end }}
73+
{{ end }}
74+
{{ end }}
75+
{{ end }}
76+
{{ end }}
77+
{{ end }}

‎layouts/_default/index.llms.txt‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
{{- with .Site.Title }}
2+
# {{ . }}
3+
{{ end -}}
4+
5+
{{- with .Site.Params.Description }}
6+
> {{ . }}
7+
{{ end -}}
8+
9+
{{- range (where (sort ((.Site.GetPage "/").Pages) "Weight" "asc" "Date" "desc" "Lastmod" "desc") "Params.sitemap_exclude" "ne" true) }}
10+
{{- if (not (or (in .RelPermalink "/_includes/") (in .RelPermalink "/ja/") (in .RelPermalink "/ko/"))) }}
11+
- [{{ .Title }}]({{ .Permalink }}): {{ .Description }}
12+
{{ end -}}
13+
{{- end }}
14+
15+
{{/* Define a partial template for recursive section processing */}}
16+
{{- define "processSection" }}
17+
{{- $section := .section }}
18+
{{- $level := .level }}
19+
{{- if (not (or (in .RelPermalink "/_includes/") (in $section.RelPermalink "/ja/") (in $section.RelPermalink "/ko/"))) }}
20+
{{- $hasEnglishPages := false }}
21+
{{- range (where (sort $section.Pages "Weight" "asc" "Date" "desc" "Lastmod" "desc") "Params.sitemap_exclude" "ne" true) }}
22+
{{- if (and .Title (not (or (in .RelPermalink "/_includes/") (in .RelPermalink "/ja/") (in .RelPermalink "/ko/")))) }}
23+
{{- $hasEnglishPages = true }}
24+
{{- end }}
25+
{{- end }}
26+
{{- $hasEnglishSubsections := false }}
27+
{{- range (where (sort $section.Sections "Weight" "asc" "Date" "desc" "Lastmod" "desc") "Params.sitemap_exclude" "ne" true) }}
28+
{{- if (not (or (in .RelPermalink "/_includes/") (in .RelPermalink "/ja/") (in .RelPermalink "/ko/"))) }}
29+
{{- $hasEnglishSubsections = true }}
30+
{{- end }}
31+
{{- end }}
32+
{{- if (or $hasEnglishPages $hasEnglishSubsections) }}
33+
{{- with $section.Title }}
34+
{{ strings.Repeat $level "#" }} {{ . }}
35+
{{ end -}}
36+
37+
{{- with $section.Description }}
38+
> {{ . }}
39+
{{ end -}}
40+
41+
{{- range (where (sort $section.Pages "Weight" "asc" "Date" "desc" "Lastmod" "desc") "Params.sitemap_exclude" "ne" true) }}
42+
{{- if (and .Title (not (or (in .RelPermalink "/_includes/") (in .RelPermalink "/ja/") (in .RelPermalink "/ko/")))) }}
43+
- [{{ .Title }}]({{ .Permalink }}){{ with .Description }}: {{ . }}{{ end }}
44+
{{ end -}}
45+
{{- end }}
46+
47+
{{- range (where (sort $section.Sections "Weight" "asc" "Date" "desc" "Lastmod" "desc") "Params.sitemap_exclude" "ne" true) }}
48+
{{- template "processSection" (dict "section" . "level" (add $level 1)) }}
49+
{{- end }}
50+
{{- end }}
51+
{{- end }}
52+
{{- end }}
53+
54+
{{/* Process all top-level sections recursively */}}
55+
{{- range (where (sort ((.Site.GetPage "/").Sections) "Weight" "asc" "Date" "desc" "Lastmod" "desc") "Params.sitemap_exclude" "ne" true) }}
56+
{{- template "processSection" (dict "section" . "level" 2) }}
57+
{{- end }}

0 commit comments

Comments
 (0)