update

Author: donutdaniel

public/robots.txt

@@ -0,0 +1,25 @@
+# https://www.danielho.xyz/robots.txt
+
+User-agent: *
+Allow: /
+
+# AI crawlers — welcome
+User-agent: GPTBot
+Allow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: Applebot-Extended
+Allow: /
+
+# LLM-friendly content
+# See /llms.txt and /llms-full.txt for clean markdown versions
+
+Sitemap: https://www.danielho.xyz/sitemap-index.xml

src/layouts/BaseLayout.astro

@@ -23,6 +23,9 @@ const canonicalURL = new URL(Astro.url.pathname, Astro.site);
     <link rel="canonical" href={canonicalURL} />
     <link rel="icon" href="/favicon.svg" type="image/svg+xml" />
 
+    <!-- LLM discovery -->
+    <link rel="alternate" type="text/plain" href="/llms.txt" title="LLM-friendly site summary" />
+
     <!-- Open Graph -->
     <meta property="og:type" content="website" />
     <meta property="og:url" content={canonicalURL} />
@@ -35,6 +38,15 @@ const canonicalURL = new URL(Astro.url.pathname, Astro.site);
     <meta name="twitter:title" content={title} />
     <meta name="twitter:description" content={description} />
     {ogImage && <meta name="twitter:image" content={new URL(ogImage, Astro.site)} />}
+    <!-- Google tag (gtag.js) -->
+    <script async src="https://www.googletagmanager.com/gtag/js?id=G-12ZHM7K2KH"></script>
+    <script is:inline>
+      window.dataLayer = window.dataLayer || [];
+      function gtag(){dataLayer.push(arguments);}
+      gtag('js', new Date());
+      gtag('config', 'G-12ZHM7K2KH');
+    </script>
+
     <ViewTransitions />
     <script is:inline>
       function applyThemeToDoc(doc) {

src/pages/llms-full.txt.ts

@@ -0,0 +1,58 @@
+import type { APIContext } from "astro";
+import { getCollection } from "astro:content";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+export async function GET(context: APIContext) {
+  const posts = (await getCollection("writing"))
+    .filter((post) => !post.data.draft)
+    .sort((a, b) => b.data.pubDate.valueOf() - a.data.pubDate.valueOf());
+
+  const articles: string[] = [];
+
+  for (const post of posts) {
+    const filePath = path.resolve(`./src/content/writing/${post.id}.mdx`);
+    const raw = await fs.readFile(filePath, "utf-8");
+
+    // Strip frontmatter
+    const content = raw.replace(/^---[\s\S]*?---\s*/, "");
+
+    // Strip MDX imports and JSX components
+    const cleaned = content
+      .replace(/^import\s+.*$/gm, "")
+      .replace(/<[A-Z]\w*\s*\/>/g, "")
+      .replace(/<figure>[\s\S]*?<\/figure>/g, "")
+      .trim();
+
+    const date = post.data.pubDate.toLocaleDateString("en-US", {
+      year: "numeric",
+      month: "long",
+      day: "numeric",
+    });
+
+    const url = new URL(`/writing/${post.id}/`, context.site!);
+
+    let header = `# ${post.data.title}\n\nPublished: ${date}`;
+    if (post.data.tags.length) header += `\nTags: ${post.data.tags.join(", ")}`;
+    if (post.data.repo) header += `\nRepo: ${post.data.repo}`;
+    if (post.data.website) header += `\nWebsite: ${post.data.website}`;
+    header += `\nURL: ${url}`;
+
+    articles.push(`${header}\n\n${cleaned}`);
+  }
+
+  const body = `# Daniel Ho — Full Site Content
+
+> Software engineer, writer, potter.
+
+This file contains the complete text content of ${context.site} in clean markdown for LLM consumption.
+
+---
+
+${articles.join("\n\n---\n\n")}
+`;
+
+  return new Response(body, {
+    headers: { "Content-Type": "text/plain; charset=utf-8" },
+  });
+}

src/pages/llms.txt.ts

@@ -0,0 +1,63 @@
+import type { APIContext } from "astro";
+import { getCollection } from "astro:content";
+
+export async function GET(context: APIContext) {
+  const posts = (await getCollection("writing"))
+    .filter((post) => !post.data.draft)
+    .sort((a, b) => b.data.pubDate.valueOf() - a.data.pubDate.valueOf());
+
+  const site = context.site!;
+
+  const writingEntries = posts
+    .map((post) => {
+      const url = new URL(`/writing/${post.id}/`, site);
+      return `- [${post.data.title}](${url}): ${post.data.description}`;
+    })
+    .join("\n");
+
+  const body = `# Daniel Ho
+
+> Software engineer, writer, potter.
+
+Personal website of Daniel Ho. I build software, write about what I'm learning, and make pottery when I'm away from the screen.
+
+## About
+
+Daniel Ho is a software engineer who has worked at Anime, Azuki, Koop (technical cofounder), Flexport, Meta, Salesforce, and NASA JPL. He studied at USC where he researched NLP and commonsense reasoning in pre-trained language models (BERT, GPT-2).
+
+## Links
+
+- Website: ${site}
+- GitHub: https://github.com/donutdaniel
+- X/Twitter: https://x.com/donutdho
+- LinkedIn: https://linkedin.com/in/danielho54
+- Email: danielho54@gmail.com
+
+## Writing
+
+${writingEntries}
+
+## Publications
+
+- [RICA: Evaluating Robust Inference Capabilities Based on Commonsense Axioms](https://arxiv.org/abs/2005.00782)
+
+## Work
+
+- Anime (anime.com): Social product. News aggregation, streaming, and experiences for anime fans.
+- Azuki (azuki.com): Web3 team. Anime NFT brand and community platform.
+- Koop (koop.xyz): Technical cofounder. On-chain rails for creators and communities.
+- Flexport: Platform and infra team. Dev environments, microservices, shipment domain modeling.
+- Meta: Messenger team. Infrastructure, profile and stories processing.
+- Salesforce: Core platform. File streaming service for real-time collaboration.
+- NASA JPL: Deep learning for CNN noise detection.
+- USC: NLP research on commonsense reasoning. TA for Discrete Math and Algorithms.
+
+## Full Content
+
+For complete article text in clean markdown, see: ${new URL("/llms-full.txt", site)}
+`;
+
+  return new Response(body, {
+    headers: { "Content-Type": "text/plain; charset=utf-8" },
+  });
+}