ensure robots.txt via test (#650)

* ensure robots.txt via test

* Add test for llms.txt reference in robots.txt

* Comment llms.txt reference to comply with robots.txt spec

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>

Author: evantahler

public/robots.txt

@@ -7,3 +7,6 @@ Host: https://docs.arcade.dev
 
 # Sitemaps
 Sitemap: https://docs.arcade.dev/sitemap.xml
+
+# AI Agent Resources
+# https://docs.arcade.dev/llms.txt

tests/sitemap.test.ts

@@ -1,3 +1,5 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
 import { expect, test } from "vitest";
 
 test("sitemap lists expected URLs", async () => {
@@ -33,3 +35,19 @@ test("sitemap lists expected URLs", async () => {
     }
   }
 });
+
+test("robots.txt references the sitemap", () => {
+  const robotsPath = join(process.cwd(), "public", "robots.txt");
+  const robotsContent = readFileSync(robotsPath, "utf-8");
+
+  expect(robotsContent).toContain(
+    "Sitemap: https://docs.arcade.dev/sitemap.xml"
+  );
+});
+
+test("robots.txt references llms.txt", () => {
+  const robotsPath = join(process.cwd(), "public", "robots.txt");
+  const robotsContent = readFileSync(robotsPath, "utf-8");
+
+  expect(robotsContent).toContain("# https://docs.arcade.dev/llms.txt");
+});