Merge pull request #652 from github/combine-llms-txt-generation

Combine llms.txt generation with website data generation

Author: aaronpowell

eng/generate-llms-txt.mjs

@@ -695,6 +695,84 @@ function generateSamplesData() {
   };
 }
 
+/**
+ * Generate llms.txt content according to the llmstxt.org specification
+ */
+function generateLlmsTxt(agents, prompts, instructions, skills) {
+  let content = "";
+
+  // H1 header (required)
+  content += "# Awesome GitHub Copilot\n\n";
+
+  // Summary blockquote (optional but recommended)
+  content +=
+    "> A community-driven collection of custom agents, prompts, instructions, and skills to enhance GitHub Copilot experiences across various domains, languages, and use cases.\n\n";
+
+  // Add overview section
+  content += "## Overview\n\n";
+  content +=
+    "This repository provides resources to customize and enhance GitHub Copilot:\n\n";
+  content +=
+    "- **Agents**: Specialized GitHub Copilot agents that integrate with MCP servers\n";
+  content +=
+    "- **Prompts**: Task-specific prompts for code generation and problem-solving\n";
+  content +=
+    "- **Instructions**: Coding standards and best practices applied to specific file patterns\n";
+  content +=
+    "- **Skills**: Self-contained folders with instructions and bundled resources for specialized tasks\n\n";
+
+  // Process Agents
+  content += "## Agents\n\n";
+  for (const agent of agents) {
+    const description = (agent.description || "No description available").replace(/\s+/g, " ").trim();
+    content += `- [${agent.title}](${agent.path}): ${description}\n`;
+  }
+  content += "\n";
+
+  // Process Prompts
+  content += "## Prompts\n\n";
+  for (const prompt of prompts) {
+    const description = (prompt.description || "No description available").replace(/\s+/g, " ").trim();
+    content += `- [${prompt.title}](${prompt.path}): ${description}\n`;
+  }
+  content += "\n";
+
+  // Process Instructions
+  content += "## Instructions\n\n";
+  for (const instruction of instructions) {
+    const description = (instruction.description || "No description available").replace(/\s+/g, " ").trim();
+    content += `- [${instruction.title}](${instruction.path}): ${description}\n`;
+  }
+  content += "\n";
+
+  // Process Skills
+  content += "## Skills\n\n";
+  for (const skill of skills) {
+    const description = (skill.description || "No description available").replace(/\s+/g, " ").trim();
+    content += `- [${skill.title}](${skill.skillFile}): ${description}\n`;
+  }
+  content += "\n";
+
+  // Add documentation links
+  content += "## Documentation\n\n";
+  content +=
+    "- [README.md](README.md): Main documentation and getting started guide\n";
+  content +=
+    "- [CONTRIBUTING.md](CONTRIBUTING.md): Guidelines for contributing to this repository\n";
+  content +=
+    "- [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md): Community standards and expectations\n";
+  content += "- [SECURITY.md](SECURITY.md): Security policies and reporting\n";
+  content += "- [AGENTS.md](AGENTS.md): Project overview and setup commands\n\n";
+
+  // Add repository information
+  content += "## Repository\n\n";
+  content += "- **GitHub**: https://github.com/github/awesome-copilot\n";
+  content += "- **License**: MIT\n";
+  content += "- **Website**: https://github.github.io/awesome-copilot\n";
+
+  return content;
+}
+
 /**
  * Main function
  */
@@ -816,6 +894,15 @@ async function main() {
   );
 
   console.log(`\n✓ All data written to website/public/data/`);
+
+  // Generate llms.txt
+  const llmsTxtContent = generateLlmsTxt(agents, prompts, instructions, skills);
+  fs.writeFileSync(
+    path.join(ROOT_FOLDER, "website", "public", "llms.txt"),
+    llmsTxtContent,
+    "utf8"
+  );
+  console.log(`✓ llms.txt generated with ${agents.length} agents, ${prompts.length} prompts, ${instructions.length} instructions, ${skills.length} skills`);
 }
 
 main().catch((err) => {

eng/generate-website-data.mjs

@@ -7,7 +7,6 @@
   "scripts": {
     "start": "npm run build",
     "build": "node ./eng/update-readme.mjs",
-    "llms:generate": "node ./eng/generate-llms-txt.mjs",
     "contributors:add": "all-contributors add",
     "contributors:report": "node ./eng/contributor-report.mjs",
     "contributors:generate": "all-contributors generate",
@@ -18,7 +17,7 @@
     "skill:create": "node ./eng/create-skill.mjs",
     "website:data": "node ./eng/generate-website-data.mjs",
     "website:dev": "npm run website:data && npm run --prefix website dev",
-    "website:build": "npm run build && npm run website:data && node ./eng/generate-llms-txt.mjs website/public && npm run --prefix website build",
+    "website:build": "npm run build && npm run website:data && npm run --prefix website build",
     "website:preview": "npm run --prefix website preview"
   },
   "repository": {