feat(installer): use GitHub API as primary fetch with raw CDN fallback (#2248)

* feat(installer): use GitHub API as primary fetch with raw CDN fallback

Corporate proxies commonly block raw.githubusercontent.com while allowing
api.github.com. Add fetchGitHubFile() to RegistryClient that tries the
GitHub Contents API first, falling back to the raw CDN transparently.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix(installer): cap redirect depth and preserve dual-fallback errors

Add maxRedirects parameter to fetch() and _fetchWithHeaders() to prevent
unbounded redirect recursion. Wrap CDN fallback in try/catch and throw
AggregateError with both API and CDN errors for better diagnostics.
Extract marketplace repo coordinates into named constants in
external-manager.

* chore(installer): drop unused fetchJson and fetchGitHubJson

Neither method has any callers. Also drop the corresponding test.

* refactor(test): fold registry tests into test-installation-components

No reason for RegistryClient tests to be a separate runner — the same
file already tests the registry consumers in Suite 33. Drop test:registry
from package.json scripts and quality gate.

* fix(installer): include URL, API message, and rate-limit info in HTTP errors

Non-2xx responses previously yielded bare `HTTP 403`. Now surface the
request URL, GitHub's JSON error message (or body snippet), X-RateLimit-Reset
when quota is exhausted, and Retry-After. Turns a mystery 403 into
'rate limit exhausted; resets at 2026-04-15T18:00:00Z' — the difference
between 'try GITHUB_TOKEN' and a wild goose chase.

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: alexeyv

test/test-installation-components.js

@@ -1926,6 +1926,112 @@ async function runTests() {
 
   console.log('');
 
+  // ============================================================
+  // Test Suite 34: RegistryClient GitHub API Cascade
+  // ============================================================
+  console.log(`${colors.yellow}Test Suite 34: RegistryClient GitHub API Cascade${colors.reset}\n`);
+
+  {
+    const { RegistryClient } = require('../tools/installer/modules/registry-client');
+
+    // Build a RegistryClient with stubbed fetch paths so we can assert on cascade behavior
+    // without making real network calls.
+    function createStubbedClient({ apiResult, rawResult }) {
+      const client = new RegistryClient();
+      const calls = [];
+
+      // Stub _fetchWithHeaders (GitHub API path)
+      client._fetchWithHeaders = async (url) => {
+        calls.push(`api:${url}`);
+        if (apiResult instanceof Error) throw apiResult;
+        return apiResult;
+      };
+
+      // Stub fetch (raw CDN path) — only intercept raw.githubusercontent.com calls
+      const originalFetch = client.fetch.bind(client);
+      client.fetch = async (url, timeout) => {
+        if (url.includes('raw.githubusercontent.com')) {
+          calls.push(`raw:${url}`);
+          if (rawResult instanceof Error) throw rawResult;
+          return rawResult;
+        }
+        return originalFetch(url, timeout);
+      };
+
+      return { client, calls };
+    }
+
+    // --- API success skips raw CDN ---
+    {
+      const { client, calls } = createStubbedClient({ apiResult: 'api-content', rawResult: 'raw-content' });
+      const result = await client.fetchGitHubFile('owner', 'repo', 'path/file.txt', 'main');
+
+      assert(result === 'api-content', 'RegistryClient API success returns API content');
+      assert(calls.length === 1, 'RegistryClient API success makes exactly one call');
+      assert(calls[0].startsWith('api:'), 'RegistryClient API success calls API endpoint');
+    }
+
+    // --- API failure falls back to raw CDN ---
+    {
+      const { client, calls } = createStubbedClient({ apiResult: new Error('HTTP 403'), rawResult: 'raw-content' });
+      const result = await client.fetchGitHubFile('owner', 'repo', 'path/file.txt', 'main');
+
+      assert(result === 'raw-content', 'RegistryClient API failure returns raw CDN content');
+      assert(calls.length === 2, 'RegistryClient API failure makes two calls');
+      assert(calls[0].startsWith('api:'), 'RegistryClient first call is to API');
+      assert(calls[1].startsWith('raw:'), 'RegistryClient second call is to raw CDN');
+    }
+
+    // --- Both endpoints failing throws ---
+    {
+      const { client } = createStubbedClient({ apiResult: new Error('HTTP 403'), rawResult: new Error('HTTP 404') });
+      let threw = false;
+      try {
+        await client.fetchGitHubFile('owner', 'repo', 'path/file.txt', 'main');
+      } catch {
+        threw = true;
+      }
+      assert(threw, 'RegistryClient both endpoints failing throws an error');
+    }
+
+    // --- API URL construction ---
+    {
+      const { client, calls } = createStubbedClient({ apiResult: 'content', rawResult: 'content' });
+      await client.fetchGitHubFile('bmad-code-org', 'bmad-plugins-marketplace', 'registry/official.yaml', 'main');
+
+      const apiCall = calls[0];
+      assert(
+        apiCall.includes('api.github.com/repos/bmad-code-org/bmad-plugins-marketplace/contents/registry/official.yaml'),
+        'RegistryClient API URL contains correct path',
+      );
+      assert(apiCall.includes('ref=main'), 'RegistryClient API URL contains ref parameter');
+    }
+
+    // --- Raw CDN URL construction ---
+    {
+      const { client, calls } = createStubbedClient({ apiResult: new Error('fail'), rawResult: 'content' });
+      await client.fetchGitHubFile('bmad-code-org', 'bmad-plugins-marketplace', 'registry/official.yaml', 'main');
+
+      const rawCall = calls[1];
+      assert(
+        rawCall.includes('raw.githubusercontent.com/bmad-code-org/bmad-plugins-marketplace/main/registry/official.yaml'),
+        'RegistryClient raw CDN URL contains correct path',
+      );
+    }
+
+    // --- fetchGitHubYaml parses YAML ---
+    {
+      const yamlContent = 'modules:\n  - name: test\n    description: A test module\n';
+      const { client } = createStubbedClient({ apiResult: yamlContent, rawResult: yamlContent });
+      const result = await client.fetchGitHubYaml('owner', 'repo', 'file.yaml', 'main');
+
+      assert(Array.isArray(result.modules), 'fetchGitHubYaml parses YAML correctly');
+      assert(result.modules[0].name === 'test', 'fetchGitHubYaml preserves YAML values');
+    }
+  }
+
+  console.log('');
+
   // ============================================================
   // Summary
   // ============================================================

tools/installer/modules/community-manager.js

@@ -5,9 +5,9 @@ const { execSync } = require('node:child_process');
 const prompts = require('../prompts');
 const { RegistryClient } = require('./registry-client');
 
-const MARKETPLACE_BASE = 'https://raw.githubusercontent.com/bmad-code-org/bmad-plugins-marketplace/main';
-const COMMUNITY_INDEX_URL = `${MARKETPLACE_BASE}/registry/community-index.yaml`;
-const CATEGORIES_URL = `${MARKETPLACE_BASE}/categories.yaml`;
+const MARKETPLACE_OWNER = 'bmad-code-org';
+const MARKETPLACE_REPO = 'bmad-plugins-marketplace';
+const MARKETPLACE_REF = 'main';
 
 /**
  * Manages community modules from the BMad marketplace registry.
@@ -33,7 +33,12 @@ class CommunityModuleManager {
     if (this._cachedIndex) return this._cachedIndex;
 
     try {
-      const config = await this._client.fetchYaml(COMMUNITY_INDEX_URL);
+      const config = await this._client.fetchGitHubYaml(
+        MARKETPLACE_OWNER,
+        MARKETPLACE_REPO,
+        'registry/community-index.yaml',
+        MARKETPLACE_REF,
+      );
       if (config?.modules?.length) {
         this._cachedIndex = config;
         return config;
@@ -54,7 +59,7 @@ class CommunityModuleManager {
     if (this._cachedCategories) return this._cachedCategories;
 
     try {
-      const config = await this._client.fetchYaml(CATEGORIES_URL);
+      const config = await this._client.fetchGitHubYaml(MARKETPLACE_OWNER, MARKETPLACE_REPO, 'categories.yaml', MARKETPLACE_REF);
       if (config?.categories) {
         this._cachedCategories = config;
         return config;

tools/installer/modules/external-manager.js

@@ -6,7 +6,9 @@ const yaml = require('yaml');
 const prompts = require('../prompts');
 const { RegistryClient } = require('./registry-client');
 
-const REGISTRY_RAW_URL = 'https://raw.githubusercontent.com/bmad-code-org/bmad-plugins-marketplace/main/registry/official.yaml';
+const MARKETPLACE_OWNER = 'bmad-code-org';
+const MARKETPLACE_REPO = 'bmad-plugins-marketplace';
+const MARKETPLACE_REF = 'main';
 const FALLBACK_CONFIG_PATH = path.join(__dirname, 'registry-fallback.yaml');
 
 /**
@@ -33,8 +35,7 @@ class ExternalModuleManager {
 
     // Try remote registry first
     try {
-      const content = await this._client.fetch(REGISTRY_RAW_URL);
-      const config = yaml.parse(content);
+      const config = await this._client.fetchGitHubYaml(MARKETPLACE_OWNER, MARKETPLACE_REPO, 'registry/official.yaml', MARKETPLACE_REF);
       if (config?.modules?.length) {
         this.cachedModules = config;
         return config;

tools/installer/modules/registry-client.js

@@ -1,6 +1,37 @@
 const https = require('node:https');
 const yaml = require('yaml');
 
+/**
+ * Build a rich Error from a non-2xx response. Includes the URL, the GitHub
+ * JSON error message (or a truncated body snippet), rate-limit reset time,
+ * and Retry-After — anything present that would help a user recover.
+ */
+function buildHttpError(url, res, body) {
+  const parts = [`HTTP ${res.statusCode} ${url}`];
+
+  if (body) {
+    try {
+      const parsed = JSON.parse(body);
+      if (parsed.message) parts.push(parsed.message);
+      if (parsed.documentation_url) parts.push(`(see ${parsed.documentation_url})`);
+    } catch {
+      const snippet = body.slice(0, 200).trim();
+      if (snippet) parts.push(snippet);
+    }
+  }
+
+  const remaining = res.headers['x-ratelimit-remaining'];
+  const reset = res.headers['x-ratelimit-reset'];
+  if (remaining === '0' && reset) {
+    parts.push(`rate limit exhausted; resets at ${new Date(Number(reset) * 1000).toISOString()}`);
+  }
+
+  const retryAfter = res.headers['retry-after'];
+  if (retryAfter) parts.push(`retry after ${retryAfter}`);
+
+  return new Error(parts.join(' — '));
+}
+
 /**
  * Shared HTTP client for fetching registry data from GitHub.
  * Used by ExternalModuleManager, CommunityModuleManager, and CustomModuleManager.
@@ -12,25 +43,31 @@ class RegistryClient {
 
   /**
    * Fetch a URL and return the response body as a string.
-   * Follows one redirect (GitHub sometimes 301s).
+   * Follows up to 3 redirects (GitHub sometimes 301s).
    * @param {string} url - URL to fetch
    * @param {number} [timeout] - Timeout in ms (overrides default)
+   * @param {number} [maxRedirects=3] - Maximum redirects to follow
    * @returns {Promise<string>} Response body
    */
-  fetch(url, timeout) {
+  fetch(url, timeout, maxRedirects = 3) {
     const timeoutMs = timeout || this.timeout;
     return new Promise((resolve, reject) => {
       const req = https
         .get(url, { timeout: timeoutMs }, (res) => {
           if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
-            return this.fetch(res.headers.location, timeoutMs).then(resolve, reject);
-          }
-          if (res.statusCode !== 200) {
-            return reject(new Error(`HTTP ${res.statusCode}`));
+            if (maxRedirects <= 0) {
+              return reject(new Error('Too many redirects'));
+            }
+            return this.fetch(res.headers.location, timeoutMs, maxRedirects - 1).then(resolve, reject);
           }
           let data = '';
           res.on('data', (chunk) => (data += chunk));
-          res.on('end', () => resolve(data));
+          res.on('end', () => {
+            if (res.statusCode !== 200) {
+              return reject(buildHttpError(url, res, data));
+            }
+            resolve(data);
+          });
         })
         .on('error', reject)
         .on('timeout', () => {
@@ -50,6 +87,101 @@ class RegistryClient {
     const content = await this.fetch(url, timeout);
     return yaml.parse(content);
   }
+
+  /**
+   * Fetch a file from a GitHub repo using the Contents API first,
+   * falling back to raw.githubusercontent.com if the API fails.
+   *
+   * The API endpoint (`api.github.com`) is tried first because corporate
+   * proxies commonly block `raw.githubusercontent.com` while allowing
+   * `api.github.com` under the "Software Development" category.
+   *
+   * @param {string} owner - Repository owner (e.g., 'bmad-code-org')
+   * @param {string} repo  - Repository name (e.g., 'bmad-plugins-marketplace')
+   * @param {string} filePath - Path within the repo (e.g., 'registry/official.yaml')
+   * @param {string} ref   - Git ref (branch, tag, or SHA; e.g., 'main')
+   * @param {number} [timeout] - Timeout in ms (overrides default)
+   * @returns {Promise<string>} Raw file content
+   */
+  async fetchGitHubFile(owner, repo, filePath, ref, timeout) {
+    const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${filePath}?ref=${ref}`;
+    const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${filePath}`;
+
+    // Try GitHub Contents API first (with raw content accept header)
+    try {
+      return await this._fetchWithHeaders(apiUrl, { Accept: 'application/vnd.github.raw+json' }, timeout);
+    } catch (apiError) {
+      // API failed — fall back to raw CDN
+      try {
+        return await this.fetch(rawUrl, timeout);
+      } catch (cdnError) {
+        throw new AggregateError([apiError, cdnError], `Both GitHub API and raw CDN failed for ${filePath}`);
+      }
+    }
+  }
+
+  /**
+   * Fetch a file from GitHub and parse as YAML.
+   * @param {string} owner - Repository owner
+   * @param {string} repo  - Repository name
+   * @param {string} filePath - Path within the repo
+   * @param {string} ref   - Git ref
+   * @param {number} [timeout] - Timeout in ms
+   * @returns {Promise<Object>} Parsed YAML content
+   */
+  async fetchGitHubYaml(owner, repo, filePath, ref, timeout) {
+    const content = await this.fetchGitHubFile(owner, repo, filePath, ref, timeout);
+    return yaml.parse(content);
+  }
+
+  /**
+   * Fetch a URL with custom headers. Used for GitHub API requests.
+   * Follows up to 3 redirects.
+   * @param {string} url - URL to fetch
+   * @param {Object} headers - Request headers
+   * @param {number} [timeout] - Timeout in ms
+   * @param {number} [maxRedirects=3] - Maximum redirects to follow
+   * @returns {Promise<string>} Response body
+   * @private
+   */
+  _fetchWithHeaders(url, headers, timeout, maxRedirects = 3) {
+    const timeoutMs = timeout || this.timeout;
+    const parsed = new URL(url);
+    const options = {
+      hostname: parsed.hostname,
+      path: parsed.pathname + parsed.search,
+      timeout: timeoutMs,
+      headers: {
+        'User-Agent': 'bmad-installer',
+        ...headers,
+      },
+    };
+
+    return new Promise((resolve, reject) => {
+      const req = https
+        .get(options, (res) => {
+          if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+            if (maxRedirects <= 0) {
+              return reject(new Error('Too many redirects'));
+            }
+            return this._fetchWithHeaders(res.headers.location, headers, timeoutMs, maxRedirects - 1).then(resolve, reject);
+          }
+          let data = '';
+          res.on('data', (chunk) => (data += chunk));
+          res.on('end', () => {
+            if (res.statusCode !== 200) {
+              return reject(buildHttpError(url, res, data));
+            }
+            resolve(data);
+          });
+        })
+        .on('error', reject)
+        .on('timeout', () => {
+          req.destroy();
+          reject(new Error('Request timed out'));
+        });
+    });
+  }
 }
 
 module.exports = { RegistryClient };