Use worker-served HTML to populate Gumroad products (#19)

- update the Gumroad Cloudflare worker to return cached profile HTML for a username
- adjust the Gumroad link builder to fetch that HTML and extract product links client-side

------
[Codex Task](https://chatgpt.com/codex/tasks/task_e_6916ee4d36e88325a6b7ac48e21b03a9)

Author: rodrigogiraoserrao

cloudflare-workers/gumroad-products/worker.js

@@ -80,7 +80,6 @@ export default {
             return respondJSON(origin, allowed, { error: "Invalid or missing Gumroad username." }, 400);
         }
 
-        const profileUrl = `https://${u}.gumroad.com/`;
         const cache = caches.default;
         const dataKey = new Request(`https://gumroad-products.internal/cache?u=${encodeURIComponent(u)}`);
         const metaKey = new Request(`https://gumroad-products.internal/meta?u=${encodeURIComponent(u)}`);
@@ -96,7 +95,7 @@ export default {
         if (cachedBody && age <= S_MAX_AGE) {
             return new Response(cachedBody, {
                 headers: {
-                    "Content-Type": "application/json; charset=utf-8",
+                    "Content-Type": "text/html; charset=utf-8",
                     "Cache-Control": `public, max-age=${S_MAX_AGE}`,
                     "X-Cache": "HIT",
                     "X-Upstream-Status": "none",
@@ -136,7 +135,7 @@ export default {
             if (cachedBody && age <= (S_MAX_AGE + STALE_WINDOW)) {
                 return new Response(cachedBody, {
                     headers: {
-                        "Content-Type": "application/json; charset=utf-8",
+                        "Content-Type": "text/html; charset=utf-8",
                         "Cache-Control": `public, max-age=0, stale-while-revalidate=${STALE_WINDOW}`,
                         "X-Cache": "STALE",
                         "X-Upstream-Status": String(upstream.status),
@@ -153,22 +152,11 @@ export default {
 
         const html = await upstream.text();
 
-        const products = extractProducts(html);
-
-        const payload = {
-            username: u,
-            profile_url: profileUrl,
-            count: products.length,
-            products,
-            fetched_at: new Date().toISOString(),
-        };
-        const body = JSON.stringify(payload);
-
         // Update cache
         const now = Math.floor(Date.now() / 1000);
-        const dataResp = new Response(body, {
+        const dataResp = new Response(html, {
             headers: {
-                "Content-Type": "application/json; charset=utf-8",
+                "Content-Type": "text/html; charset=utf-8",
                 "Cache-Control": `public, max-age=${S_MAX_AGE}`,
             },
         });
@@ -178,9 +166,9 @@ export default {
         ctx.waitUntil(cache.put(dataKey, dataResp.clone()));
         ctx.waitUntil(cache.put(metaKey, metaResp.clone()));
 
-        return new Response(body, {
+        return new Response(html, {
             headers: {
-                "Content-Type": "application/json; charset=utf-8",
+                "Content-Type": "text/html; charset=utf-8",
                 "Cache-Control": `public, max-age=${S_MAX_AGE}`,
                 "X-Cache": cachedBody ? "MISS-REVAL" : "MISS",
                 "X-Upstream-Status": "200",
@@ -189,31 +177,3 @@ export default {
         });
     },
 };
-
-function extractProducts(html) {
-    // Lightweight HTML parsing without DOM: heuristic regex over links.
-    // For more robustness, you could use an HTML parser lib with Workers Bundler.
-    const linkRe = /<a\b[^>]*href=["']([^"']*\/l\/[^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi;
-    const tagRe = /<\/?[^>]+>/g;
-    const nbspRe = /&nbsp;/g;
-
-    const seen = new Set();
-    const items = [];
-    let m;
-    while ((m = linkRe.exec(html)) !== null) {
-        const href = m[1];
-        let title = m[2].replace(tagRe, '').replace(nbspRe, ' ').trim();
-        if (!title) continue;
-
-        // Normalize absolute vs relative
-        const url = href.startsWith('http') ? href : new URL(href, 'https://example.com').href;
-        const slug = url.split('/').filter(Boolean).pop();
-
-        const key = url + '|' + title;
-        if (seen.has(key)) continue;
-        seen.add(key);
-
-        items.push({ title, url, slug });
-    }
-    return items;
-}

gumroad-links.html

@@ -653,34 +653,49 @@ <h3 style="margin: 0; font-size: 1.1rem;">Included parameters</h3>
                 }
             }
 
-            function parseProductsFromProfile(html, profileUrl, username) {
+            function parseProductsFromHTML(html, username) {
                 const parser = new DOMParser();
                 const doc = parser.parseFromString(html, 'text/html');
-                const productCards = Array.from(doc.querySelectorAll('article.product-card'));
+                const anchors = Array.from(doc.querySelectorAll('a[href]'));
                 const products = new Map();
+                const baseUrl = `https://${username}.gumroad.com/`;
 
-                for (const card of productCards) {
-                    const link = card.querySelector('a[href]');
-                    if (!link) {
+                for (const anchor of anchors) {
+                    const rawHref = anchor.getAttribute('href');
+                    if (!rawHref) {
                         continue;
                     }
+
                     let resolvedUrl;
                     try {
-                        resolvedUrl = new URL(link.getAttribute('href'), profileUrl);
+                        resolvedUrl = new URL(rawHref, baseUrl);
                     } catch (error) {
                         continue;
                     }
+
+                    if (!resolvedUrl.hostname.endsWith('gumroad.com')) {
+                        continue;
+                    }
+
+                    const segments = resolvedUrl.pathname.split('/').filter(Boolean);
+                    if (segments.length < 2 || segments[0].toLowerCase() !== 'l') {
+                        continue;
+                    }
+
                     const slug = sanitiseSlug(resolvedUrl.pathname);
                     if (!slug || products.has(slug)) {
                         continue;
                     }
-                    const titleElement = card.querySelector('[itemprop="name"], h4, h3') || link;
-                    const name = (titleElement.textContent || '').trim();
-                    const productUrl = composeProductUrl(username, slug) || resolvedUrl.href;
+
+                    const textContent = (anchor.textContent || '').replace(/\s+/g, ' ').trim();
+                    const titleAttr = (anchor.getAttribute('title') || '').trim();
+                    const label = textContent || titleAttr || slug;
+                    const canonicalUrl = composeProductUrl(username, slug) || resolvedUrl.href;
+
                     products.set(slug, {
                         slug,
-                        name: name || slug,
-                        url: productUrl,
+                        name: label,
+                        url: canonicalUrl,
                     });
                 }
 
@@ -780,12 +795,8 @@ <h3 style="margin: 0; font-size: 1.1rem;">Included parameters</h3>
                             throw new Error(`Request failed: ${response.status} (X-Cache=${xCache}, Upstream=${xUp})`);
                         }
 
-                        const data = await response.json();
-                        const products = (data.products || []).map((p) => ({
-                            name: p.name || p.title || p.slug || '',
-                            slug: p.slug,
-                            url: p.url,
-                        })).filter((product) => Boolean(product.slug));
+                        const html = await response.text();
+                        const products = parseProductsFromHTML(html, cleanUsername);
 
                         if (products.length === 0) {
                             populateProductSelect([]);