[pull] main from dubinc:main

apps/web/app/api/metatags/route.ts → apps/web/app/api/links/metatags/route.ts

@@ -6,37 +6,41 @@ import { getMetaTags } from "./utils";
 
 export const runtime = "edge";
 
-const CORS_HEADERS = {
-  "Access-Control-Allow-Origin": "*",
-  "Access-Control-Allow-Methods": "GET, OPTIONS",
-};
-
 export async function GET(req: NextRequest) {
   try {
+    const origin = req.headers.get("origin");
+    // Validate the origin header and set CORS headers accordingly
+    const corsHeaders = {
+      "Access-Control-Allow-Methods": "GET",
+      "Access-Control-Allow-Headers": "Content-Type",
+    };
+
+    if (origin && origin.endsWith(".dub.co")) {
+      corsHeaders["Access-Control-Allow-Origin"] = origin;
+    }
+
+    // Validate URL parameter
     const { url } = getUrlQuerySchema.parse({
       url: req.nextUrl.searchParams.get("url"),
     });
 
+    // Rate limit by IP
     await ratelimitOrThrow(req, "metatags");
 
+    // Get metatags
     const metatags = await getMetaTags(url);
+
+    // Return response
     return NextResponse.json(
       {
         ...metatags,
         poweredBy: "Dub.co - Link management for modern marketing teams",
       },
       {
-        headers: CORS_HEADERS,
+        headers: corsHeaders,
       },
     );
   } catch (error) {
-    return handleAndReturnErrorResponse(error, CORS_HEADERS);
+    return handleAndReturnErrorResponse(error);
   }
 }
-
-export function OPTIONS() {
-  return new Response(null, {
-    status: 204,
-    headers: CORS_HEADERS,
-  });
-}

apps/web/app/api/metatags/utils.ts → apps/web/app/api/links/metatags/utils.ts

@@ -5,13 +5,36 @@ import he from "he";
 import { parse } from "node-html-parser";
 
 export const getHtml = async (url: string) => {
-  return await fetchWithTimeout(url, {
-    headers: {
-      "User-Agent": "Dub.co Metatags API (https://api.dub.co/metatags)",
-    },
-  })
-    .then((r) => r.text())
-    .catch(() => null);
+  try {
+    const response = await fetchWithTimeout(url);
+
+    if (!response.ok) {
+      // If we get a 406 or other error, check if it's a Cloudflare-protected site
+      const isCloudflare = response.headers.get("server") === "cloudflare";
+      if (isCloudflare) {
+        console.warn(`Cloudflare-protected site detected: ${url}`);
+        return null;
+      }
+      console.error(`HTTP error! status: ${response.status} for URL: ${url}`);
+      return null;
+    }
+
+    const text = await response.text();
+
+    // Check if the response contains Cloudflare's challenge page
+    if (
+      text.includes("challenge-platform") ||
+      text.includes("cf-browser-verification")
+    ) {
+      console.warn(`Cloudflare challenge page detected for: ${url}`);
+      return null;
+    }
+
+    return text;
+  } catch (error) {
+    console.error(`Error fetching ${url}:`, error);
+    return null;
+  }
 };
 
 export const getHeadChildNodes = (html) => {
@@ -47,15 +70,42 @@ export const getRelativeUrl = (url: string, imageUrl: string) => {
   return new URL(imageUrl, baseURL).toString();
 };
 
-export const getMetaTags = async (url: string) => {
-  const html = await getHtml(url);
-  if (!html) {
+const generateFallbackMetadata = (url: string) => {
+  try {
+    const parsedUrl = new URL(url);
+    const hostname = parsedUrl.hostname;
+    const path = parsedUrl.pathname;
+
+    // Clean up the path for title
+    const pathParts = path.split("/").filter(Boolean);
+    const lastPathPart = pathParts[pathParts.length - 1] || "";
+    const formattedPath = lastPathPart
+      .split("-")
+      .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+      .join(" ");
+
+    return {
+      title: formattedPath || hostname.replace(/^www\./, ""),
+      description: `Visit ${hostname}${path}`,
+      image: null,
+    };
+  } catch (e) {
     return {
       title: url,
-      description: "No description",
+      description: "No description available",
       image: null,
     };
   }
+};
+
+export const getMetaTags = async (url: string) => {
+  const html = await getHtml(url);
+  if (!html) {
+    // If we couldn't fetch the HTML (e.g., due to Cloudflare protection),
+    // generate fallback metadata from the URL
+    return generateFallbackMetadata(url);
+  }
+
   const { metaTags, title: titleTag, linkTags } = getHeadChildNodes(html);
 
   let object = {};

apps/web/app/api/track/lead/route.ts

@@ -44,8 +44,8 @@ export const POST = withWorkspace(
     } = trackLeadRequestSchema
       .extend({
         // add backwards compatibility
-        externalId: z.string().optional(),
-        customerId: z.string().optional(),
+        externalId: z.string().nullish(),
+        customerId: z.string().nullish(),
       })
       .parse(body);
 

apps/web/app/api/track/sale/route.ts

@@ -42,8 +42,8 @@ export const POST = withWorkspace(
     } = trackSaleRequestSchema
       .extend({
         // add backwards compatibility
-        externalId: z.string().optional(),
-        customerId: z.string().optional(),
+        externalId: z.string().nullish(),
+        customerId: z.string().nullish(),
       })
       .parse(body);
 

apps/web/app/app.dub.co/(auth)/oauth/authorize/page.tsx

@@ -5,7 +5,7 @@ import { authorizeRequestSchema } from "@/lib/zod/schemas/oauth";
 import EmptyState from "@/ui/shared/empty-state";
 import { BlurImage, Logo } from "@dub/ui";
 import { CircleWarning, CubeSettings } from "@dub/ui/icons";
-import { HOME_DOMAIN, constructMetadata } from "@dub/utils";
+import { constructMetadata } from "@dub/utils";
 import { ArrowLeftRight } from "lucide-react";
 import { redirect } from "next/navigation";
 import { Suspense } from "react";
@@ -62,7 +62,7 @@ export default async function Authorize({
             )}
           </a>
           <ArrowLeftRight className="size-5 text-neutral-500" />
-          <a href={HOME_DOMAIN} target="_blank" rel="noreferrer">
+          <a href="https://dub.co" target="_blank" rel="noreferrer">
             <Logo className="size-12" />
           </a>
         </div>

apps/web/app/app.dub.co/(onboarding)/onboarding/(steps)/link/form.tsx

@@ -73,7 +73,7 @@ export function Form() {
           // If url is valid, continue to generate metatags, else return null
           new URL(debouncedUrl);
           setLoadingPreviewImage(true);
-          const res = await fetch(`/api/metatags?url=${debouncedUrl}`);
+          const res = await fetch(`/api/links/metatags?url=${debouncedUrl}`);
           if (res.ok) {
             const results = await res.json();
             setPreviewImage(results.image);

apps/web/app/cloaked/[url]/page.tsx

@@ -3,7 +3,7 @@ import {
   constructMetadata,
   getApexDomain,
 } from "@dub/utils";
-import { getMetaTags } from "app/api/metatags/utils";
+import { getMetaTags } from "app/api/links/metatags/utils";
 
 export const runtime = "edge";
 export const fetchCache = "force-no-store";

apps/web/lib/middleware/api.ts

@@ -1,18 +1,14 @@
 import { parse } from "@/lib/middleware/utils";
-import { HOME_DOMAIN } from "@dub/utils";
 import { NextRequest, NextResponse } from "next/server";
 
 export default function ApiMiddleware(req: NextRequest) {
-  const { path, fullPath } = parse(req);
+  const { fullPath } = parse(req);
 
-  // special case for /metatags
-  if (path === "/metatags") {
-    const url = req.nextUrl.searchParams.get("url");
-    if (!url) {
-      return NextResponse.redirect(`${HOME_DOMAIN}/tools/metatags`, {
-        status: 301,
-      });
-    }
+  // redirect to dub.co for /metatags
+  if (fullPath.startsWith("/metatags")) {
+    return NextResponse.redirect("https://dub.co", {
+      status: 301,
+    });
   }
   // Note: we don't have to account for paths starting with `/api`
   // since they're automatically excluded via our middleware matcher

apps/web/lib/middleware/utils/bots-list.ts

@@ -1,7 +1,6 @@
 export const UA_BOTS = [
   "bot", // most bots
   "crawler", // most crawlers
-  "metatags", // Dub.co Metatags API (https://api.dub.co/metatags)
   "chatgpt", // ChatGPT
   "bluesky", // Bluesky crawler
   "facebookexternalhit", // Facebook crawler
@@ -18,6 +17,7 @@ export const UA_BOTS = [
   "MetaInspector", // metatags.io
   "Go-http-client", // Go-http-client/1.1 is a bot: https://user-agents.net/string/go-http-client-1-1
   "iframely", // https://iframely.com/docs/about (used by Notion, Linear)
+  "H1cbA69", // internal links/metatags API
 
   // new
   "ia_archiver",

apps/web/lib/openapi/index.ts

@@ -13,7 +13,6 @@ import { embedTokensPaths } from "./embed-tokens";
 import { eventsPath } from "./events";
 import { foldersPaths } from "./folders";
 import { linksPaths } from "./links";
-import { metatagsPath } from "./metatags";
 import { partnersPaths } from "./partners";
 import { qrCodePaths } from "./qr";
 import { tagsPaths } from "./tags";
@@ -56,7 +55,6 @@ export const document = createDocument({
     ...workspacesPaths,
     ...embedTokensPaths,
     ...qrCodePaths,
-    ...metatagsPath,
   },
   components: {
     schemas: {

apps/web/lib/types.ts

@@ -1,5 +1,4 @@
 import z from "@/lib/zod";
-import { metaTagsSchema } from "@/lib/zod/schemas/metatags";
 import {
   PartnerEarningsSchema,
   PartnerProfileCustomerSchema,
@@ -278,8 +277,6 @@ export const tagColors = [
 
 export type DashboardProps = z.infer<typeof dashboardSchema>;
 
-export type MetaTag = z.infer<typeof metaTagsSchema>;
-
 export type TokenProps = z.infer<typeof tokenSchema>;
 
 export type OAuthAppProps = z.infer<typeof oAuthAppSchema>;

apps/web/lib/upstash/record-metatags.ts

@@ -2,7 +2,7 @@ import { getDomainWithoutWWW } from "@dub/utils";
 import { redis } from "./redis";
 
 /**
- * Recording metatags that were generated via `/api/metatags`
+ * Recording metatags that were generated via "/api/links/metatags"
  * If there's an error, it will be logged to a separate redis list for debugging
  **/
 export async function recordMetatags(url: string, error: boolean) {

apps/web/lib/zod/schemas/links.ts

@@ -538,19 +538,19 @@ export const LinkSchema = z
       .string()
       .nullable()
       .describe(
-        "The title of the short link generated via `api.dub.co/metatags`. Will be used for Custom Social Media Cards if `proxy` is true.",
+        "The title of the short link. Will be used for Custom Social Media Cards if `proxy` is true.",
       ),
     description: z
       .string()
       .nullable()
       .describe(
-        "The description of the short link generated via `api.dub.co/metatags`. Will be used for Custom Social Media Cards if `proxy` is true.",
+        "The description of the short link. Will be used for Custom Social Media Cards if `proxy` is true.",
       ),
     image: z
       .string()
       .nullable()
       .describe(
-        "The image of the short link generated via `api.dub.co/metatags`. Will be used for Custom Social Media Cards if `proxy` is true.",
+        "The image of the short link. Will be used for Custom Social Media Cards if `proxy` is true.",
       ),
     video: z
       .string()

apps/web/lib/zod/schemas/sales.ts

@@ -26,7 +26,7 @@ export const trackSaleRequestSchema = z.object({
     .optional()
     .default("Purchase")
     .describe("The name of the sale event.")
-    .openapi({ examples: ["Purchase", "Upgrade", "Payment"] }),
+    .openapi({ example: "Invoice paid" }),
   invoiceId: z
     .string()
     .nullish()