refactor: seo indexing flag

Author: ben-fornefeld

.env.example

@@ -39,10 +39,6 @@ NEXT_PUBLIC_SUPABASE_ANON_KEY=your_supabase_anon_key
 ### ZeroBounce API key for email validation
 # ZEROBOUNCE_API_KEY=
 
-### Set to 1 to enable no-indexing
-### This will take precautions to prevent search engines from indexing the site
-# NO_INDEX=0
-
 ### =================================
 ### OPTIONAL CLIENT ENVIRONMENT VARIABLES
 ### =================================

src/app/(rewrites)/[[...slug]]/route.ts

@@ -6,7 +6,7 @@ import { ERROR_CODES } from '@/configs/logs'
 import { NextRequest } from 'next/server'
 import sitemap from '@/app/sitemap'
 import { BASE_URL } from '@/configs/urls'
-import { NO_INDEX } from '@/lib/utils/flags'
+import { ALLOW_SEO_INDEXING } from '@/configs/flags'
 import { logError } from '@/lib/clients/logger'
 import { ROUTE_REWRITE_CONFIG } from '@/configs/rewrites'
 
@@ -71,7 +71,7 @@ export async function GET(request: NextRequest): Promise<Response> {
         html = rewriteContentPagesHtml(html, {
           seo: {
             pathname: url.pathname,
-            isNoIndex: NO_INDEX,
+            allowIndexing: ALLOW_SEO_INDEXING,
           },
           hrefPrefixes: [rewrittenPrefix, 'https://e2b.dev'],
         })

src/app/layout.tsx

@@ -12,7 +12,7 @@ import { Toaster } from '@/ui/primitives/toaster'
 import Head from 'next/head'
 import { GTMHead } from '@/features/google-tag-manager'
 import { Analytics } from '@vercel/analytics/next'
-import { NO_INDEX } from '@/lib/utils/flags'
+import { ALLOW_SEO_INDEXING } from '@/configs/flags'
 
 export const metadata: Metadata = {
   metadataBase: new URL(BASE_URL),
@@ -29,7 +29,7 @@ export const metadata: Metadata = {
     title: METADATA.title,
     description: METADATA.description,
   },
-  robots: NO_INDEX ? 'noindex, nofollow' : 'index, follow',
+  robots: ALLOW_SEO_INDEXING ? 'index, follow' : 'noindex, nofollow',
 }
 
 export default function RootLayout({

src/app/robots.ts

@@ -1,8 +1,8 @@
 import type { MetadataRoute } from 'next'
-import { NO_INDEX } from '@/lib/utils/flags'
+import { ALLOW_SEO_INDEXING } from '@/configs/flags'
 
 export default function robots(): MetadataRoute.Robots {
-  if (NO_INDEX) {
+  if (!ALLOW_SEO_INDEXING) {
     // We serve an empty robots.txt for a 200 status code
     return {
       rules: {},

src/app/sitemap.ts

@@ -15,7 +15,7 @@ import {
   ROUTE_REWRITE_CONFIG,
 } from '@/configs/rewrites'
 import { DomainConfig } from '@/types/rewrites.types'
-import { NO_INDEX } from '@/lib/utils/flags'
+import { ALLOW_SEO_INDEXING } from '@/configs/flags'
 
 // Cache the sitemap for 15 minutes (in seconds)
 const SITEMAP_CACHE_TIME = 15 * 60
@@ -229,8 +229,8 @@ async function getSitemap(site: Site): Promise<MetadataRoute.Sitemap> {
  * @returns Complete sitemap for the E2B website
  */
 export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
-  // Return empty sitemap if NO_INDEX is set
-  if (NO_INDEX) {
+  // Return empty sitemap if SEO indexing is not allowed
+  if (!ALLOW_SEO_INDEXING) {
     return []
   }
 

src/configs/flags.ts

@@ -1 +1,5 @@
+export const ALLOW_SEO_INDEXING = process.env.ALLOW_SEO_INDEXING === '1'
+export const VERBOSE =
+  process.env.NODE_ENV === 'development' ||
+  process.env.NEXT_PUBLIC_VERBOSE === '1'
 export const INCLUDE_BILLING = process.env.NEXT_PUBLIC_INCLUDE_BILLING === '1'

src/lib/clients/action.ts

@@ -4,7 +4,7 @@ import { z } from 'zod'
 import { UnknownError } from '@/types/errors'
 import { logDebug, logError, logSuccess } from './logger'
 import { ActionError } from '../utils/action'
-import { VERBOSE } from '../utils/flags'
+import { VERBOSE } from '../../configs/flags'
 
 // keys that should not be logged for security/privacy reasons
 const BLACKLISTED_INPUT_KEYS = [

src/lib/env.ts

@@ -3,7 +3,6 @@ import { z } from 'zod'
 export const serverSchema = z.object({
   SUPABASE_SERVICE_ROLE_KEY: z.string().min(1),
   INFRA_API_URL: z.string().url(),
-  NO_INDEX: z.string().optional(),
   KV_REST_API_TOKEN: z.string().min(1),
   KV_REST_API_URL: z.string().url(),
 

src/lib/utils/flags.ts

@@ -46,7 +46,7 @@ function getRewriteForPath(
 
 interface SeoTagOptions {
   pathname: string
-  isNoIndex: boolean
+  allowIndexing: boolean
 }
 
 /**
@@ -58,12 +58,12 @@ interface SeoTagOptions {
  * @param options Configuration options including the pathname and NO_INDEX flag.
  */
 function rewriteSeoTags($: cheerio.CheerioAPI, options: SeoTagOptions): void {
-  const { pathname, isNoIndex } = options
+  const { pathname, allowIndexing } = options
 
   $('meta[name="robots"]').remove()
   $('link[rel="canonical"]').remove()
 
-  const robotsContent = isNoIndex ? 'noindex,nofollow' : 'index,follow'
+  const robotsContent = allowIndexing ? 'index,follow' : 'noindex,nofollow'
 
   const formattedPathname = pathname.startsWith('/') ? pathname : `/${pathname}`
   const canonicalUrl = `https://e2b.dev${formattedPathname}`