chore: tweaks

Author: tmm

scripts/deployWaf.ts

@@ -49,12 +49,14 @@ const botPaths = [
 ]
 
 const excludePrefixes = ['/assets/', '/.well-known/', '/sheep/']
-const excludePaths = ['/api/og.png', '/dark.svg', '/favicon.svg', '/light.svg']
+const excludePaths = ['/api/og.png', '/dark.svg', '/favicon.svg', '/light.svg', '/og.png']
+const excludeWildcards = ['/*.*/*']
 
 const expression = [
   `(${[
     ...excludePrefixes.map((p) => `not starts_with(http.request.uri.path, "${p}")`),
     ...excludePaths.map((p) => `http.request.uri.path ne "${p}"`),
+    ...excludeWildcards.map((p) => `not (http.request.uri.path wildcard "${p}")`),
   ].join(
     ' and ',
   )} and (${extensions.map((ext) => `ends_with(http.request.uri.path, "${ext}")`).join(' or ')}))`,

src/md/mod.ts

@@ -586,7 +586,7 @@ function parseHtmlTagAttributes(tag: string): Record<string, string> {
 }
 
 function normalizeMarkdown(content: string, origin?: string, profile?: DetectedProfile): string {
-  const normalized = profile?.normalize?.(content) ?? content
+  const normalized = normalizeCodeFencePadding(profile?.normalize?.(content) ?? content)
   return (
     normalized
       // Remove links with no text (e.g. `[](/)`)
@@ -632,3 +632,32 @@ function normalizeMarkdown(content: string, origin?: string, profile?: DetectedP
       .trim()
   )
 }
+
+function normalizeCodeFencePadding(content: string): string {
+  const lines = content.split('\n')
+  const normalized: string[] = []
+  let codeFence: { char: '`' | '~'; length: number; lines: string[] } | undefined
+
+  for (const line of lines) {
+    if (codeFence) {
+      const closeMatch = line.match(/^(`{3,}|~{3,})\s*$/)
+      if (closeMatch?.[1]?.startsWith(codeFence.char) && closeMatch[1].length >= codeFence.length) {
+        while (codeFence.lines[0]?.trim() === '') codeFence.lines.shift()
+        while (codeFence.lines.at(-1)?.trim() === '') codeFence.lines.pop()
+        normalized.push(...codeFence.lines, line)
+        codeFence = undefined
+        continue
+      }
+      codeFence.lines.push(line)
+      continue
+    }
+
+    const openMatch = line.match(/^(`{3,}|~{3,})/)
+    if (openMatch?.[1])
+      codeFence = { char: openMatch[1][0] as '`' | '~', length: openMatch[1].length, lines: [] }
+    normalized.push(line)
+  }
+
+  if (codeFence) normalized.push(...codeFence.lines)
+  return normalized.join('\n')
+}

src/md/rules/cloudflare.test.ts

@@ -43,6 +43,25 @@ description: Review available Cloudflare API token permissions.
   expect(result.content).toContain('| User Details Edit | Grants write access to user details. |')
 })
 
+test('cloudflare trims blank padding inside first-party markdown code fences', async () => {
+  const md = create({
+    fetch: async () =>
+      new Response('```\n\nlower(http.host) == "www.cloudflare.com"\n\n\n```\n', {
+        headers: { 'content-type': 'text/markdown; charset=utf-8' },
+        status: 200,
+      }),
+    rules: [cloudflare()],
+  })
+
+  const result = await md.fetch(
+    'https://developers.cloudflare.com/ruleset-engine/rules-language/functions/',
+  )
+  expect(result.ok).toBe(true)
+  if (!result.ok) return
+
+  expect(result.content).toBe('```\nlower(http.host) == "www.cloudflare.com"\n```')
+})
+
 test('cloudflare rewrites slashless docs pages to /index.md', async () => {
   let request: Request | undefined
 

src/og.tsx

@@ -31,8 +31,7 @@ async function getElement(env: Cloudflare.Env, db: Database, query: query) {
       return playgroundVariant()
     }
     case 'index': {
-      const tokensSaved = await getTokensSaved(env, db)
-      return indexVariant(tokensSaved)
+      return indexVariant()
     }
   }
 }
@@ -58,18 +57,15 @@ function docsVariant(title: string, description: string | undefined) {
   )
 }
 
-function indexVariant(tokensSaved: number) {
+function indexVariant() {
   return (
     <div tw="relative flex w-full h-full bg-black text-[#ededed] font-[Geist_Mono] px-[80px] py-[40px]">
-      <div tw="flex flex-1 flex-col items-center justify-center">
+      <div tw="flex flex-1 flex-col items-center justify-center pt-[56px]">
         <BrandLogo height={56} width={772} />
         <div tw="text-[#a1a1a1] text-[54px] mt-[44px] text-center">URL to markdown for agents</div>
-      </div>
-      <div tw="absolute right-[80px] bottom-[40px] left-[80px] flex items-end justify-between">
-        <div tw="text-[#a1a1a1] text-[36px] tracking-[0.08em] uppercase leading-none">
-          tokens saved
+        <div tw="bg-[#171717] text-[#ededed] text-[42px] mt-[44px] px-[28px] py-[16px]">
+          npm i -g curl.md
         </div>
-        <div tw="text-[#ededed] text-[36px] leading-none">{formatNumber(tokensSaved)}</div>
       </div>
     </div>
   )

src/routes/-home.tsx

@@ -56,7 +56,7 @@ export function Home(props: { login?: string | null | undefined }) {
             URL to markdown
             <br className="hidden md:block" /> for agents
           </h1>
-          <p className="text-gray8 mt-4 text-lg leading-relaxed">
+          <p className="text-gray8 mt-6 text-lg leading-relaxed">
             Turn websites into optimized, low token output to supercharge your context. Works with
             every agent.
           </p>

test/vitest.config.ts

@@ -13,6 +13,18 @@ import { Env } from './env.ts'
 
 const root = path.resolve(import.meta.dirname, '..')
 
+// TODO: remove once porsager/postgres fixes CF polyfill socket teardown.
+// https://github.com/cloudflare/workers-sdk/issues/11532
+process.on('uncaughtException', (error) => {
+  if (
+    error instanceof Error &&
+    'code' in error &&
+    (error.code === 'ECONNRESET' || error.code === 'EPIPE')
+  )
+    return
+  throw error
+})
+
 const reporters = [isAgent ? 'agent' : 'default']
 if (process.env.GITHUB_ACTIONS === 'true') reporters.push('github-actions')
 

test/workers.setup.ts

@@ -9,6 +9,16 @@ process.on('unhandledRejection', (reason) => {
   throw reason
 })
 
+process.on('uncaughtException', (error) => {
+  if (
+    error instanceof Error &&
+    'code' in error &&
+    (error.code === 'ECONNRESET' || error.code === 'EPIPE')
+  )
+    return
+  throw error
+})
+
 beforeAll(() => {
   server.listen({ onUnhandledRequest: 'error' })
   return () => {