Memoize secret scanning YAML reads to reduce disk IO (#60389)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: heiskr

src/article-api/transformers/secret-scanning-transformer.ts

@@ -1,12 +1,12 @@
-import type { Context, Page, SecretScanningData } from '@/types'
+import type { Context, Page } from '@/types'
 import type { PageTransformer } from './types'
-import fs from 'fs'
-import yaml from 'js-yaml'
+import { load } from 'js-yaml'
 import path from 'path'
 import { getVersionInfo } from '@/app/lib/constants'
 import { liquid, renderContent } from '@/content-render/index'
 import { allVersions } from '@/versions/lib/all-versions'
 import { loadTemplate } from '@/article-api/lib/load-template'
+import { getSecretScanningData } from '@/secret-scanning/lib/get-secret-scanning-data'
 
 /**
  * Transformer for Secret Scanning pages.
@@ -35,15 +35,15 @@ export class SecretScanningTransformer implements PageTransformer {
       const secretScanningDir = path.join(process.cwd(), 'src/secret-scanning/data/pattern-docs')
       const filepath = path.join(secretScanningDir, versionPath, 'public-docs.yml')
 
-      if (fs.existsSync(filepath)) {
-        const data = yaml.load(fs.readFileSync(filepath, 'utf-8')) as SecretScanningData[]
+      try {
+        const data = await getSecretScanningData(filepath)
 
         // Process Liquid in values
         for (const entry of data) {
           // Only process Liquid for the hasValidityCheck field, as in the middleware
           if (typeof entry.hasValidityCheck === 'string' && entry.hasValidityCheck.includes('{%')) {
             // Render Liquid and parse as YAML to get correct boolean type
-            entry.hasValidityCheck = yaml.load(
+            entry.hasValidityCheck = load(
               await liquid.parseAndRender(entry.hasValidityCheck, context),
             ) as boolean
           }
@@ -54,9 +54,12 @@ export class SecretScanningTransformer implements PageTransformer {
         }
 
         context.secretScanningData = data
-      } else {
-        // If the file does not exist, set to empty array to ensure predictable behavior
-        context.secretScanningData = []
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+          context.secretScanningData = []
+        } else {
+          throw error
+        }
       }
     }
 

src/secret-scanning/lib/get-secret-scanning-data.ts

@@ -0,0 +1,16 @@
+import fs from 'fs/promises'
+import path from 'path'
+import { load } from 'js-yaml'
+import type { SecretScanningData } from '@/types'
+
+const cache = new Map<string, SecretScanningData[]>()
+
+export async function getSecretScanningData(filepath: string): Promise<SecretScanningData[]> {
+  const key = path.resolve(filepath)
+  if (cache.has(key)) return structuredClone(cache.get(key)!)
+
+  const raw = await fs.readFile(key, 'utf-8')
+  const data = (load(raw) as SecretScanningData[]) ?? []
+  cache.set(key, data)
+  return structuredClone(data)
+}

src/secret-scanning/middleware/secret-scanning.ts

@@ -4,9 +4,10 @@ import yaml from 'js-yaml'
 import type { NextFunction, Response } from 'express'
 
 import { liquid } from '@/content-render/index'
-import { ExtendedRequest, SecretScanningData } from '@/types'
+import { ExtendedRequest } from '@/types'
 import { allVersions } from '@/versions/lib/all-versions'
 import { getVersionInfo } from '@/app/lib/constants'
+import { getSecretScanningData } from '@/secret-scanning/lib/get-secret-scanning-data'
 
 const secretScanningDir = 'src/secret-scanning/data/pattern-docs'
 
@@ -41,9 +42,7 @@ export default async function secretScanning(
       : 'fpt'
   const filepath = `${secretScanningDir}/${versionPath}/public-docs.yml`
 
-  req.context.secretScanningData = yaml.load(
-    fs.readFileSync(filepath, 'utf-8'),
-  ) as SecretScanningData[]
+  req.context.secretScanningData = await getSecretScanningData(filepath)
 
   // Some entries might use Liquid syntax, so we need
   // to execute that Liquid to get the actual value.