perf(video): share iframe fetch cache across rules (#817)

Author: Kikobeats

packages/metascraper-video/src/index.js

@@ -70,18 +70,45 @@ const imageRules = [toUrl($ => $('video').attr('poster'))]
 const _getIframe = (url, $, { src }) =>
   loadIframe(url, $.load(`<iframe src="${src}"></iframe>`))
 
+const createGetIframeCached = getIframe => {
+  const cacheByHtmlDom = new WeakMap()
+
+  return async (url, $, src) => {
+    let cacheBySrc = cacheByHtmlDom.get($)
+    if (!cacheBySrc) {
+      cacheBySrc = new Map()
+      cacheByHtmlDom.set($, cacheBySrc)
+    }
+
+    const cachedHtmlDom = cacheBySrc.get(src)
+    if (cachedHtmlDom) return cachedHtmlDom
+
+    const pendingHtmlDom = getIframe(url, $, { src }).catch(error => {
+      cacheBySrc.delete(src)
+      throw error
+    })
+
+    cacheBySrc.set(src, pendingHtmlDom)
+    return pendingHtmlDom
+  }
+}
+
 const withIframe = (rules, getIframe) =>
   rules.concat(
     async ({ htmlDom: $, url }) => {
       const srcs = $('iframe[src^="http"], iframe[src^="/"]')
         .map((_, element) => $(element).attr('src'))
         .get()
       if (srcs.length === 0) return
+      const seenSrcs = new Set()
       for (const src of srcs) {
         try {
           const normalizedSrc = normalizeUrl(url, src)
           if (!normalizedSrc) continue
-          const htmlDom = await getIframe(url, $, { src: normalizedSrc })
+          if (seenSrcs.has(normalizedSrc)) continue
+          seenSrcs.add(normalizedSrc)
+
+          const htmlDom = await getIframe(url, $, normalizedSrc)
           const result = await findRule(rules, { htmlDom, url })
           if (has(result)) return result
         } catch (_) {}
@@ -91,17 +118,18 @@ const withIframe = (rules, getIframe) =>
       const src = $('meta[name="twitter:player"]').attr('content')
       return src
         ? findRule(rules, {
-          htmlDom: await getIframe(url, $, { src }),
+          htmlDom: await getIframe(url, $, src),
           url
         })
         : undefined
     }
   )
 
 module.exports = ({ getIframe = _getIframe } = {}) => {
+  const getIframeCached = createGetIframeCached(getIframe)
   const rules = {
-    image: withIframe(imageRules, getIframe),
-    video: withIframe(videoRules, getIframe)
+    image: withIframe(imageRules, getIframeCached),
+    video: withIframe(videoRules, getIframeCached)
   }
 
   rules.pkgName = 'metascraper-video'

packages/metascraper-video/test/iframe.js

@@ -68,3 +68,28 @@ test('stop iframe probing after first video match', async t => {
   t.is(metadata.video, 'https://cdn.microlink.io/file-examples/sample.mp4')
   t.deepEqual(calls, ['https://example.com/ok'])
 })
+
+test('reuse iframe fetch across image and video extraction', async t => {
+  let calls = 0
+  const metascraper = createMetascraper({
+    getIframe: async (url, $, { src }) => {
+      calls += 1
+      t.is(src, 'https://example.com/ok')
+      return cheerio.load(`
+        <video
+          poster="https://cdn.microlink.io/file-examples/sample.png"
+          src="https://cdn.microlink.io/file-examples/sample.mp4"
+        ></video>
+      `)
+    }
+  })
+
+  const metadata = await metascraper({
+    url: 'https://example.com',
+    html: '<iframe src="/ok"></iframe>'
+  })
+
+  t.is(metadata.image, 'https://cdn.microlink.io/file-examples/sample.png')
+  t.is(metadata.video, 'https://cdn.microlink.io/file-examples/sample.mp4')
+  t.is(calls, 1)
+})