add backgroundFetchSize option

Author: isaacs

CHANGELOG.md

@@ -1,5 +1,11 @@
 # cringe lorg
 
+## 11.5
+
+- Add `backgroundFetchSize` option, defaulting to 1, to set an
+  effective size for provisional background fetch objects while
+  in flight, if they do not shadow an existing stale entry.
+
 ## 11.4
 
 - Add `cache` property to status objects, in order to

src/index.ts

@@ -929,6 +929,21 @@ export namespace LRUCache {
      */
     maxSize?: Size
 
+    /**
+     * The effective size for background fetch promises.
+     *
+     * This has no effect unless `maxSize` and `sizeCalculation` are used,
+     * and a {@link LRUCache.OptionsBase.fetchMethod} is provided to
+     * support {@link LRUCache#fetch}.
+     *
+     * If a stale value is present in the cache, then the effective size of
+     * the background fetch is the size of the stale item it will eventually
+     * replace. If not, then this value is used as its effective size.
+     *
+     * @default 1
+     */
+    backgroundFetchSize?: number
+
     /**
      * The maximum allowed size for any single item in the cache.
      *
@@ -1274,6 +1289,9 @@ export class LRUCache<K extends {}, V extends {}, FC = unknown> {
    */
   ignoreFetchAbort: boolean
 
+  /** {@link LRUCache.OptionsBase.backgroundFetchSize} */
+  backgroundFetchSize: number
+
   // computed properties
   #size: LRUCache.Count
   #calculatedSize: LRUCache.Size
@@ -1428,9 +1446,12 @@ export class LRUCache<K extends {}, V extends {}, FC = unknown> {
       allowStaleOnFetchRejection,
       allowStaleOnFetchAbort,
       ignoreFetchAbort,
+      backgroundFetchSize = 1,
       perf,
     } = options
 
+    this.backgroundFetchSize = backgroundFetchSize
+
     if (perf !== undefined) {
       if (typeof perf?.now !== 'function') {
         throw new TypeError(
@@ -1707,12 +1728,15 @@ export class LRUCache<K extends {}, V extends {}, FC = unknown> {
       sizes[index] = 0
     }
     this.#requireSize = (k, v, size, sizeCalculation) => {
-      // provisionally accept background fetches.
-      // actual value size will be checked when they return.
-      if (this.#isBackgroundFetch(v)) {
-        return 0
-      }
       if (!isPosInt(size)) {
+        // provisionally accept background fetches.
+        // actual value size will be checked when they return.
+        if (this.#isBackgroundFetch(v)) {
+          // NB: this cannot occur if v.__staleWhileFetching is set,
+          // because in that case, it would take on the size of the
+          // existing entry that it temporarily replaces.
+          return this.backgroundFetchSize
+        }
         if (sizeCalculation) {
           if (typeof sizeCalculation !== 'function') {
             throw new TypeError('sizeCalculation must be a function')
@@ -2161,14 +2185,14 @@ export class LRUCache<K extends {}, V extends {}, FC = unknown> {
       status,
     } = setOptions
 
+    const isBF = this.#isBackgroundFetch(v)
     if (v === undefined) {
       if (status) status.set = 'deleted'
       this.delete(k)
       return this
     }
     let { noUpdateTTL = this.noUpdateTTL } = setOptions
 
-    const isBF = this.#isBackgroundFetch(v)
     if (status && !isBF) status.value = v
 
     const size = this.#requireSize(
@@ -2598,6 +2622,10 @@ export class LRUCache<K extends {}, V extends {}, FC = unknown> {
       this.#set(k, bf, { ...fetchOpts.options, status: undefined })
       index = this.#keyMap.get(k)
     } else {
+      // do not call #set, because we do not want to adjust its place
+      // in the lru queue, as it has not yet been "used". Also, we don't
+      // need to worry about evicting for size, because a background fetch
+      // over a stale value is treated as the same size as its stale value.
       this.#valList[index] = bf
     }
     return bf

tap-snapshots/test/tracing.ts.test.cjs

@@ -15369,7 +15369,7 @@ Map {
     "op": "fetch",
     "result": 4,
     "set": "add",
-    "totalCalculatedSize": 4,
+    "totalCalculatedSize": 6,
     "trace": true,
     "value": 4,
   } => Array [
@@ -15398,7 +15398,7 @@ Map {
     "op": "fetch",
     "result": 5,
     "set": "add",
-    "totalCalculatedSize": 9,
+    "totalCalculatedSize": 10,
     "trace": true,
     "value": 5,
   } => Array [

test/background-fetch-size.ts

@@ -0,0 +1,44 @@
+import t from 'tap'
+import { LRUCache } from '../dist/esm/node/index.js'
+
+const clock = t.clock
+clock.advance(1)
+clock.enter()
+
+t.test('background fetch size tests', async t => {
+  const res: Record<number, (n: number) => void> = {}
+  const c = new LRUCache<number, number>({
+    maxSize: 10,
+    sizeCalculation: () => 5,
+    allowStale: true,
+    ttl: 10,
+    // never returns on purpose
+    fetchMethod: k =>
+      new Promise<number>(r => {
+        res[k] = r
+      }),
+  })
+  t.equal(c.calculatedSize, 0)
+  const p1 = c.fetch(1).catch(er => er)
+  t.equal(c.calculatedSize, 1)
+  c.set(1, 1)
+  t.match(await p1, new Error('replaced'))
+  t.equal(c.calculatedSize, 5)
+  clock.advance(100)
+  t.equal(c.getRemainingTTL(1), -90)
+  // verify correct behavior of a fetch that shadows a stale value
+  const p = c.fetch(1)
+  t.equal(c.calculatedSize, 5)
+  const p2 = c.fetch(2)
+  t.equal(c.calculatedSize, 6)
+  const p3 = c.fetch(3)
+  t.equal(c.calculatedSize, 7)
+  res[1]?.(1)
+  await p
+  // no change, that one had a stale value
+  t.equal(c.calculatedSize, 7)
+  res[2]?.(2)
+  await p2
+  t.equal(c.calculatedSize, 10)
+  await t.rejects(p3, new Error('evicted'))
+})

test/onInsert.ts

@@ -84,9 +84,12 @@ t.test('no onInsert for background fetch promises', async t => {
     ttl: 10,
   })
   c.set('y', 'Y')
-  t.strictSame(inserted, [['Y', 'y' ,'add']])
+  t.strictSame(inserted, [['Y', 'y', 'add']])
   c.set('z', 'Z')
-  t.strictSame(inserted, [['Y', 'y' ,'add'], ['Z', 'z', 'add']])
+  t.strictSame(inserted, [
+    ['Y', 'y', 'add'],
+    ['Z', 'z', 'add'],
+  ])
   const x = await c.fetch('x')
   const y = await c.fetch('y')
   await new Promise(res => setTimeout(res, 50))