Implement function loaders (#290)

Author: kibertoad

README.md

@@ -161,6 +161,45 @@ const loader = new Loader<string>({
 const classifier = await loader.get('1')
 ```
 
+### Simplified loader syntax
+
+It is also possible to inline datasource definition:
+
+```ts
+const loader = new Loader<string>({
+    // this cache will be checked first
+    inMemoryCache: {
+        cacheType: 'lru-object', // you can choose between lru and fifo caches, fifo being 10% slightly faster
+        ttlInMsecs: 1000 * 60,
+        maxItems: 100,
+    },
+
+    // this cache will be checked if in-memory one returns undefined
+    asyncCache: new RedisCache(ioRedis, {
+        json: true, // this instructs loader to serialize passed objects as string and deserialize them back to objects
+        ttlInMsecs: 1000 * 60 * 10,
+    }),
+    
+    // data source will be generated from one or both provided data loading functions
+    dataSourceGetOneFn: async (key: string) => {
+    const results = await this.db('classifiers')
+        .select('*')
+        .where({
+            id: parseInt(key),
+        })
+    return results[0]
+    },
+    dataSourceGetManyFn: (keys: string[]) => {
+    return this.db('classifiers')
+        .select('*')
+        .whereIn('id', keys.map(parseInt))
+    }
+})
+
+// If cache is empty, but there is data in the DB, after this operation is completed, both caches will be populated
+const classifier = await loader.get('1')
+```
+
 ## Loader API
 
 Loader has the following config parameters:

lib/GeneratedDataSource.ts

@@ -0,0 +1,35 @@
+import type { DataSource } from './types/DataSources'
+
+export type GeneratedDataSourceParams<LoadedValue, LoaderParams = undefined> = {
+  name?: string
+  dataSourceGetOneFn?: (key: string, loadParams?: LoaderParams) => Promise<LoadedValue | undefined | null>
+  dataSourceGetManyFn?: (keys: string[], loadParams?: LoaderParams) => Promise<LoadedValue[]>
+}
+
+export class GeneratedDataSource<LoadedValue, LoadParams = undefined> implements DataSource<LoadedValue, LoadParams> {
+  private readonly getOneFn: (key: string, loadParams?: LoadParams) => Promise<LoadedValue | undefined | null>
+  private readonly getManyFn: (keys: string[], loadParams?: LoadParams) => Promise<LoadedValue[]>
+  public readonly name: string
+  constructor(params: GeneratedDataSourceParams<LoadedValue, LoadParams>) {
+    this.name = params.name ?? 'Generated loader'
+    this.getOneFn =
+      params.dataSourceGetOneFn ??
+      function () {
+        throw new Error('Retrieval of a single entity is not implemented')
+      }
+
+    this.getManyFn =
+      params.dataSourceGetManyFn ??
+      function () {
+        throw new Error('Retrieval of multiple entities is not implemented')
+      }
+  }
+
+  get(key: string, loadParams: LoadParams | undefined): Promise<LoadedValue | undefined | null> {
+    return this.getOneFn(key, loadParams)
+  }
+
+  getMany(keys: string[], loadParams: LoadParams | undefined): Promise<LoadedValue[]> {
+    return this.getManyFn(keys, loadParams)
+  }
+}

lib/Loader.ts

@@ -6,6 +6,7 @@ import type { InMemoryGroupCacheConfiguration } from './memory/InMemoryGroupCach
 import type { SynchronousCache, SynchronousGroupCache, GetManyResult } from './types/SyncDataSources'
 import type { NotificationPublisher } from './notifications/NotificationPublisher'
 import type { GroupNotificationPublisher } from './notifications/GroupNotificationPublisher'
+import { GeneratedDataSource } from './GeneratedDataSource'
 
 export type LoaderConfig<
   LoadedValue,
@@ -23,6 +24,9 @@ export type LoaderConfig<
     | GroupNotificationPublisher<LoadedValue> = NotificationPublisher<LoadedValue>,
 > = {
   dataSources?: readonly DataSourceType[]
+  dataSourceGetOneFn?: (key: string, loadParams?: LoaderParams) => Promise<LoadedValue | undefined | null>
+  dataSourceGetManyFn?: (keys: string[], loadParams?: LoaderParams) => Promise<LoadedValue[]>
+  dataSourceName?: string
   throwIfLoadError?: boolean
   throwIfUnresolved?: boolean
 } & CommonCacheConfig<LoadedValue, CacheType, InMemoryCacheConfigType, InMemoryCacheType, NotificationPublisherType>
@@ -35,7 +39,30 @@ export class Loader<LoadedValue, LoaderParams = undefined> extends AbstractFlatC
 
   constructor(config: LoaderConfig<LoadedValue, Cache<LoadedValue>, LoaderParams>) {
     super(config)
-    this.dataSources = config.dataSources ?? []
+
+    // generated datasource
+    if (config.dataSourceGetManyFn || config.dataSourceGetOneFn) {
+      if (config.dataSources) {
+        throw new Error('Cannot set both "dataSources" and "dataSourceGetManyFn"/"dataSourceGetOneFn" parameters.')
+      }
+
+      this.dataSources = [
+        new GeneratedDataSource({
+          dataSourceGetOneFn: config.dataSourceGetOneFn,
+          dataSourceGetManyFn: config.dataSourceGetManyFn,
+          name: config.dataSourceName,
+        }),
+      ]
+    }
+    // defined datasource
+    else if (config.dataSources) {
+      this.dataSources = config.dataSources
+    }
+    // no datasource
+    else {
+      this.dataSources = []
+    }
+
     this.throwIfLoadError = config.throwIfLoadError ?? true
     this.throwIfUnresolved = config.throwIfUnresolved ?? false
     this.isKeyRefreshing = new Set()

package.json

@@ -53,12 +53,12 @@
     "toad-cache": "^3.3.1"
   },
   "devDependencies": {
-    "@types/node": "^20.10.1",
+    "@types/node": "^20.10.2",
     "@typescript-eslint/eslint-plugin": "^6.13.1",
     "@typescript-eslint/parser": "^6.13.1",
     "@vitest/coverage-v8": "0.34.6",
     "del-cli": "^5.1.0",
-    "eslint": "^8.54.0",
+    "eslint": "^8.55.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-import": "^2.29.0",
     "eslint-plugin-prettier": "^5.0.1",

test/Loader-main.spec.ts

@@ -360,6 +360,19 @@ describe('Loader Main', () => {
     })
   })
 
+  describe('constructor', () => {
+    it('throws an error if both datasource and datasource fns are provided', () => {
+      expect(() => {
+        new Loader({
+          dataSources: [],
+          dataSourceGetOneFn: () => {
+            return Promise.resolve('x')
+          },
+        })
+      }).toThrow(/Cannot set both/)
+    })
+  })
+
   describe('get', () => {
     it('returns undefined when fails to resolve value', async () => {
       const operation = new Loader({})
@@ -461,6 +474,36 @@ describe('Loader Main', () => {
       expect(result).toBe('value')
     })
 
+    it('returns value when resolved via generated loader', async () => {
+      const operation = new Loader<string>({
+        inMemoryCache: IN_MEMORY_CACHE_CONFIG,
+        dataSourceGetOneFn: (key) => {
+          if (key === 'key') {
+            return Promise.resolve('value')
+          }
+          throw new Error('Not found')
+        },
+      })
+
+      const result = await operation.get('key')
+
+      expect(result).toBe('value')
+    })
+
+    it('throws an error if requested generated loader is not set', async () => {
+      const operation = new Loader<string>({
+        inMemoryCache: IN_MEMORY_CACHE_CONFIG,
+        dataSourceGetManyFn: (keys) => {
+          if (keys[0] === 'key') {
+            return Promise.resolve(['value'])
+          }
+          throw new Error('Not found')
+        },
+      })
+
+      await expect(operation.get('key')).rejects.toThrow(/Retrieval of a single entity is not/)
+    })
+
     it('returns value when resolved via multiple loaders', async () => {
       const asyncCache = new DummyCache(undefined)
 
@@ -664,6 +707,37 @@ describe('Loader Main', () => {
       expect(result).toEqual(['value'])
     })
 
+    it('returns value when resolved via generated loader', async () => {
+      const operation = new Loader<string>({
+        inMemoryCache: IN_MEMORY_CACHE_CONFIG,
+        dataSourceGetManyFn: (keys: string[]) => {
+          if (keys.includes('key')) {
+            return Promise.resolve(['value'])
+          }
+
+          throw new Error('Not found')
+        },
+      })
+
+      const result = await operation.getMany(['key'], idResolver)
+
+      expect(result).toEqual(['value'])
+    })
+
+    it('throws an error if requested generated loader is not set', async () => {
+      const operation = new Loader<string>({
+        inMemoryCache: IN_MEMORY_CACHE_CONFIG,
+        dataSourceGetOneFn: (key) => {
+          if (key === 'key') {
+            return Promise.resolve('value')
+          }
+          throw new Error('Not found')
+        },
+      })
+
+      await expect(operation.getMany(['key'], idResolver)).rejects.toThrow(/Retrieval of multiple entities/)
+    })
+
     it('returns value when resolved via multiple caches', async () => {
       const asyncCache = new DummyCache(undefined)