fix: refresh background

Signed-off-by: Umberto Sgueglia <usgueglia@contractor.linuxfoundation.org>

Author: ulemons

services/libs/data-access-layer/src/members/base.ts

@@ -225,6 +225,10 @@ export async function queryMembersAdvanced(
   const cachedCount = countOnly ? await cache.getCount(countCacheKey) : null
 
   if (cachedResult) {
+    log.info(
+      { cacheKey, segmentId, search: normalizedSearch, limit, offset, orderBy },
+      'Members advanced query cache hit — returning cached result, scheduling background refresh',
+    )
     refreshCacheInBackground(bgQx, redis, cacheKey, {
       filter,
       search: normalizedSearch,
@@ -238,12 +242,14 @@ export async function queryMembersAdvanced(
       includeAllAttributes,
       attributeSettings,
     })
-
-    log.info(`Members advanced query cache hit: ${cacheKey}`)
     return cachedResult
   }
 
   if (countOnly && cachedCount !== null) {
+    log.info(
+      { countCacheKey, segmentId, search: normalizedSearch },
+      'Members advanced count cache hit — returning cached count, scheduling background refresh',
+    )
     refreshCountCacheInBackground(bgQx, redis, countCacheKey, {
       filter,
       search: normalizedSearch,
@@ -252,8 +258,6 @@ export async function queryMembersAdvanced(
       includeAllAttributes,
       attributeSettings,
     })
-
-    log.debug(`Members advanced count query cache hit: ${countCacheKey}`)
     return {
       rows: [],
       count: cachedCount,
@@ -262,19 +266,65 @@ export async function queryMembersAdvanced(
     }
   }
 
-  return await executeQuery(qx, redis, cacheKey, {
-    filter,
-    search: normalizedSearch,
-    limit,
-    offset,
-    orderBy,
-    segmentId,
-    countOnly,
-    fields,
-    include,
-    includeAllAttributes,
-    attributeSettings,
-  })
+  log.info(
+    {
+      cacheKey,
+      countCacheKey,
+      segmentId,
+      search: normalizedSearch,
+      limit,
+      offset,
+      orderBy,
+      countOnly,
+    },
+    'Members advanced query cache miss — executing query synchronously',
+  )
+
+  try {
+    return await executeQuery(qx, redis, cacheKey, {
+      filter,
+      search: normalizedSearch,
+      limit,
+      offset,
+      orderBy,
+      segmentId,
+      countOnly,
+      fields,
+      include,
+      includeAllAttributes,
+      attributeSettings,
+    })
+  } catch (error) {
+    log.warn(
+      { cacheKey, countCacheKey, segmentId, search: normalizedSearch, countOnly, err: error },
+      'Members advanced query failed on cache miss — scheduling background refresh for next retry',
+    )
+    if (countOnly) {
+      refreshCountCacheInBackground(bgQx, redis, countCacheKey, {
+        filter,
+        search: normalizedSearch,
+        segmentId,
+        include,
+        includeAllAttributes,
+        attributeSettings,
+      })
+    } else {
+      refreshCacheInBackground(bgQx, redis, cacheKey, {
+        filter,
+        search: normalizedSearch,
+        limit,
+        offset,
+        orderBy,
+        segmentId,
+        countOnly: false,
+        fields,
+        include,
+        includeAllAttributes,
+        attributeSettings,
+      })
+    }
+    throw error
+  }
 }
 
 export async function executeQuery(
@@ -567,29 +617,36 @@ async function refreshCacheInBackground(
   redis: RedisClient,
   cacheKey: string,
   params: IQueryMembersAdvancedParams,
+  countOnly = false,
 ): Promise<void> {
+  const label = countOnly ? 'count cache' : 'query cache'
+  const cache = new MemberQueryCache(redis)
+  const acquired = await cache.tryAcquireRefreshLock(cacheKey)
+  if (!acquired) {
+    log.debug(
+      { cacheKey },
+      `Members advanced ${label} refresh already in progress — skipping duplicate`,
+    )
+    return
+  }
   try {
-    log.info(`Refreshing members advanced query cache in background: ${cacheKey}`)
-    await executeQuery(qx, redis, cacheKey, params)
-    log.info(`Members advanced query cache refreshed in background: ${cacheKey}`)
+    log.info({ cacheKey }, `Members advanced ${label} background refresh started`)
+    await executeQuery(qx, redis, cacheKey, countOnly ? { ...params, countOnly: true } : params)
+    log.info({ cacheKey }, `Members advanced ${label} background refresh completed`)
   } catch (error) {
-    log.warn('Background cache refresh failed:', error)
+    log.warn({ cacheKey, err: error }, `Members advanced ${label} background refresh failed`)
+  } finally {
+    await cache.releaseRefreshLock(cacheKey)
   }
 }
 
-async function refreshCountCacheInBackground(
+function refreshCountCacheInBackground(
   qx: QueryExecutor,
   redis: RedisClient,
   cacheKey: string,
   params: IQueryMembersAdvancedParams,
 ): Promise<void> {
-  try {
-    log.info(`Refreshing members advanced count cache in background: ${cacheKey}`)
-    await executeQuery(qx, redis, cacheKey, { ...params, countOnly: true })
-    log.info(`Members advanced count cache refreshed in background: ${cacheKey}`)
-  } catch (error) {
-    log.warn('Background count cache refresh failed:', error)
-  }
+  return refreshCacheInBackground(qx, redis, cacheKey, params, true)
 }
 
 export async function queryMembers<T extends MemberField>(

services/libs/data-access-layer/src/members/queryCache.ts

@@ -1,4 +1,4 @@
-import { createHash } from 'crypto'
+import { createHash, randomBytes } from 'crypto'
 
 import { getServiceLogger } from '@crowd/logging'
 import { RedisCache, RedisClient } from '@crowd/redis'
@@ -19,10 +19,33 @@ const log = getServiceLogger()
 export class MemberQueryCache {
   private cache: RedisCache
   private countCache: RedisCache
+  private lockCache: RedisCache
 
   constructor(redis: RedisClient) {
     this.cache = new RedisCache('members-advanced', redis, log)
     this.countCache = new RedisCache('members-count', redis, log)
+    this.lockCache = new RedisCache('members-refresh-lock', redis, log)
+  }
+
+  // Returns true if lock was acquired (no other refresh in progress for this key).
+  // Uses a cryptographically random token to distinguish "we set it" from "already existed".
+  // TTL ensures the lock auto-expires if the refresh crashes without releasing it.
+  async tryAcquireRefreshLock(cacheKey: string, ttlSeconds = 90): Promise<boolean> {
+    try {
+      const token = randomBytes(16).toString('hex')
+      const stored = await this.lockCache.setIfNotExistsOrGet(cacheKey, token, ttlSeconds)
+      return stored === token
+    } catch {
+      return true // fail open: if Redis is down, let the refresh proceed
+    }
+  }
+
+  async releaseRefreshLock(cacheKey: string): Promise<void> {
+    try {
+      await this.lockCache.delete(cacheKey)
+    } catch {
+      // best effort
+    }
   }
 
   buildCacheKey(params: {
@@ -116,12 +139,13 @@ export class MemberQueryCache {
 
   async invalidateAll(): Promise<void> {
     try {
-      const [resultsDeleted, countsDeleted] = await Promise.all([
+      const [resultsDeleted, countsDeleted, locksDeleted] = await Promise.all([
         this.cache.deleteAll(),
         this.countCache.deleteAll(),
+        this.lockCache.deleteAll(),
       ])
       log.info(
-        `Invalidated member query cache: ${resultsDeleted} result entries, ${countsDeleted} count entries`,
+        `Invalidated member query cache: ${resultsDeleted} result entries, ${countsDeleted} count entries, ${locksDeleted} locks`,
       )
     } catch (error) {
       log.warn('Error invalidating member query cache', { error })
@@ -130,12 +154,13 @@ export class MemberQueryCache {
 
   async invalidateByPattern(pattern: string): Promise<void> {
     try {
-      const [resultsDeleted, countsDeleted] = await Promise.all([
+      const [resultsDeleted, countsDeleted, locksDeleted] = await Promise.all([
         this.cache.deleteByKeyPattern(pattern),
         this.countCache.deleteByKeyPattern(pattern),
+        this.lockCache.deleteByKeyPattern(pattern),
       ])
       log.info(
-        `Invalidated member query cache by pattern: ${resultsDeleted} result entries, ${countsDeleted} count entries deleted for pattern ${pattern}`,
+        `Invalidated member query cache by pattern: ${resultsDeleted} result entries, ${countsDeleted} count entries, ${locksDeleted} locks deleted for pattern ${pattern}`,
       )
     } catch (error) {
       log.warn('Error invalidating member query cache by pattern', { error, pattern })