Skip to content

Commit 5c03709

Browse files
BigGoodnessBigGoodness
committed
feat(cache): add Redis cache warming service
Add standalone NestJS cache-warming microservice with preload entities, popularity scoring, adaptive warming, scheduled invalidation, metrics, and Docker/Postgres/Redis configuration.
1 parent 807d71e commit 5c03709

28 files changed

Lines changed: 12317 additions & 2 deletions

‎.env.example‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,17 @@ LOG_LEVEL=info
2626
#Environment
2727
NODE_ENV=development
2828

29+
#Cache Warming Service
30+
CACHE_WARMING_PORT=3021
31+
CACHE_WARMING_DB_NAME=cache_warming_db
32+
CACHE_WARMING_SCHEDULER_ENABLED=true
33+
CACHE_WARMING_SEED_DEFAULTS=true
34+
CACHE_WARM_ON_STARTUP=true
35+
CACHE_WARMING_LIMIT=50
36+
CACHE_ADAPTIVE_WARMING_LIMIT=30
37+
CACHE_HIT_RATE_TARGET=0.85
38+
REDIS_PASSWORD=redis123
39+
2940
#Backup Configuration
3041
BACKUP_PATH=./backups
3142
BACKUP_RETENTION_DAYS=30
@@ -43,4 +54,4 @@ DISCORD_PUBLIC_KEY=your-discord-public-key
4354
#Twitter/X Integration
4455
TWITTER_CLIENT_ID=your-twitter-client-id
4556
TWITTER_CLIENT_SECRET=your-twitter-client-secret
46-
APP_URL=http://localhost:3000
57+
APP_URL=http://localhost:3000

‎docker-compose.yml‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,38 @@ services:
119119
networks:
120120
- quest-network
121121
logging: *logging_config
122+
123+
cache-warming-service:
124+
build:
125+
context: ./microservices/cache-warming-service
126+
dockerfile: Dockerfile
127+
container_name: cache_warming_service
128+
ports:
129+
- "${CACHE_WARMING_PORT:-3021}:3021"
130+
environment:
131+
- NODE_ENV=development
132+
- PORT=3021
133+
- DB_HOST=postgres
134+
- DB_PORT=5432
135+
- DB_USER=${DB_USER:-postgres}
136+
- DB_PASSWORD=${DB_PASSWORD:-password}
137+
- DB_NAME=cache_warming_db
138+
- REDIS_HOST=redis
139+
- REDIS_PORT=6379
140+
- REDIS_PASSWORD=${REDIS_PASSWORD:-redis123}
141+
- CACHE_WARM_ON_STARTUP=${CACHE_WARM_ON_STARTUP:-true}
142+
- CACHE_WARMING_LIMIT=${CACHE_WARMING_LIMIT:-50}
143+
- CACHE_ADAPTIVE_WARMING_LIMIT=${CACHE_ADAPTIVE_WARMING_LIMIT:-30}
144+
- CACHE_HIT_RATE_TARGET=${CACHE_HIT_RATE_TARGET:-0.85}
145+
- JAEGER_ENDPOINT=http://jaeger:14268/api/traces
146+
depends_on:
147+
postgres:
148+
condition: service_healthy
149+
redis:
150+
condition: service_healthy
151+
networks:
152+
- quest-network
153+
logging: *logging_config
122154

123155
recommendation-service:
124156
build:
@@ -373,14 +405,18 @@ services:
373405
container_name: redis_main
374406
ports:
375407
- '${REDIS_PORT:-6379}:6379'
408+
environment:
409+
- REDIS_PASSWORD=${REDIS_PASSWORD:-redis123}
376410
volumes:
377411
- redis_data:/data
378412
command: redis-server --appendonly yes --requirepass ${REDIS_PASSWORD:-redis123}
379413
healthcheck:
380-
test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping']
414+
test: ['CMD-SHELL', 'redis-cli -a "$$REDIS_PASSWORD" --raw incr ping']
381415
interval: 10s
382416
timeout: 3s
383417
retries: 5
418+
networks:
419+
- quest-network
384420
restart: unless-stopped
385421

386422
rabbitmq:

‎docker/postgres/init-databases.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,6 @@ create_db "player_db"
2323
create_db "moderation_db"
2424
create_db "analytics_db"
2525
create_db "payment_db"
26+
create_db "cache_warming_db"
2627

2728
echo "All microservice databases created successfully!"

‎microservices/cache-warming-service/.env.example‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
NODE_ENV=development
2+
PORT=3021
3+
4+
DB_HOST=localhost
5+
DB_PORT=5432
6+
DB_USER=postgres
7+
DB_PASSWORD=password
8+
DB_NAME=cache_warming_db
9+
10+
REDIS_HOST=localhost
11+
REDIS_PORT=6379
12+
REDIS_PASSWORD=redis123
13+
# REDIS_URL=redis://:redis123@localhost:6379
14+
15+
CACHE_WARMING_SCHEDULER_ENABLED=true
16+
CACHE_WARMING_SEED_DEFAULTS=true
17+
CACHE_WARM_ON_STARTUP=true
18+
CACHE_WARMING_LIMIT=50
19+
CACHE_ADAPTIVE_WARMING_LIMIT=30
20+
CACHE_HIT_RATE_TARGET=0.85
21+
PRELOAD_FETCH_TIMEOUT_MS=2500

‎microservices/cache-warming-service/.eslintrc.cjs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
module.exports = {
2+
parser: '@typescript-eslint/parser',
3+
parserOptions: {
4+
project: './tsconfig.json',
5+
tsconfigRootDir: __dirname,
6+
sourceType: 'module',
7+
},
8+
plugins: ['@typescript-eslint', 'prettier'],
9+
extends: [
10+
'eslint:recommended',
11+
'plugin:@typescript-eslint/recommended',
12+
'plugin:prettier/recommended',
13+
],
14+
env: {
15+
node: true,
16+
jest: true,
17+
},
18+
ignorePatterns: ['dist', 'node_modules'],
19+
rules: {
20+
'@typescript-eslint/no-explicit-any': 'off',
21+
'@typescript-eslint/no-floating-promises': 'warn',
22+
'@typescript-eslint/no-unsafe-argument': 'warn',
23+
},
24+
};

‎microservices/cache-warming-service/Dockerfile‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
FROM node:20-alpine AS build
2+
3+
WORKDIR /app
4+
5+
COPY package*.json ./
6+
RUN npm install
7+
8+
COPY . .
9+
RUN npm run build
10+
11+
FROM node:20-alpine
12+
13+
WORKDIR /app
14+
15+
COPY --from=build /app/dist ./dist
16+
COPY --from=build /app/node_modules ./node_modules
17+
COPY package*.json ./
18+
19+
ENV NODE_ENV=production
20+
EXPOSE 3021
21+
22+
CMD ["npm", "run", "start:prod"]

‎microservices/cache-warming-service/README.md‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Cache Warming Service
2+
3+
Independent NestJS service that proactively warms Redis with popular Quest Service data.
4+
5+
## What It Does
6+
7+
- Tracks preload candidates in Postgres with popularity, priority, TTL, and time-of-day windows.
8+
- Warms Redis on a schedule and skips keys that still have healthy TTL.
9+
- Raises priority for missed keys and runs hit-rate optimization.
10+
- Schedules immediate, delayed, repeating, and per-entry invalidation.
11+
- Persists warming jobs and metric samples for monitoring.
12+
13+
## Run Locally
14+
15+
```bash
16+
cp .env.example .env
17+
npm install
18+
npm run start:dev
19+
```
20+
21+
Or run the service with its local Postgres and Redis:
22+
23+
```bash
24+
docker-compose up --build
25+
```
26+
27+
## Useful Endpoints
28+
29+
- `GET /api/health`
30+
- `GET /api/cache-warming/dashboard`
31+
- `POST /api/cache-warming/preload-data`
32+
- `POST /api/cache-warming/access`
33+
- `POST /api/cache-warming/warm`
34+
- `POST /api/cache-warming/invalidate`
35+
- `POST /api/cache-warming/invalidate/schedule`
36+
- `POST /api/cache-warming/optimize`
37+
- `GET /api/cache-warming/jobs`
38+
- `GET /api/cache-warming/metrics`

‎microservices/cache-warming-service/docker-compose.yml‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
version: '3.8'
2+
3+
services:
4+
cache-warming-service:
5+
build:
6+
context: .
7+
dockerfile: Dockerfile
8+
container_name: cache_warming_service
9+
ports:
10+
- '${CACHE_WARMING_PORT:-3021}:3021'
11+
environment:
12+
- NODE_ENV=development
13+
- PORT=3021
14+
- DB_HOST=postgres
15+
- DB_PORT=5432
16+
- DB_USER=${DB_USER:-postgres}
17+
- DB_PASSWORD=${DB_PASSWORD:-password}
18+
- DB_NAME=${CACHE_WARMING_DB_NAME:-cache_warming_db}
19+
- REDIS_HOST=redis
20+
- REDIS_PORT=6379
21+
- REDIS_PASSWORD=${REDIS_PASSWORD:-redis123}
22+
- CACHE_WARM_ON_STARTUP=${CACHE_WARM_ON_STARTUP:-true}
23+
- CACHE_WARMING_LIMIT=${CACHE_WARMING_LIMIT:-50}
24+
- CACHE_ADAPTIVE_WARMING_LIMIT=${CACHE_ADAPTIVE_WARMING_LIMIT:-30}
25+
- CACHE_HIT_RATE_TARGET=${CACHE_HIT_RATE_TARGET:-0.85}
26+
depends_on:
27+
postgres:
28+
condition: service_healthy
29+
redis:
30+
condition: service_healthy
31+
networks:
32+
- cache-warming-network
33+
34+
postgres:
35+
image: postgres:15-alpine
36+
container_name: cache_warming_postgres
37+
ports:
38+
- '${CACHE_WARMING_DB_PORT:-5434}:5432'
39+
environment:
40+
POSTGRES_USER: ${DB_USER:-postgres}
41+
POSTGRES_PASSWORD: ${DB_PASSWORD:-password}
42+
POSTGRES_DB: ${CACHE_WARMING_DB_NAME:-cache_warming_db}
43+
volumes:
44+
- cache_warming_postgres_data:/var/lib/postgresql/data
45+
healthcheck:
46+
test: ['CMD-SHELL', 'pg_isready -U ${DB_USER:-postgres}']
47+
interval: 5s
48+
timeout: 5s
49+
retries: 5
50+
networks:
51+
- cache-warming-network
52+
53+
redis:
54+
image: redis:7-alpine
55+
container_name: cache_warming_redis
56+
ports:
57+
- '${CACHE_WARMING_REDIS_PORT:-6381}:6379'
58+
command: redis-server --appendonly yes --requirepass ${REDIS_PASSWORD:-redis123}
59+
volumes:
60+
- cache_warming_redis_data:/data
61+
healthcheck:
62+
test: ['CMD', 'redis-cli', '-a', '${REDIS_PASSWORD:-redis123}', '--raw', 'incr', 'ping']
63+
interval: 10s
64+
timeout: 3s
65+
retries: 5
66+
networks:
67+
- cache-warming-network
68+
69+
volumes:
70+
cache_warming_postgres_data:
71+
cache_warming_redis_data:
72+
73+
networks:
74+
cache-warming-network:
75+
driver: bridge

‎microservices/cache-warming-service/nest-cli.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"$schema": "https://json.schemastore.org/nest-cli",
3+
"collection": "@nestjs/schematics",
4+
"sourceRoot": "src",
5+
"compilerOptions": {
6+
"deleteOutDir": true
7+
}
8+
}

0 commit comments

Comments
 (0)