Skip to content

Commit 39a7ea7

Browse files
thehabesthehabes
committed
Add PM2 cluster synchronization for cache operations
- Wrap cache.set(), cache.invalidate(), cache.clear() to broadcast to all PM2 instances - Listen for 'process:msg' events to sync cache operations across cluster - Syncs cache data (set), invalidations, and clears across all instances - No overhead in non-cluster mode (checks process.send) - Minimal overhead in cluster mode (~1-5ms per operation) - Test script updated to handle load-balanced environments
1 parent 468810d commit 39a7ea7

2 files changed

Lines changed: 429 additions & 0 deletions

File tree

‎cache/__tests__/test-cache-fill.sh‎

Lines changed: 312 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,312 @@
1+
#!/bin/bash
2+
3+
# Test script to verify cache fills to 1000 entries properly
4+
# Tests the improved parallelism handling with reduced batch size and timeouts
5+
6+
# Configuration
7+
BASE_URL="${BASE_URL:-http://localhost:3005}"
8+
TARGET_SIZE=1000
9+
BATCH_SIZE=20
10+
11+
# Determine API paths based on URL
12+
if [[ "$BASE_URL" == *"devstore.rerum.io"* ]] || [[ "$BASE_URL" == *"store.rerum.io"* ]]; then
13+
# Production/dev server paths
14+
CACHE_STATS_PATH="/v1/api/cache/stats"
15+
CACHE_CLEAR_PATH="/v1/api/cache/clear"
16+
API_QUERY_PATH="/v1/api/query"
17+
else
18+
# Local server paths
19+
CACHE_STATS_PATH="/cache/stats"
20+
CACHE_CLEAR_PATH="/cache/clear"
21+
API_QUERY_PATH="/api/query"
22+
fi
23+
24+
# Colors for output
25+
RED='\033[0;31m'
26+
GREEN='\033[0;32m'
27+
YELLOW='\033[1;33m'
28+
BLUE='\033[0;34m'
29+
NC='\033[0m' # No Color
30+
31+
echo "═══════════════════════════════════════════════════════════════════════"
32+
echo " RERUM Cache Fill Test"
33+
echo "═══════════════════════════════════════════════════════════════════════"
34+
echo ""
35+
echo "Testing cache fill to $TARGET_SIZE entries with improved parallelism handling"
36+
echo "Server: $BASE_URL"
37+
echo "Batch size: $BATCH_SIZE requests per batch"
38+
echo ""
39+
40+
# Check server connectivity
41+
echo -n "[INFO] Checking server connectivity... "
42+
if ! curl -sf "$BASE_URL" > /dev/null 2>&1; then
43+
echo -e "${RED}FAIL${NC}"
44+
echo "Server at $BASE_URL is not responding"
45+
exit 1
46+
fi
47+
echo -e "${GREEN}OK${NC}"
48+
49+
# Clear cache
50+
echo -n "[INFO] Clearing cache... "
51+
if [[ "$BASE_URL" == *"devstore.rerum.io"* ]] || [[ "$BASE_URL" == *"store.rerum.io"* ]]; then
52+
# Production/dev servers may be load-balanced with multiple instances
53+
# Clear multiple times to hit all instances
54+
for i in {1..5}; do
55+
curl -sf -X POST "$BASE_URL$CACHE_CLEAR_PATH" > /dev/null 2>&1
56+
done
57+
sleep 1
58+
echo -e "${YELLOW}WARN${NC}"
59+
echo "[INFO] Note: Server appears to be load-balanced across multiple instances"
60+
echo "[INFO] Cache clear may not affect all instances - continuing with test"
61+
else
62+
# Local server - single instance
63+
curl -sf -X POST "$BASE_URL$CACHE_CLEAR_PATH" > /dev/null 2>&1
64+
sleep 1
65+
initial_stats=$(curl -sf "$BASE_URL$CACHE_STATS_PATH")
66+
initial_length=$(echo "$initial_stats" | grep -o '"length":[0-9]*' | cut -d: -f2)
67+
if [ "$initial_length" = "0" ]; then
68+
echo -e "${GREEN}OK${NC} (length: 0)"
69+
else
70+
echo -e "${YELLOW}WARN${NC} (length: $initial_length)"
71+
fi
72+
fi
73+
74+
# Fill cache function with improved error handling
75+
SUCCESSFUL_REQUESTS=0
76+
FAILED_REQUESTS=0
77+
TIMEOUT_REQUESTS=0
78+
79+
fill_cache() {
80+
local target_size=$1
81+
local successful_requests=0
82+
local failed_requests=0
83+
local timeout_requests=0
84+
85+
echo ""
86+
echo "▓▓▓ Filling Cache to $target_size Entries ▓▓▓"
87+
echo ""
88+
89+
for ((i=0; i<target_size; i+=BATCH_SIZE)); do
90+
local batch_end=$((i + BATCH_SIZE))
91+
if [ $batch_end -gt $target_size ]; then
92+
batch_end=$target_size
93+
fi
94+
95+
# Clear temp file for this batch
96+
rm -f /tmp/cache_fill_results_$$.tmp
97+
98+
# Send batch of requests in parallel
99+
for ((j=i; j<batch_end; j++)); do
100+
(
101+
# Use different query types that return actual data
102+
# Cycle through known query patterns that return results
103+
queries=(
104+
'{"type":"Annotation"}'
105+
'{"@type":"Annotation"}'
106+
'{"@type":"Gloss"}'
107+
'{"@type":"Person"}'
108+
'{"type":"Person"}'
109+
'{"type":"Manifest"}'
110+
'{"type":"Canvas"}'
111+
'{"type":"AnnotationPage"}'
112+
)
113+
114+
# Select query based on index to create variety
115+
query_index=$((j % 8))
116+
query_body="${queries[$query_index]}"
117+
118+
# Add a unique parameter to each query to ensure they're cached separately
119+
# Use the request number to make each query unique
120+
query_body=$(echo "$query_body" | sed "s/}/, \"_test_id\": $j}/")
121+
122+
response=$(curl -s \
123+
--max-time 30 \
124+
--connect-timeout 10 \
125+
-w "\n%{http_code}" \
126+
-X POST \
127+
-H "Content-Type: application/json" \
128+
-d "$query_body" \
129+
"$BASE_URL$API_QUERY_PATH" 2>&1)
130+
131+
exit_code=$?
132+
http_code=$(echo "$response" | tail -1)
133+
134+
if [ $exit_code -eq 28 ]; then
135+
# Timeout
136+
echo "timeout" >> /tmp/cache_fill_results_$$.tmp
137+
elif [ $exit_code -ne 0 ]; then
138+
# Network error
139+
echo "fail:network_error_$exit_code" >> /tmp/cache_fill_results_$$.tmp
140+
elif [ "$http_code" = "200" ]; then
141+
# Success
142+
echo "success" >> /tmp/cache_fill_results_$$.tmp
143+
else
144+
# HTTP error
145+
echo "fail:http_$http_code" >> /tmp/cache_fill_results_$$.tmp
146+
fi
147+
) &
148+
done
149+
150+
# Wait for all requests in this batch to complete
151+
wait
152+
153+
# Count results from temp file
154+
batch_success=0
155+
batch_timeout=0
156+
batch_fail=0
157+
if [ -f /tmp/cache_fill_results_$$.tmp ]; then
158+
batch_success=$(grep -c "^success$" /tmp/cache_fill_results_$$.tmp 2>/dev/null)
159+
batch_timeout=$(grep -c "^timeout$" /tmp/cache_fill_results_$$.tmp 2>/dev/null)
160+
batch_fail=$(grep -c "^fail:" /tmp/cache_fill_results_$$.tmp 2>/dev/null)
161+
# grep -c returns 0 if no matches, so these are safe
162+
batch_success=${batch_success:-0}
163+
batch_timeout=${batch_timeout:-0}
164+
batch_fail=${batch_fail:-0}
165+
rm /tmp/cache_fill_results_$$.tmp
166+
fi
167+
168+
successful_requests=$((successful_requests + batch_success))
169+
timeout_requests=$((timeout_requests + batch_timeout))
170+
failed_requests=$((failed_requests + batch_fail))
171+
172+
completed=$batch_end
173+
local pct=$((completed * 100 / target_size))
174+
echo -ne "\r Progress: $completed/$target_size requests sent (${pct}%) | Success: $successful_requests | Timeout: $timeout_requests | Failed: $failed_requests "
175+
176+
# Add small delay between batches to prevent overwhelming the server
177+
sleep 0.5
178+
done
179+
echo ""
180+
181+
# Summary
182+
echo ""
183+
echo "▓▓▓ Request Statistics ▓▓▓"
184+
echo ""
185+
echo " Total requests sent: $target_size"
186+
echo -e " Successful (200 OK): ${GREEN}$successful_requests${NC}"
187+
if [ $timeout_requests -gt 0 ]; then
188+
echo " Timeouts: $timeout_requests"
189+
else
190+
echo " Timeouts: $timeout_requests"
191+
fi
192+
if [ $failed_requests -gt 0 ]; then
193+
echo -e " Failed: ${RED}$failed_requests${NC}"
194+
else
195+
echo " Failed: $failed_requests"
196+
fi
197+
echo ""
198+
199+
# Store in global variables for later use
200+
SUCCESSFUL_REQUESTS=$successful_requests
201+
FAILED_REQUESTS=$failed_requests
202+
TIMEOUT_REQUESTS=$timeout_requests
203+
}
204+
205+
# Fill the cache
206+
fill_cache $TARGET_SIZE
207+
208+
# Get final cache stats
209+
echo "[INFO] Getting final cache statistics..."
210+
final_stats=$(curl -sf "$BASE_URL$CACHE_STATS_PATH")
211+
final_length=$(echo "$final_stats" | grep -o '"length":[0-9]*' | cut -d: -f2)
212+
total_sets=$(echo "$final_stats" | grep -o '"sets":[0-9]*' | cut -d: -f2)
213+
total_hits=$(echo "$final_stats" | grep -o '"hits":[0-9]*' | cut -d: -f2)
214+
total_misses=$(echo "$final_stats" | grep -o '"misses":[0-9]*' | cut -d: -f2)
215+
total_evictions=$(echo "$final_stats" | grep -o '"evictions":[0-9]*' | cut -d: -f2)
216+
217+
echo ""
218+
echo "▓▓▓ Final Cache Statistics ▓▓▓"
219+
echo ""
220+
echo " Cache entries: $final_length"
221+
echo " Total sets: $total_sets"
222+
echo " Total hits: $total_hits"
223+
echo " Total misses: $total_misses"
224+
echo " Total evictions: $total_evictions"
225+
echo ""
226+
227+
# Analyze results
228+
echo "▓▓▓ Analysis ▓▓▓"
229+
echo ""
230+
231+
success=true
232+
233+
# Check request success rate first (most important)
234+
success_rate=$((SUCCESSFUL_REQUESTS * 100 / TARGET_SIZE))
235+
if [ $success_rate -ge 95 ]; then
236+
echo -e "${GREEN}✓${NC} Excellent request success rate: ${success_rate}% (${SUCCESSFUL_REQUESTS}/${TARGET_SIZE})"
237+
elif [ $success_rate -ge 90 ]; then
238+
echo -e "${YELLOW}⚠${NC} Good request success rate: ${success_rate}% (${SUCCESSFUL_REQUESTS}/${TARGET_SIZE})"
239+
else
240+
echo -e "${RED}✗${NC} Poor request success rate: ${success_rate}% (${SUCCESSFUL_REQUESTS}/${TARGET_SIZE})"
241+
success=false
242+
fi
243+
244+
# Check timeouts
245+
if [ $TIMEOUT_REQUESTS -eq 0 ]; then
246+
echo -e "${GREEN}✓${NC} No timeouts"
247+
elif [ $TIMEOUT_REQUESTS -lt $((TARGET_SIZE / 20)) ]; then
248+
echo -e "${GREEN}✓${NC} Very few timeouts: $TIMEOUT_REQUESTS"
249+
else
250+
echo -e "${YELLOW}⚠${NC} Some timeouts: $TIMEOUT_REQUESTS"
251+
fi
252+
253+
# Check failures
254+
if [ $FAILED_REQUESTS -eq 0 ]; then
255+
echo -e "${GREEN}✓${NC} No failed requests"
256+
elif [ $FAILED_REQUESTS -lt $((TARGET_SIZE / 20)) ]; then
257+
echo -e "${GREEN}✓${NC} Very few failures: $FAILED_REQUESTS"
258+
else
259+
echo -e "${YELLOW}⚠${NC} Some failures: $FAILED_REQUESTS"
260+
fi
261+
262+
# Check if cache filled (but this depends on query results)
263+
if [ "$final_length" -ge 990 ]; then
264+
echo -e "${GREEN}✓${NC} Cache filled successfully (${final_length}/${TARGET_SIZE} entries)"
265+
elif [ "$final_length" -ge 300 ]; then
266+
echo -e "${YELLOW}ℹ${NC} Cache has ${final_length} entries (many queries returned empty results)"
267+
echo " Note: Cache only stores non-empty array responses by design"
268+
else
269+
echo -e "${RED}✗${NC} Cache fill lower than expected (${final_length}/${TARGET_SIZE} entries)"
270+
success=false
271+
fi
272+
273+
# Diagnose issues if any
274+
if [ "$success" != "true" ]; then
275+
echo ""
276+
echo "▓▓▓ Diagnosis ▓▓▓"
277+
echo ""
278+
279+
if [ $TIMEOUT_REQUESTS -gt $((TARGET_SIZE / 10)) ]; then
280+
echo -e "${YELLOW}⚠${NC} High number of timeouts detected"
281+
echo " Recommendation: Increase --max-time or reduce batch size"
282+
fi
283+
284+
if [ $FAILED_REQUESTS -gt $((TARGET_SIZE / 10)) ]; then
285+
echo -e "${YELLOW}⚠${NC} High number of failed requests"
286+
echo " Recommendation: Check server logs for errors"
287+
fi
288+
289+
# Check if responses weren't cached (might not be arrays)
290+
if [ -n "$total_sets" ] && [ -n "$SUCCESSFUL_REQUESTS" ] && [ "$total_sets" -lt $((SUCCESSFUL_REQUESTS - 50)) ]; then
291+
echo -e "${YELLOW}⚠${NC} Many successful responses were NOT cached"
292+
echo " Reason: Responses may not be arrays (cache only stores array responses)"
293+
echo " Sets: $total_sets vs Successful requests: $SUCCESSFUL_REQUESTS"
294+
fi
295+
296+
if [ -n "$total_evictions" ] && [ "$total_evictions" -gt 0 ]; then
297+
echo -e "${YELLOW}⚠${NC} Cache evictions occurred during fill"
298+
echo " Evictions: $total_evictions"
299+
echo " Reason: Cache may be full or entries timing out"
300+
fi
301+
fi
302+
303+
echo ""
304+
echo "═══════════════════════════════════════════════════════════════════════"
305+
306+
if [ "$success" = "true" ]; then
307+
echo -e "${GREEN}TEST PASSED${NC}"
308+
exit 0
309+
else
310+
echo -e "${YELLOW}TEST COMPLETED WITH WARNINGS${NC}"
311+
exit 1
312+
fi

0 commit comments

Comments
 (0)