Skip to content

Commit 55e1732

Browse files
nkaradzhovclaude
andcommitted
feat(client): sticky FT.CURSOR routing via special request policy
FT.CURSOR READ/DEL carry no key, so hash-slot routing can't reach the coordinator that minted the cursor via FT.AGGREGATE ...WITHCURSOR and the server rejects the unknown cursor. Flip ft.cursor to the HLD `special` request policy and add client-side sticky machinery: - cluster-slots: per-instance cursor binding registry ((index,cursorId) -> address) with bind/lookup/evict + opportunistic idle sweep, plus nodeAddressByClient reverse-lookup. - ft-cursor: routeFtCursor (pin bound node / throw MISS before any network call), SPECIAL_REQUEST_ROUTERS, extractCursorId, and the captureCursorBinding post-reply hook (AGGREGATE bind / READ rebind-refresh-evict / DEL evict). - dispatch: routeSpecial short-circuits into SPECIAL_REQUEST_ROUTERS, keeps the warn-and-random fallback for unregistered special commands. - _executeWithPolicies: best-effort capture hook after the reply resolves; _execute untouched. - static policies: override ft.cursor -> special (+ READ/DEL subcommands, response stays default-keyless), regenerated data table. Response stays single-node pass-through (no special reducer). Bindings are per client instance; cross-instance/cross-process cursors MISS by design. Unit tests cover router/capture/lifecycle/collision; cluster integration tests cover pagination-to-completion, DEL->READ MISS, and cross-instance MISS. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 93a5789 commit 55e1732

10 files changed

Lines changed: 541 additions & 22 deletions

File tree

packages/client/lib/cluster/cluster-slots.ts

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,26 @@ export type NodeAddressMap = {
2222

2323
export const RESUBSCRIBE_LISTENERS_EVENT = '__resubscribeListeners'
2424

25+
/**
26+
* Sticky-cursor binding: which node served a RediSearch cursor. FT.CURSOR
27+
* READ/DEL carry no key, so hash-slot routing can't reach the coordinator that
28+
* minted the cursor — we pin by `address` ("host:port"), the durable handle
29+
* (clients are recreated on reconnect/topology refresh, addresses aren't).
30+
*/
31+
export interface CursorBinding {
32+
address: string;
33+
createdAt: number;
34+
maxIdleMs?: number;
35+
}
36+
37+
/**
38+
* Fallback idle TTL for the opportunistic cursor-binding sweep when the
39+
* FT.AGGREGATE didn't declare MAXIDLE. Mirrors the RediSearch default (300s)
40+
* so abandoned cursors don't leak the binding map (timer-free, like
41+
* `smigratedSeqIdsSeen`).
42+
*/
43+
const DEFAULT_CURSOR_MAX_IDLE_MS = 300_000;
44+
2545
export interface Node<
2646
M extends RedisModules,
2747
F extends RedisFunctions,
@@ -121,6 +141,8 @@ export default class RedisClusterSlots<
121141
pubSubNode?: PubSubNode<M, F, S, RESP, TYPE_MAPPING>;
122142
clientSideCache?: PooledClientSideCacheProvider;
123143
smigratedSeqIdsSeen = new Set<number>;
144+
/** Per-instance sticky-cursor bindings, keyed `${index}:${cursorId}`. */
145+
readonly cursorBindings = new Map<string, CursorBinding>();
124146
#topologyRefreshPromise?: Promise<boolean | void>;
125147

126148
#isOpen = false;
@@ -918,6 +940,50 @@ export default class RedisClusterSlots<
918940
return this.nodeClient(master);
919941
}
920942

943+
/**
944+
* Reverse-resolve a routed client to its node address. FT.AGGREGATE is
945+
* keyless, so the plan carries only the client; we need its address to bind
946+
* the cursor. Clients are few per cluster, so the linear scan is negligible.
947+
*/
948+
nodeAddressByClient(client: RedisClientType<M, F, S, RESP, TYPE_MAPPING>): string | undefined {
949+
for (const [address, node] of this.nodeByAddress) {
950+
if (node.client === client) return address;
951+
}
952+
return undefined;
953+
}
954+
955+
#cursorKey(index: string, cursorId: number) {
956+
return `${index}:${cursorId}`;
957+
}
958+
959+
/**
960+
* Drop bindings idle past their MAXIDLE (or the default TTL). Opportunistic —
961+
* runs on each `bindCursor` so there's no timer to manage (see
962+
* `smigratedSeqIdsSeen`). Cheap: the map holds only live cursors.
963+
*/
964+
#sweepStaleCursors(now: number) {
965+
for (const [key, binding] of this.cursorBindings) {
966+
const ttl = binding.maxIdleMs ?? DEFAULT_CURSOR_MAX_IDLE_MS;
967+
if (now - binding.createdAt > ttl) {
968+
this.cursorBindings.delete(key);
969+
}
970+
}
971+
}
972+
973+
bindCursor(index: string, cursorId: number, address: string, maxIdleMs?: number) {
974+
const now = Date.now();
975+
this.#sweepStaleCursors(now);
976+
this.cursorBindings.set(this.#cursorKey(index, cursorId), { address, createdAt: now, maxIdleMs });
977+
}
978+
979+
lookupCursor(index: string, cursorId: number): CursorBinding | undefined {
980+
return this.cursorBindings.get(this.#cursorKey(index, cursorId));
981+
}
982+
983+
evictCursor(index: string, cursorId: number) {
984+
this.cursorBindings.delete(this.#cursorKey(index, cursorId));
985+
}
986+
921987
getPubSubClient(): Promise<RedisClientType<M, F, S, RESP, TYPE_MAPPING>> {
922988
if (!this.pubSubNode) return this.#initiatePubSubClient();
923989

packages/client/lib/cluster/index.ts

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import { ClientIdentity, ClientRole, generateClusterClientId } from '../client/i
1818
import { DEFAULT_COMMAND_TIMEOUT } from '../defaults';
1919
import { POLICIES, PolicyResolver, StaticPolicyResolver, REQUEST_POLICIES_WITH_DEFAULTS, RESPONSE_POLICIES_WITH_DEFAULTS, type CommandPolicies } from './request-response-policies';
2020
import { REQUEST_ROUTERS, RESPONSE_REDUCERS } from './request-response-policies/dispatch';
21+
import { captureCursorBinding } from './request-response-policies/ft-cursor';
2122

2223
export type ClusterTopologyRefreshOnReconnectionAttemptStrategy =
2324
false |
@@ -535,7 +536,22 @@ export default class RedisCluster<
535536
throw new Error(`Unknown response policy ${responsePolicy}`);
536537
}
537538
const positionHints = plan.map(entry => entry.groupIndices);
538-
return reducer(responsePromises, parser, positionHints) as Promise<T>;
539+
const reply = await (reducer(responsePromises, parser, positionHints) as Promise<T>);
540+
541+
// Sticky-cursor bookkeeping: FT.AGGREGATE/FT.CURSOR bind/rebind/evict the
542+
// serving node from the resolved reply. Command-name gated and best-effort
543+
// (a bad binding only downgrades to a MISS throw on the next READ/DEL), so
544+
// never let it mask the caller's reply.
545+
try {
546+
captureCursorBinding(
547+
this._slots as unknown as Parameters<typeof captureCursorBinding>[0],
548+
parser,
549+
plan,
550+
reply
551+
);
552+
} catch { /* binding capture is best-effort */ }
553+
554+
return reply;
539555
}
540556

541557
/**

packages/client/lib/cluster/request-response-policies/dispatch.ts

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
type RequestPolicyWithDefaults,
2121
type ResponsePolicyWithDefaults
2222
} from './policies-constants';
23+
import { SPECIAL_REQUEST_ROUTERS } from './ft-cursor';
2324

2425
// Routing runs *below* the typed command surface: routers never inspect the
2526
// command's M/F/S/RESP/TM parameters, they just shuffle opaque clients from
@@ -121,15 +122,17 @@ function specialKey(parser: CommandParser): string {
121122
}
122123

123124
/**
124-
* Fallback router for `special` request commands without a dedicated handler.
125-
* A `special` request policy means non-trivial routing that no generic rule
126-
* captures; we don't know the correct target, so route to a single (random)
127-
* node like a keyless command. This keeps the command working instead of
128-
* throwing, but the reply reflects only that one node — warn so the gap is
129-
* visible. Commands with a real handler never reach this.
125+
* Router for the `special` request policy. Commands with a dedicated handler
126+
* (e.g. FT.CURSOR sticky routing) short-circuit into `SPECIAL_REQUEST_ROUTERS`
127+
* first. Everything else has non-trivial routing no generic rule captures and
128+
* no handler yet: route to a single (random) node like a keyless command so it
129+
* still works, but warn — the reply reflects only that one node.
130130
*/
131131
export const routeSpecial: RequestRouter =
132-
async (slots, parser) => {
132+
async (slots, parser, isReadonly, keySpecs) => {
133+
const handler = SPECIAL_REQUEST_ROUTERS[specialKey(parser)];
134+
if (handler) return handler(slots, parser, isReadonly, keySpecs);
135+
133136
console.warn(
134137
`node-redis: no cluster routing implemented for the "special" request policy of ` +
135138
`"${specialKey(parser)}"; routing to a single node. The reply may be incomplete.`
Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
import { strict as assert } from 'node:assert';
2+
import type { CommandParser } from '../../client/parser';
3+
import { routeFtCursor, captureCursorBinding, extractCursorId } from './ft-cursor';
4+
5+
/**
6+
* Minimal stand-in for the cursor-relevant surface of `RedisClusterSlots`,
7+
* mirroring the real `${index}:${cursorId}` keying and address→client map so
8+
* the router/capture logic is exercised without spinning a cluster.
9+
*/
10+
class FakeSlots {
11+
cursorBindings = new Map<string, { address: string; createdAt: number; maxIdleMs?: number }>();
12+
clientsByAddress = new Map<string, object>();
13+
14+
#key(index: string, cursorId: number) { return `${index}:${cursorId}`; }
15+
bindCursor(index: string, cursorId: number, address: string, maxIdleMs?: number) {
16+
this.cursorBindings.set(this.#key(index, cursorId), { address, createdAt: 0, maxIdleMs });
17+
}
18+
lookupCursor(index: string, cursorId: number) { return this.cursorBindings.get(this.#key(index, cursorId)); }
19+
evictCursor(index: string, cursorId: number) { this.cursorBindings.delete(this.#key(index, cursorId)); }
20+
async getMasterByAddress(address: string) { return this.clientsByAddress.get(address); }
21+
nodeAddressByClient(client: object) {
22+
for (const [address, c] of this.clientsByAddress) if (c === client) return address;
23+
return undefined;
24+
}
25+
}
26+
27+
const parserOf = (...args: Array<string>) =>
28+
({ redisArgs: args, commandIdentifier: { command: args[0], subcommand: args[1] } }) as unknown as CommandParser;
29+
30+
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- routers/capture run below the typed surface
31+
const asSlots = (s: FakeSlots) => s as any;
32+
33+
describe('extractCursorId', () => {
34+
it('reads the transformed-path `{ cursor }` object (RESP2 + RESP3)', () => {
35+
assert.equal(extractCursorId({ total: 1, results: [], cursor: 42 }), 42);
36+
});
37+
38+
it('reads raw RESP2 `[result, cursor]` at index 1', () => {
39+
assert.equal(extractCursorId([['result'], 7]), 7);
40+
});
41+
42+
it('reads raw RESP3 map key `cursor`', () => {
43+
assert.equal(extractCursorId(new Map<string, unknown>([['results', []], ['cursor', 9]])), 9);
44+
});
45+
46+
it('returns undefined when there is no cursor (e.g. FT.CURSOR DEL "OK")', () => {
47+
assert.equal(extractCursorId('OK'), undefined);
48+
assert.equal(extractCursorId(null), undefined);
49+
});
50+
});
51+
52+
describe('routeFtCursor', () => {
53+
it('pins the bound client on HIT', async () => {
54+
const slots = new FakeSlots();
55+
const client = { id: 'node-a' };
56+
slots.clientsByAddress.set('127.0.0.1:7000', client);
57+
slots.bindCursor('idx', 123, '127.0.0.1:7000');
58+
59+
const plan = await routeFtCursor(asSlots(slots), parserOf('FT.CURSOR', 'READ', 'idx', '123'), undefined, undefined);
60+
assert.deepEqual(plan, [{ client }]);
61+
});
62+
63+
it('throws on MISS (cursor never bound)', async () => {
64+
const slots = new FakeSlots();
65+
await assert.rejects(
66+
routeFtCursor(asSlots(slots), parserOf('FT.CURSOR', 'READ', 'idx', '404'), undefined, undefined),
67+
/no known node for cursor 404 on index "idx"/
68+
);
69+
});
70+
71+
it('throws when the bound node is gone (getMasterByAddress → undefined)', async () => {
72+
const slots = new FakeSlots();
73+
slots.bindCursor('idx', 5, '127.0.0.1:9999'); // address not in clientsByAddress
74+
await assert.rejects(
75+
routeFtCursor(asSlots(slots), parserOf('FT.CURSOR', 'DEL', 'idx', '5'), undefined, undefined),
76+
/left the cluster/
77+
);
78+
});
79+
});
80+
81+
describe('captureCursorBinding — FT.AGGREGATE', () => {
82+
it('binds (index, cursor) → serving node address (RESP2 array reply)', () => {
83+
const slots = new FakeSlots();
84+
const client = {};
85+
slots.clientsByAddress.set('10.0.0.1:6379', client);
86+
87+
captureCursorBinding(asSlots(slots), parserOf('FT.AGGREGATE', 'idx', '*', 'WITHCURSOR'), [{ client } as any], [[], 55]);
88+
assert.deepEqual(slots.lookupCursor('idx', 55), { address: '10.0.0.1:6379', createdAt: 0, maxIdleMs: undefined });
89+
});
90+
91+
it('binds from the transformed `{ cursor }` reply and captures MAXIDLE', () => {
92+
const slots = new FakeSlots();
93+
const client = {};
94+
slots.clientsByAddress.set('10.0.0.1:6379', client);
95+
96+
captureCursorBinding(
97+
asSlots(slots),
98+
parserOf('FT.AGGREGATE', 'idx', '*', 'WITHCURSOR', 'MAXIDLE', '5000'),
99+
[{ client } as any],
100+
{ total: 0, results: [], cursor: 88 }
101+
);
102+
assert.deepEqual(slots.lookupCursor('idx', 88), { address: '10.0.0.1:6379', createdAt: 0, maxIdleMs: 5000 });
103+
});
104+
105+
it('does not bind when the aggregate exhausts in one batch (cursor 0)', () => {
106+
const slots = new FakeSlots();
107+
const client = {};
108+
slots.clientsByAddress.set('10.0.0.1:6379', client);
109+
110+
captureCursorBinding(asSlots(slots), parserOf('FT.AGGREGATE', 'idx', '*', 'WITHCURSOR'), [{ client } as any], { cursor: 0 });
111+
assert.equal(slots.cursorBindings.size, 0);
112+
});
113+
});
114+
115+
describe('captureCursorBinding — FT.CURSOR lifecycle', () => {
116+
const seed = () => {
117+
const slots = new FakeSlots();
118+
const client = {};
119+
slots.clientsByAddress.set('10.0.0.1:6379', client);
120+
slots.bindCursor('idx', 100, '10.0.0.1:6379');
121+
return { slots, client };
122+
};
123+
124+
it('rebinds a continuation cursor (evict old, bind new, same address)', () => {
125+
const { slots, client } = seed();
126+
captureCursorBinding(asSlots(slots), parserOf('FT.CURSOR', 'READ', 'idx', '100'), [{ client } as any], { cursor: 200 });
127+
assert.equal(slots.lookupCursor('idx', 100), undefined);
128+
assert.deepEqual(slots.lookupCursor('idx', 200), { address: '10.0.0.1:6379', createdAt: 0, maxIdleMs: undefined });
129+
});
130+
131+
it('evicts on READ → cursor 0 (exhausted)', () => {
132+
const { slots, client } = seed();
133+
captureCursorBinding(asSlots(slots), parserOf('FT.CURSOR', 'READ', 'idx', '100'), [{ client } as any], { cursor: 0 });
134+
assert.equal(slots.lookupCursor('idx', 100), undefined);
135+
});
136+
137+
it('keeps the binding when the continuation id is unchanged', () => {
138+
const { slots, client } = seed();
139+
captureCursorBinding(asSlots(slots), parserOf('FT.CURSOR', 'READ', 'idx', '100'), [{ client } as any], { cursor: 100 });
140+
assert.deepEqual(slots.lookupCursor('idx', 100), { address: '10.0.0.1:6379', createdAt: 0, maxIdleMs: undefined });
141+
});
142+
143+
it('evicts on DEL regardless of reply, then a follow-up READ MISSes', async () => {
144+
const { slots, client } = seed();
145+
captureCursorBinding(asSlots(slots), parserOf('FT.CURSOR', 'DEL', 'idx', '100'), [{ client } as any], 'OK');
146+
assert.equal(slots.lookupCursor('idx', 100), undefined);
147+
await assert.rejects(routeFtCursor(asSlots(slots), parserOf('FT.CURSOR', 'READ', 'idx', '100'), undefined, undefined));
148+
});
149+
});
150+
151+
describe('cursor-id collision across indexes', () => {
152+
it('keys on (index, cursorId) so same id under two indexes routes independently', async () => {
153+
const slots = new FakeSlots();
154+
const clientA = { id: 'a' }, clientB = { id: 'b' };
155+
slots.clientsByAddress.set('a:1', clientA);
156+
slots.clientsByAddress.set('b:1', clientB);
157+
slots.bindCursor('idxA', 1, 'a:1');
158+
slots.bindCursor('idxB', 1, 'b:1');
159+
160+
assert.deepEqual(await routeFtCursor(asSlots(slots), parserOf('FT.CURSOR', 'READ', 'idxA', '1'), undefined, undefined), [{ client: clientA }]);
161+
assert.deepEqual(await routeFtCursor(asSlots(slots), parserOf('FT.CURSOR', 'READ', 'idxB', '1'), undefined, undefined), [{ client: clientB }]);
162+
});
163+
});

0 commit comments

Comments
 (0)