We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 3d01273 commit 3bf91d5Copy full SHA for 3bf91d5
1 file changed
src/gateway.py
@@ -47,10 +47,25 @@ async def refresh_workers():
47
48
def get_next_worker(model: str = "") -> Optional[dict]:
49
cache = _worker_cache
50
- available = cache["workers"]
51
- if not available:
+ workers = cache["workers"]
+ if not workers:
52
return None
53
- worker = available[cache["index"] % len(available)]
+
54
+ candidates = workers
55
+ if model:
56
+ current_time = time.time()
57
+ candidates = []
58
+ for w in workers:
59
+ limits = w.get("rate_limited_models", {})
60
+ if model in limits:
61
+ if limits[model] > current_time:
62
+ continue
63
+ candidates.append(w)
64
65
+ if not candidates:
66
+ return None
67
68
+ worker = candidates[cache["index"] % len(candidates)]
69
cache["index"] += 1
70
return worker
71
0 commit comments