ray/examples/config/ray.1b.8gb.generic.json at main · Agent-Pattern-Labs/ray · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
{
  "profile": "1b-8gb",
  "server": {
    "requestBodyLimitBytes": 64000
  },
  "model": {
    "id": "local-1b-q4",
    "family": "generic-1b",
    "quantization": "q4_k_m",
    "maxOutputTokens": 256,
    "operational": {
      "recommendedPromptFormat": "native-template",
      "supportsJsonMode": true,
      "tokensPerSecondTarget": 18,
      "memoryClassMiB": 8192,
      "preferredCtxSize": 4096,
      "chatTemplateKnown": true
    },
    "adapter": {
      "kind": "llama.cpp",
      "modelRef": "local-1b-q4",
      "timeoutMs": 24000,
      "slotStateTtlMs": 250,
      "slotSnapshotTimeoutMs": 300,
      "promptScaffoldCacheEntries": 384,
      "launchProfile": {
        "preset": "single-vps-1b-8gb",
        "modelPath": "/var/lib/ray/models/local-1b-q4.gguf",
        "alias": "local-1b-q4",
        "ctxSize": 4096,
        "parallel": 2,
        "threads": 4,
        "threadsBatch": 4,
        "threadsHttp": 2,
        "batchSize": 256,
        "ubatchSize": 128,
        "cacheReuse": 256,
        "cacheRamMiB": 768
      }
    }
  },
  "scheduler": {
    "concurrency": 2,
    "maxQueue": 96,
    "maxQueuedTokens": 48000,
    "maxInflightTokens": 6144,
    "requestTimeoutMs": 28000,
    "dedupeInflight": true,
    "batchWindowMs": 10,
    "affinityLookahead": 24,
    "shortJobMaxTokens": 96
  },
  "cache": {
    "maxEntries": 512,
    "maxBytes": 4194304,
    "ttlMs": 120000
  },
  "telemetry": {
    "slowRequestThresholdMs": 1800
  },
  "asyncQueue": {
    "minFreeStorageMiB": 512
  },
  "gracefulDegradation": {
    "queueDepthThreshold": 20,
    "maxPromptChars": 8000,
    "degradeToMaxTokens": 160,
    "memoryRssThresholdMiB": 768
  },
  "adaptiveTuning": {
    "queueLatencyThresholdMs": 450,
    "minCompletionTokensPerSecond": 14
  },
  "rateLimit": {
    "maxRequests": 150,
    "maxKeys": 8192
  },
  "tags": {
    "target": "1b",
    "hardware": "generic-8gb-vps-class",
    "engine": "llama.cpp",
    "modelSize": "1b-class",
    "exposure": "private-local",
    "portability": "generic-model"
  }
}