Skip to content

Commit f26c18f

Browse files
UbuntuCopilot
Ubuntu
and
Copilot
committed
Merge origin/users/zhangt/merge-distributed-to-tikv: split sdr configs by suffix
Upstream agent (eng-spfresh-worker) committed 4 sdr-host-specific config tweaks (paths to /mnt/nvme, TiKV memory to 40GB) plus refresh-reference.sh and run_perf_validation.sh helpers. Auto-merge took upstream's path/memory changes since our side had not modified these templates. To keep both environments working without fighting on every push: - *_sdr.ini / tikv_sdr.toml upstream's sdr-host config (saved) - *_template.ini / tikv.toml restored to .9/.7 paths + 200GB cache New files added: - configs/benchmark_insert_dominant_template.ini 10\u00d7 expansion workload - /cluster_2node.conf .9 driver + .7 worker Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2 parents 6194ef7 + 33cdc7f commit f26c18f

8 files changed

Lines changed: 604 additions & 0 deletions

‎cluster_2node.conf‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# 2-node cluster: driver/worker0 on .9, worker1 on .7
2+
[cluster]
3+
ssh_user=superbench
4+
ssh_key=/home/superbench/.ssh/rsa
5+
sptag_dir=/home/superbench/zhangt/SPTAG
6+
data_dir=/mnt_ssd/data
7+
tikv_version=v7.5.1
8+
pd_version=v7.5.1
9+
10+
[nodes]
11+
# host router_port
12+
10.11.0.9 30001
13+
10.11.0.7 30002
14+
15+
[tikv]
16+
# host pd_client_port pd_peer_port tikv_port
17+
10.11.0.9 2379 2380 20161
18+
10.11.0.7 2379 2380 20161

‎evaluation/distributed/configs/benchmark_100m_template_sdr.ini‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
[Benchmark]
2+
VectorPath=/mnt/nvme/sift1b/bigann_base.u8bin
3+
QueryPath=/mnt/nvme/sift1b/query.10K.u8bin
4+
TruthPath=/mnt/nvme/sift1b/bigann_truth.bin
5+
IndexPath=PLACEHOLDER
6+
ValueType=UInt8
7+
Dimension=128
8+
BaseVectorCount=99000000
9+
InsertVectorCount=1000000
10+
DeleteVectorCount=0
11+
BatchNum=10
12+
TopK=5
13+
NumSearchThreads=4
14+
NumInsertThreads=16
15+
NumQueries=200
16+
DistMethod=L2
17+
Rebuild=true
18+
BuildOnly=false
19+
Resume=-1
20+
Layers=2
21+
22+
Storage=TIKVIO
23+
TiKVKeyPrefix=PLACEHOLDER
24+
25+
[SelectHead]
26+
ParallelBKTBuild=true
27+
28+
[BuildHead]
29+
ParallelBKTBuild=true
30+
31+
[BuildSSDIndex]
32+
VersionCacheTTLMs=0
33+
VersionCacheMaxChunks=100000
34+
UseMultiChunkPosting=false
35+
AppendThreadNum=48
36+
AsyncMergeInSearch=true
37+
LatencyLimit=100
38+
39+
[Distributed]
40+
Enabled=true
41+
DispatcherAddr=PLACEHOLDER
42+
WorkerAddrs=PLACEHOLDER
43+
StoreAddrs=PLACEHOLDER
44+
PDAddrs=PLACEHOLDER

‎evaluation/distributed/configs/benchmark_10m_template_sdr.ini‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
[Benchmark]
2+
VectorPath=/mnt/nvme/sift1b/bigann_base.u8bin
3+
QueryPath=/mnt/nvme/sift1b/query.10K.u8bin
4+
TruthPath=/mnt/nvme/sift1b/bigann_truth.bin
5+
IndexPath=PLACEHOLDER
6+
ValueType=UInt8
7+
Dimension=128
8+
BaseVectorCount=9900000
9+
InsertVectorCount=100000
10+
DeleteVectorCount=0
11+
BatchNum=10
12+
TopK=5
13+
NumSearchThreads=4
14+
NumInsertThreads=16
15+
NumQueries=200
16+
DistMethod=L2
17+
Rebuild=true
18+
BuildOnly=false
19+
Resume=-1
20+
Layers=2
21+
22+
Storage=TIKVIO
23+
TiKVKeyPrefix=PLACEHOLDER
24+
25+
[SelectHead]
26+
ParallelBKTBuild=true
27+
28+
[BuildHead]
29+
ParallelBKTBuild=true
30+
31+
[BuildSSDIndex]
32+
VersionCacheTTLMs=0
33+
VersionCacheMaxChunks=100000
34+
UseMultiChunkPosting=false
35+
AppendThreadNum=48
36+
AsyncMergeInSearch=true
37+
LatencyLimit=100
38+
39+
[Distributed]
40+
Enabled=true
41+
DispatcherAddr=PLACEHOLDER
42+
WorkerAddrs=PLACEHOLDER
43+
StoreAddrs=PLACEHOLDER
44+
PDAddrs=PLACEHOLDER

‎evaluation/distributed/configs/benchmark_1m_template_sdr.ini‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
[Benchmark]
2+
VectorPath=/mnt/nvme/sift1b/bigann_base.u8bin
3+
QueryPath=/mnt/nvme/sift1b/query.10K.u8bin
4+
TruthPath=/mnt/nvme/sift1b/bigann_truth.bin
5+
IndexPath=PLACEHOLDER
6+
ValueType=UInt8
7+
Dimension=128
8+
BaseVectorCount=990000
9+
InsertVectorCount=10000
10+
DeleteVectorCount=0
11+
BatchNum=10
12+
TopK=5
13+
NumSearchThreads=4
14+
NumInsertThreads=16
15+
NumQueries=200
16+
DistMethod=L2
17+
Rebuild=true
18+
BuildOnly=false
19+
Resume=-1
20+
Layers=2
21+
22+
Storage=TIKVIO
23+
TiKVKeyPrefix=PLACEHOLDER
24+
25+
[SelectHead]
26+
ParallelBKTBuild=true
27+
28+
[BuildHead]
29+
ParallelBKTBuild=true
30+
31+
[BuildSSDIndex]
32+
VersionCacheTTLMs=0
33+
VersionCacheMaxChunks=10000
34+
AppendThreadNum=48
35+
AsyncMergeInSearch=true
36+
LatencyLimit=100
37+
38+
[Distributed]
39+
Enabled=true
40+
DispatcherAddr=PLACEHOLDER
41+
WorkerAddrs=PLACEHOLDER
42+
StoreAddrs=PLACEHOLDER
43+
PDAddrs=PLACEHOLDER

‎evaluation/distributed/configs/benchmark_insert_dominant_template.ini‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# insert_dominant: 1M base + 10M insert (10× scale-up) with concurrent search-during-insert.
2+
# Tests how the index handles insertion-dominated workloads where insertion volume
3+
# is much larger than the initial baseline. Layers=2, L2 distance, SIFT1B dataset.
4+
[Benchmark]
5+
WorkerTimeout=14400
6+
VectorPath=/mnt_ssd/data/sift1b/bigann_base.u8bin
7+
QueryPath=/mnt_ssd/data/sift1b/query.10K.u8bin
8+
TruthPath=/mnt_ssd/data/sift1b/bigann_truth.bin
9+
IndexPath=PLACEHOLDER
10+
ValueType=UInt8
11+
Dimension=128
12+
BaseVectorCount=1000000
13+
InsertVectorCount=10000000
14+
DeleteVectorCount=0
15+
BatchNum=10
16+
TopK=5
17+
NumSearchThreads=4
18+
NumInsertThreads=16
19+
AppendThreadNum=48
20+
NumSearchDuringInsertThreads=1
21+
NumQueries=200
22+
DistMethod=L2
23+
Rebuild=true
24+
BuildOnly=false
25+
Resume=-1
26+
Layers=2
27+
28+
Storage=TIKVIO
29+
TiKVKeyPrefix=PLACEHOLDER
30+
31+
[SelectHead]
32+
ParallelBKTBuild=true
33+
34+
[BuildHead]
35+
ParallelBKTBuild=true
36+
37+
[BuildSSDIndex]
38+
LatencyLimit=100
39+
MaxCheck=8192
40+
SearchInternalResultNum=64
41+
UseMultiChunkPosting=false
42+
ReassignK=64
43+
AsyncMergeInSearch=true
44+
VersionCacheMaxChunks=100000
45+
VersionCacheTTLMs=0
46+
47+
[Distributed]
48+
Enabled=true
49+
DispatcherAddr=PLACEHOLDER
50+
WorkerAddrs=PLACEHOLDER
51+
StoreAddrs=PLACEHOLDER
52+
PDAddrs=PLACEHOLDER

‎evaluation/distributed/configs/tikv_sdr.toml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# TiKV configuration for SPTAG distributed benchmarks
2+
# From BENCHMARK_GUIDE_SCALE_COMPUTE.md Section 4.1
3+
4+
memory-usage-limit = "80GB"
5+
6+
[server]
7+
grpc-concurrency = 8
8+
grpc-memory-pool-quota = "4GB"
9+
10+
[raftstore]
11+
region-max-size = "512MB"
12+
region-split-size = "384MB"
13+
region-max-keys = 5120000
14+
region-split-keys = 3840000
15+
16+
[rocksdb]
17+
max-background-jobs = 8
18+
max-sub-compactions = 3
19+
rate-bytes-per-sec = "0"
20+
21+
[rocksdb.defaultcf]
22+
write-buffer-size = "256MB"
23+
max-write-buffer-number = 5
24+
min-write-buffer-number-to-merge = 2
25+
level0-file-num-compaction-trigger = 8
26+
level0-slowdown-writes-trigger = 20
27+
level0-stop-writes-trigger = 36
28+
max-bytes-for-level-base = "1GB"
29+
compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"]
30+
target-file-size-base = "64MB"
31+
32+
[rocksdb.writecf]
33+
write-buffer-size = "128MB"
34+
max-write-buffer-number = 5
35+
36+
[storage]
37+
reserve-space = "1GB"
38+
39+
[storage.block-cache]
40+
capacity = "40GB"
41+
42+
[coprocessor]
43+
region-max-size = "512MB"
44+
region-split-size = "384MB"
45+
region-max-keys = 5120000
46+
region-split-keys = 3840000

‎evaluation/distributed/refresh-reference.sh‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/usr/bin/env bash
2+
# refresh-reference.sh — re-run the merge-base reference perf at <scale>
3+
# and cache the result under
4+
# <workspace-repo>/tasks/distributed-index-scale-2k/results/_reference/<SCALE>/.
5+
#
6+
# This is a DELIBERATE HUMAN ACTION. Subagents must NOT run it on their
7+
# own — the protocol caches the reference once and refreshes only when
8+
# the merge-base advances by more than a trivial commit set. Running
9+
# this rotates the perf gate's reference for every fault case.
10+
#
11+
# Usage (on sdr):
12+
# evaluation/distributed/refresh-reference.sh --scale 1m|10m
13+
# [--workspace-repo PATH]
14+
# [--cluster-conf PATH]
15+
# [--mergebase-ref REF]
16+
# [--yes]
17+
#
18+
# Defaults:
19+
# --scale (required)
20+
# --workspace-repo $HOME/workspace/openclaw-tasks
21+
# --cluster-conf evaluation/distributed/configs/sdr_single.conf
22+
# (auto-generated by run_perf_validation.sh on first run)
23+
# --mergebase-ref users/zhangt/merge-distributed-to-tikv
24+
# --yes skip the interactive confirmation
25+
#
26+
# Hard guards (same as run_perf_validation.sh): no spfresh-dev-amd-005,
27+
# no scale != {1m, 10m}.
28+
29+
set -euo pipefail
30+
31+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
32+
SPTAG_REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
33+
34+
HOSTSHORT="$(hostname -s 2>/dev/null || hostname)"
35+
if [[ "$HOSTSHORT" == *"spfresh-dev-amd-005"* || "$HOSTSHORT" == *"msccl-dev-000005"* ]]; then
36+
echo "REFUSING: never run refresh-reference on spfresh-dev-amd-005" >&2
37+
exit 10
38+
fi
39+
40+
SCALE_LOW=""
41+
WS_REPO="${HOME}/workspace/openclaw-tasks"
42+
CLUSTER_CONF="${SCRIPT_DIR}/configs/sdr_single.conf"
43+
MERGEBASE_REF="users/zhangt/merge-distributed-to-tikv"
44+
ASSUME_YES=0
45+
46+
while [[ $# -gt 0 ]]; do
47+
case "$1" in
48+
--scale) SCALE_LOW="$2"; shift 2 ;;
49+
--workspace-repo) WS_REPO="$2"; shift 2 ;;
50+
--cluster-conf) CLUSTER_CONF="$2"; shift 2 ;;
51+
--mergebase-ref) MERGEBASE_REF="$2"; shift 2 ;;
52+
--yes|-y) ASSUME_YES=1; shift ;;
53+
*) echo "unknown arg: $1" >&2; exit 2 ;;
54+
esac
55+
done
56+
57+
[[ -n "$SCALE_LOW" ]] || { echo "Usage: $0 --scale 1m|10m [--yes]" >&2; exit 2; }
58+
case "$SCALE_LOW" in 1m|10m) ;; *) echo "REFUSING: scale '$SCALE_LOW'" >&2; exit 11 ;; esac
59+
SCALE_HI="${SCALE_LOW^^}"
60+
61+
REF_DIR="$WS_REPO/tasks/distributed-index-scale-2k/results/_reference/$SCALE_HI"
62+
mkdir -p "$REF_DIR"
63+
64+
# Choose a clean worktree at MERGEBASE_REF.
65+
REF_WT="$HOME/workspace/sptag-ft/wt/_reference"
66+
if [[ ! -d "$REF_WT" ]]; then
67+
git -C "$SPTAG_REPO_ROOT" worktree add "$REF_WT" "$MERGEBASE_REF"
68+
fi
69+
git -C "$REF_WT" fetch origin "$MERGEBASE_REF" --quiet || true
70+
git -C "$REF_WT" checkout "$MERGEBASE_REF" --quiet
71+
COMMIT="$(git -C "$REF_WT" rev-parse HEAD)"
72+
73+
if [[ "$ASSUME_YES" != "1" ]]; then
74+
cat <<EOF
75+
About to refresh perf reference at $SCALE_HI.
76+
branch: $MERGEBASE_REF @ $COMMIT
77+
worktree: $REF_WT
78+
output: $REF_DIR/{metrics.json,run.log,COMMIT}
79+
This rotates the gate for every fault case. Continue? [y/N]
80+
EOF
81+
read -r ans; [[ "$ans" =~ ^[Yy] ]] || { echo "aborted"; exit 0; }
82+
fi
83+
84+
[[ -x "$REF_WT/Release/SPTAGTest" ]] || { echo "Build SPTAGTest in $REF_WT first." >&2; exit 3; }
85+
86+
# Run via the existing single-node harness in the reference worktree.
87+
pushd "$REF_WT" >/dev/null
88+
"$REF_WT/evaluation/distributed/run_distributed.sh" run "$CLUSTER_CONF" "$SCALE_LOW" 1 \
89+
2>&1 | tee "$REF_DIR/run.log"
90+
popd >/dev/null
91+
92+
EMITTED="$REF_WT/output_${SCALE_LOW}_1node.json"
93+
[[ -f "$EMITTED" ]] || { echo "ERROR: $EMITTED missing"; exit 4; }
94+
cp "$EMITTED" "$REF_DIR/metrics.json"
95+
echo "$COMMIT" > "$REF_DIR/COMMIT"
96+
date -u +%Y%m%dT%H%M%SZ > "$REF_DIR/REFRESHED_AT"
97+
echo "[refresh-reference] cached reference at $REF_DIR (commit $COMMIT)"

0 commit comments

Comments
 (0)