Skip to content

Commit 91a7ce9

Browse files
huanghaoyuanhhyhuanghaoyuanhhy
committed
storage: support local filesystem storage with localPath config
Support Milvus standalone with local storage (COMMON_STORAGETYPE=local) by mapping minio.localPath / minio.backupLocalPath to the host directory that backs Milvus localStorage.path. LocalClient now resolves keys against baseDir = localPath/bucket so all keys returned by ListPrefix are relative, matching the S3 client. The CI test dumps the milvus and backup volume trees around backup and restore, and keeps restore temp files, so failures expose what is actually on disk where Milvus expects to read it. Signed-off-by: huanghaoyuanhhy <haoyuan.huang@zilliz.com>
1 parent adae7f7 commit 91a7ce9

9 files changed

Lines changed: 428 additions & 81 deletions

File tree

‎.github/workflows/main.yaml‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -972,6 +972,215 @@ jobs:
972972
973973
974974
975+
test-backup-restore-local-storage:
976+
needs: unit-test-go
977+
name: Backup and restore with local storage
978+
runs-on: ubuntu-latest
979+
strategy:
980+
fail-fast: false
981+
matrix:
982+
image_tag: [v2.5.20, 2.6-latest]
983+
984+
steps:
985+
- uses: actions/checkout@v6
986+
987+
- name: Set up Python 3.10
988+
uses: actions/setup-python@v6
989+
with:
990+
python-version: '3.10'
991+
cache: pip
992+
993+
- uses: actions/setup-go@v6
994+
name: Set up Go ${{ env.go-version }}
995+
with:
996+
go-version: ${{ env.go-version }}
997+
cache: true
998+
999+
- name: Build
1000+
timeout-minutes: 5
1001+
shell: bash
1002+
run: |
1003+
go get
1004+
go build
1005+
1006+
- name: Install dependency
1007+
timeout-minutes: 5
1008+
working-directory: tests
1009+
shell: bash
1010+
run: |
1011+
pip install -r requirements.txt --trusted-host https://test.pypi.org
1012+
1013+
- name: Deploy Milvus with local storage
1014+
timeout-minutes: 15
1015+
shell: bash
1016+
working-directory: deployment/standalone
1017+
run: |
1018+
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag
1019+
mkdir -p volumes/milvus
1020+
sudo chmod -R 777 volumes
1021+
1022+
# Create embedEtcd.yaml
1023+
cat > embedEtcd.yaml <<'ETCD'
1024+
listen-client-urls: http://0.0.0.0:2379
1025+
advertise-client-urls: http://0.0.0.0:2379
1026+
quota-backend-bytes: 4294967296
1027+
auto-compaction-mode: revision
1028+
auto-compaction-retention: '1000'
1029+
ETCD
1030+
1031+
# Create user.yaml for custom config
1032+
cat > user.yaml <<'USERCFG'
1033+
log:
1034+
level: debug
1035+
dataNode:
1036+
segment:
1037+
insertBufSize: 4096
1038+
USERCFG
1039+
cat user.yaml
1040+
1041+
# Match official standalone docker deployment:
1042+
# https://milvus.io/docs/install_standalone-docker.md
1043+
sudo docker run -d \
1044+
--name milvus-standalone \
1045+
--security-opt seccomp:unconfined \
1046+
-e ETCD_USE_EMBED=true \
1047+
-e ETCD_DATA_DIR=/var/lib/milvus/etcd \
1048+
-e ETCD_CONFIG_PATH=/milvus/configs/embedEtcd.yaml \
1049+
-e COMMON_STORAGETYPE=local \
1050+
-e DEPLOY_MODE=STANDALONE \
1051+
-v $(pwd)/volumes/milvus:/var/lib/milvus \
1052+
-v $(pwd)/embedEtcd.yaml:/milvus/configs/embedEtcd.yaml \
1053+
-v $(pwd)/user.yaml:/milvus/configs/user.yaml \
1054+
-p 19530:19530 \
1055+
-p 9091:9091 \
1056+
-p 2379:2379 \
1057+
--health-cmd="curl -f http://localhost:9091/healthz" \
1058+
--health-interval=30s \
1059+
--health-start-period=90s \
1060+
--health-timeout=20s \
1061+
--health-retries=3 \
1062+
milvusdb/milvus:${tag} \
1063+
milvus run standalone
1064+
1065+
# Wait for healthy
1066+
for i in $(seq 1 60); do
1067+
status=$(sudo docker inspect --format='{{.State.Health.Status}}' milvus-standalone 2>/dev/null || echo "not ready")
1068+
echo "Attempt $i: $status"
1069+
if [ "$status" = "healthy" ]; then break; fi
1070+
sleep 5
1071+
done
1072+
sudo docker ps -a
1073+
# Fix permissions on Milvus data dir created by container root
1074+
sudo chmod -R 777 volumes
1075+
1076+
- name: Export container status after deploy
1077+
if: ${{ always() }}
1078+
shell: bash
1079+
working-directory: deployment/standalone
1080+
run: |
1081+
echo "=== Container Status ==="
1082+
sudo docker ps -a || true
1083+
echo "=== Standalone Container Logs ==="
1084+
sudo docker logs milvus-standalone 2>&1 | tail -100 || true
1085+
1086+
- name: Configure backup.yaml for local storage
1087+
timeout-minutes: 1
1088+
shell: bash
1089+
run: |
1090+
yq -i '.log.level = "debug"' configs/backup.yaml
1091+
yq -i '.minio.storageType = "local"' configs/backup.yaml
1092+
yq -i '.minio.localPath = "deployment/standalone/volumes/milvus/data"' configs/backup.yaml
1093+
yq -i '.minio.bucketName = ""' configs/backup.yaml
1094+
yq -i '.minio.rootPath = ""' configs/backup.yaml
1095+
yq -i '.minio.backupStorageType = "local"' configs/backup.yaml
1096+
yq -i '.minio.backupLocalPath = "deployment/standalone/volumes/backup"' configs/backup.yaml
1097+
yq -i '.minio.backupBucketName = ""' configs/backup.yaml
1098+
yq -i '.minio.backupRootPath = "backup"' configs/backup.yaml
1099+
yq -i '.minio.crossStorage = true' configs/backup.yaml
1100+
# Keep restore temp files so we can inspect them if bulk insert fails.
1101+
yq -i '.backup.keepTempFiles = true' configs/backup.yaml
1102+
cat configs/backup.yaml
1103+
1104+
- name: Prepare data
1105+
timeout-minutes: 5
1106+
shell: bash
1107+
run: |
1108+
python example/prepare_data.py
1109+
1110+
- name: Fix permissions after data preparation
1111+
shell: bash
1112+
working-directory: deployment/standalone
1113+
run: |
1114+
sudo chmod -R 777 volumes
1115+
1116+
- name: List milvus volume before backup
1117+
shell: bash
1118+
working-directory: deployment/standalone
1119+
run: |
1120+
sudo apt-get install -y tree > /dev/null 2>&1 || true
1121+
echo "=== milvus volume tree (before backup) ==="
1122+
sudo tree volumes/milvus/data | head -500
1123+
1124+
- name: Backup
1125+
timeout-minutes: 5
1126+
shell: bash
1127+
run: |
1128+
./milvus-backup check
1129+
./milvus-backup list
1130+
./milvus-backup create -n my_backup
1131+
./milvus-backup list
1132+
1133+
- name: List backup volume after backup
1134+
if: ${{ always() }}
1135+
shell: bash
1136+
working-directory: deployment/standalone
1137+
run: |
1138+
echo "=== backup volume tree (after backup) ==="
1139+
sudo tree volumes/backup | head -500
1140+
1141+
- name: Restore backup
1142+
timeout-minutes: 5
1143+
shell: bash
1144+
run: |
1145+
./milvus-backup restore -n my_backup -s _recover
1146+
1147+
- name: List milvus volume after restore
1148+
if: ${{ always() }}
1149+
shell: bash
1150+
working-directory: deployment/standalone
1151+
run: |
1152+
echo "=== milvus volume tree (after restore) ==="
1153+
sudo tree volumes/milvus/data | head -500
1154+
1155+
- name: Verify data
1156+
timeout-minutes: 5
1157+
shell: bash
1158+
run: |
1159+
python example/verify_data.py
1160+
1161+
- name: Delete backup
1162+
timeout-minutes: 5
1163+
shell: bash
1164+
run: |
1165+
./milvus-backup delete -n my_backup
1166+
./milvus-backup list
1167+
1168+
- name: Export logs
1169+
if: ${{ always() }}
1170+
shell: bash
1171+
run: |
1172+
mkdir -p /tmp/ci_logs
1173+
sudo docker logs milvus-standalone > /tmp/ci_logs/standalone.log 2>&1 || true
1174+
1175+
- name: Upload logs
1176+
if: ${{ ! success() }}
1177+
uses: actions/upload-artifact@v7
1178+
with:
1179+
name: local-storage-logs-${{ matrix.image_tag }}
1180+
path: |
1181+
/tmp/ci_logs
1182+
./server.log
1183+
9751184
test-backup-restore-api:
9761185
name: Backup and restore api
9771186
runs-on: ubuntu-latest

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,7 @@ dist/
1111
.DS_Store
1212

1313
# Claude Code
14-
CLAUDE.local.md
14+
CLAUDE.local.md
15+
16+
# Git worktrees
17+
.worktrees/

‎configs/backup-local.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Example config for Milvus Standalone with local storage.
2+
#
3+
# This matches the official docker deployment:
4+
# https://milvus.io/docs/install_standalone-docker.md
5+
#
6+
# The official script runs:
7+
# docker run ... -e COMMON_STORAGETYPE=local -v $(pwd)/volumes/milvus:/var/lib/milvus ...
8+
#
9+
# In local storage mode, Milvus uses localStorage.path as the chunk manager root.
10+
# Unlike remote mode, minio.rootPath ("files") is NOT used as a path prefix.
11+
# Files are stored directly under localStorage.path:
12+
# /var/lib/milvus/data / insert_log / <collID> / ...
13+
#
14+
# Set localPath to the HOST path that maps to Milvus localStorage.path.
15+
# Set bucketName and rootPath to empty (not used in local storage mode).
16+
17+
log:
18+
level: info
19+
console: true
20+
21+
milvus:
22+
address: localhost
23+
port: 19530
24+
25+
minio:
26+
# Milvus storage — match your Milvus local storage config
27+
storageType: "local"
28+
localPath: "volumes/milvus/data" # host path to Milvus localStorage.path
29+
bucketName: "" # not used in local storage mode
30+
rootPath: "" # not used in local storage mode
31+
32+
# Backup storage
33+
backupStorageType: "local"
34+
backupLocalPath: "volumes/backup" # separate host directory for backup data
35+
backupBucketName: ""
36+
backupRootPath: "backup"

‎configs/backup.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,12 @@ minio:
5858
bucketName: "a-bucket" # Milvus Bucket name in MinIO/S3, make it the same as your milvus instance
5959
rootPath: "files" # Milvus storage root path in MinIO/S3, make it the same as your milvus instance
6060

61+
# Local storage path, only used when storageType is "local".
62+
# Should match Milvus localStorage.path. Full path: localPath/bucketName/key.
63+
# In local mode, set bucketName and rootPath to empty since Milvus local storage
64+
# does not use them — files are stored directly under localStorage.path.
65+
localPath: ""
66+
6167
# Backup storage configs, the storage you want to put the backup data
6268
backupStorageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent)
6369
backupAddress: localhost # Address of MinIO/S3
@@ -72,6 +78,10 @@ minio:
7278
backupRootPath: "backup" # Rootpath to store backup data. Backup data will store to backupBucketName/backupRootPath
7379
backupUseSSL: false # Access to MinIO/S3 with SSL
7480

81+
# Backup local storage path, only used when backupStorageType is "local".
82+
# Defaults to localPath if not set. Full path: backupLocalPath/backupBucketName/key
83+
backupLocalPath: ""
84+
7585
# If you need to back up or restore data between two different storage systems, direct client-side copying is not supported.
7686
# Set this option to true to enable data transfer through Milvus Backup.
7787
# Note: This option will be automatically set to true if `minio.storageType` and `minio.backupStorageType` differ.

‎internal/cfg/cfg.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,9 @@ type MinioConfig struct {
231231
BackupUseIAM Value[bool]
232232
BackupIAMEndpoint Value[string]
233233

234+
LocalPath Value[string]
235+
BackupLocalPath Value[string]
236+
234237
CrossStorage Value[bool]
235238

236239
// MultipartCopyThresholdMiB is the file size threshold above which multipart copy is used.
@@ -274,6 +277,9 @@ func newMinioConfig() MinioConfig {
274277
BackupUseIAM: Value[bool]{Default: false, Keys: []string{"minio.backupUseIAM"}, EnvKeys: []string{"MINIO_BACKUP_USE_IAM"}},
275278
BackupIAMEndpoint: Value[string]{Default: "", Keys: []string{"minio.backupIamEndpoint"}, EnvKeys: []string{"MINIO_BACKUP_IAM_ENDPOINT"}},
276279

280+
LocalPath: Value[string]{Default: "", Keys: []string{"minio.localPath"}, EnvKeys: []string{"MINIO_LOCAL_PATH"}},
281+
BackupLocalPath: Value[string]{Default: "", Keys: []string{"minio.backupLocalPath"}, EnvKeys: []string{"MINIO_BACKUP_LOCAL_PATH"}},
282+
277283
CrossStorage: Value[bool]{Default: false, Keys: []string{"minio.crossStorage"}},
278284

279285
MultipartCopyThresholdMiB: Value[int64]{Default: 500, Keys: []string{"minio.multipartCopyThresholdMiB"}},
@@ -286,6 +292,7 @@ func (c *MinioConfig) Resolve(s *source) error {
286292
&c.StorageType, &c.Address, &c.Port, &c.Region,
287293
&c.AccessKeyID, &c.SecretAccessKey, &c.Token, &c.GcpCredentialJSON,
288294
&c.UseSSL, &c.BucketName, &c.RootPath, &c.UseIAM, &c.IAMEndpoint,
295+
&c.LocalPath,
289296
); err != nil {
290297
return err
291298
}
@@ -303,11 +310,13 @@ func (c *MinioConfig) Resolve(s *source) error {
303310
c.BackupRootPath.Default = cmp.Or(c.BackupRootPath.Default, c.RootPath.Val, "backup")
304311
c.BackupUseIAM.Default = c.BackupUseIAM.Default || c.UseIAM.Val
305312
c.BackupIAMEndpoint.Default = cmp.Or(c.BackupIAMEndpoint.Default, c.IAMEndpoint.Val)
313+
c.BackupLocalPath.Default = cmp.Or(c.BackupLocalPath.Default, c.LocalPath.Val)
306314

307315
return resolve(s,
308316
&c.BackupStorageType, &c.BackupAddress, &c.BackupPort, &c.BackupRegion,
309317
&c.BackupAccessKeyID, &c.BackupSecretAccessKey, &c.BackupToken, &c.BackupGcpCredentialJSON,
310318
&c.BackupUseSSL, &c.BackupBucketName, &c.BackupRootPath, &c.BackupUseIAM, &c.BackupIAMEndpoint,
319+
&c.BackupLocalPath,
311320
&c.CrossStorage,
312321
&c.MultipartCopyThresholdMiB,
313322
)

‎internal/storage/client.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@ type Config struct {
4848

4949
Bucket string
5050

51+
// LocalPath is the base directory for local storage, corresponding to Milvus localStorage.path.
52+
// Only used by LocalClient. The full path is: LocalPath / Bucket / key.
53+
LocalPath string
54+
5155
// MultipartCopyThresholdMiB is the file size threshold above which multipart copy is used.
5256
// Default is 500 MiB if not set. GCP does not support multipart copy.
5357
MultipartCopyThresholdMiB int64

‎internal/storage/factory.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ func BackupStorageConfig(params *cfg.MinioConfig) Config {
4545
Endpoint: ep,
4646
UseSSL: params.BackupUseSSL.Val,
4747
Bucket: params.BackupBucketName.Val,
48+
LocalPath: params.BackupLocalPath.Val,
4849
Credential: newBackupCredential(params),
4950
Region: params.BackupRegion.Val,
5051
MultipartCopyThresholdMiB: params.MultipartCopyThresholdMiB.Val,
@@ -104,6 +105,7 @@ func MilvusStorageConfig(params *cfg.MinioConfig) Config {
104105
UseSSL: params.UseSSL.Val,
105106
Credential: newMilvusCredential(params),
106107
Bucket: params.BucketName.Val,
108+
LocalPath: params.LocalPath.Val,
107109
Region: params.Region.Val,
108110
MultipartCopyThresholdMiB: params.MultipartCopyThresholdMiB.Val,
109111
}

0 commit comments

Comments
 (0)