Skip to content

Commit e6567f6

Browse files
huanghaoyuanhhyhuanghaoyuanhhy
committed
storage: support local filesystem storage with localPath config
Signed-off-by: huanghaoyuanhhy <haoyuan.huang@zilliz.com>
1 parent 20289ed commit e6567f6

9 files changed

Lines changed: 400 additions & 81 deletions

File tree

‎.github/workflows/main.yaml‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -836,6 +836,188 @@ jobs:
836836
837837
838838
839+
test-backup-restore-local-storage:
840+
needs: unit-test-go
841+
name: Backup and restore with local storage
842+
runs-on: ubuntu-latest
843+
strategy:
844+
fail-fast: false
845+
matrix:
846+
image_tag: [v2.5.20, 2.6-latest]
847+
848+
steps:
849+
- uses: actions/checkout@v6
850+
851+
- name: Set up Python 3.10
852+
uses: actions/setup-python@v6
853+
with:
854+
python-version: '3.10'
855+
cache: pip
856+
857+
- uses: actions/setup-go@v6
858+
name: Set up Go ${{ env.go-version }}
859+
with:
860+
go-version: ${{ env.go-version }}
861+
cache: true
862+
863+
- name: Build
864+
timeout-minutes: 5
865+
shell: bash
866+
run: |
867+
go get
868+
go build
869+
870+
- name: Install dependency
871+
timeout-minutes: 5
872+
working-directory: tests
873+
shell: bash
874+
run: |
875+
pip install -r requirements.txt --trusted-host https://test.pypi.org
876+
877+
- name: Deploy Milvus with local storage
878+
timeout-minutes: 15
879+
shell: bash
880+
working-directory: deployment/standalone
881+
run: |
882+
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag
883+
mkdir -p volumes/milvus
884+
sudo chmod -R 777 volumes
885+
886+
# Create embedEtcd.yaml
887+
cat > embedEtcd.yaml <<'ETCD'
888+
listen-client-urls: http://0.0.0.0:2379
889+
advertise-client-urls: http://0.0.0.0:2379
890+
quota-backend-bytes: 4294967296
891+
auto-compaction-mode: revision
892+
auto-compaction-retention: '1000'
893+
ETCD
894+
895+
# Create user.yaml for custom config
896+
cat > user.yaml <<'USERCFG'
897+
log:
898+
level: debug
899+
dataNode:
900+
segment:
901+
insertBufSize: 4096
902+
USERCFG
903+
cat user.yaml
904+
905+
# Match official standalone docker deployment:
906+
# https://milvus.io/docs/install_standalone-docker.md
907+
sudo docker run -d \
908+
--name milvus-standalone \
909+
--security-opt seccomp:unconfined \
910+
-e ETCD_USE_EMBED=true \
911+
-e ETCD_DATA_DIR=/var/lib/milvus/etcd \
912+
-e ETCD_CONFIG_PATH=/milvus/configs/embedEtcd.yaml \
913+
-e COMMON_STORAGETYPE=local \
914+
-e DEPLOY_MODE=STANDALONE \
915+
-v $(pwd)/volumes/milvus:/var/lib/milvus \
916+
-v $(pwd)/embedEtcd.yaml:/milvus/configs/embedEtcd.yaml \
917+
-v $(pwd)/user.yaml:/milvus/configs/user.yaml \
918+
-p 19530:19530 \
919+
-p 9091:9091 \
920+
-p 2379:2379 \
921+
--health-cmd="curl -f http://localhost:9091/healthz" \
922+
--health-interval=30s \
923+
--health-start-period=90s \
924+
--health-timeout=20s \
925+
--health-retries=3 \
926+
milvusdb/milvus:${tag} \
927+
milvus run standalone
928+
929+
# Wait for healthy
930+
for i in $(seq 1 60); do
931+
status=$(sudo docker inspect --format='{{.State.Health.Status}}' milvus-standalone 2>/dev/null || echo "not ready")
932+
echo "Attempt $i: $status"
933+
if [ "$status" = "healthy" ]; then break; fi
934+
sleep 5
935+
done
936+
sudo docker ps -a
937+
# Fix permissions on Milvus data dir created by container root
938+
sudo chmod -R 777 volumes
939+
940+
- name: Export container status after deploy
941+
if: ${{ always() }}
942+
shell: bash
943+
working-directory: deployment/standalone
944+
run: |
945+
echo "=== Container Status ==="
946+
sudo docker ps -a || true
947+
echo "=== Standalone Container Logs ==="
948+
sudo docker logs milvus-standalone 2>&1 | tail -100 || true
949+
950+
- name: Configure backup.yaml for local storage
951+
timeout-minutes: 1
952+
shell: bash
953+
run: |
954+
yq -i '.log.level = "debug"' configs/backup.yaml
955+
yq -i '.minio.storageType = "local"' configs/backup.yaml
956+
yq -i '.minio.localPath = "deployment/standalone/volumes/milvus/data"' configs/backup.yaml
957+
yq -i '.minio.bucketName = ""' configs/backup.yaml
958+
yq -i '.minio.rootPath = "files"' configs/backup.yaml
959+
yq -i '.minio.backupStorageType = "local"' configs/backup.yaml
960+
yq -i '.minio.backupLocalPath = "deployment/standalone/volumes/backup"' configs/backup.yaml
961+
yq -i '.minio.backupBucketName = ""' configs/backup.yaml
962+
yq -i '.minio.backupRootPath = "backup"' configs/backup.yaml
963+
cat configs/backup.yaml
964+
965+
- name: Prepare data
966+
timeout-minutes: 5
967+
shell: bash
968+
run: |
969+
python example/prepare_data.py
970+
971+
- name: Fix permissions after data preparation
972+
shell: bash
973+
working-directory: deployment/standalone
974+
run: |
975+
sudo chmod -R 777 volumes
976+
977+
- name: Backup
978+
timeout-minutes: 5
979+
shell: bash
980+
run: |
981+
./milvus-backup check
982+
./milvus-backup list
983+
./milvus-backup create -n my_backup
984+
./milvus-backup list
985+
986+
- name: Restore backup
987+
timeout-minutes: 5
988+
shell: bash
989+
run: |
990+
./milvus-backup restore -n my_backup -s _recover
991+
992+
- name: Verify data
993+
timeout-minutes: 5
994+
shell: bash
995+
run: |
996+
python example/verify_data.py
997+
998+
- name: Delete backup
999+
timeout-minutes: 5
1000+
shell: bash
1001+
run: |
1002+
./milvus-backup delete -n my_backup
1003+
./milvus-backup list
1004+
1005+
- name: Export logs
1006+
if: ${{ always() }}
1007+
shell: bash
1008+
run: |
1009+
mkdir -p /tmp/ci_logs
1010+
sudo docker logs milvus-standalone > /tmp/ci_logs/standalone.log 2>&1 || true
1011+
1012+
- name: Upload logs
1013+
if: ${{ ! success() }}
1014+
uses: actions/upload-artifact@v7
1015+
with:
1016+
name: local-storage-logs-${{ matrix.image_tag }}
1017+
path: |
1018+
/tmp/ci_logs
1019+
./server.log
1020+
8391021
test-backup-restore-api:
8401022
name: Backup and restore api
8411023
runs-on: ubuntu-latest

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,7 @@ dist/
1111
.DS_Store
1212

1313
# Claude Code
14-
CLAUDE.local.md
14+
CLAUDE.local.md
15+
16+
# Git worktrees
17+
.worktrees/

‎configs/backup-local.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Example config for Milvus Standalone with local storage.
2+
#
3+
# This matches the official docker deployment:
4+
# https://milvus.io/docs/install_standalone-docker.md
5+
#
6+
# The official script runs:
7+
# docker run ... -e COMMON_STORAGETYPE=local -v $(pwd)/volumes/milvus:/var/lib/milvus ...
8+
#
9+
# In local storage mode, Milvus writes data under localStorage.path with rootPath prefix
10+
# but without bucket prefix:
11+
# volumes/milvus/data / <rootPath> / insert_log / <collID> / ...
12+
# volumes/milvus/data / files / insert_log / 123 / ...
13+
#
14+
# Set localPath to the HOST path that maps to Milvus localStorage.path.
15+
# Set bucketName to empty (local storage does not use bucket).
16+
# Set rootPath to match Milvus minio.rootPath (default: "files").
17+
18+
log:
19+
level: info
20+
console: true
21+
22+
milvus:
23+
address: localhost
24+
port: 19530
25+
26+
minio:
27+
# Milvus storage — match your Milvus minio config
28+
storageType: "local"
29+
localPath: "volumes/milvus/data" # host path to Milvus localStorage.path
30+
bucketName: "" # not used in local storage mode
31+
rootPath: "files" # Milvus minio.rootPath (default)
32+
33+
# Backup storage
34+
backupStorageType: "local"
35+
backupLocalPath: "volumes/backup" # separate host directory for backup data
36+
backupBucketName: ""
37+
backupRootPath: "backup"

‎configs/backup.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@ minio:
5858
bucketName: "a-bucket" # Milvus Bucket name in MinIO/S3, make it the same as your milvus instance
5959
rootPath: "files" # Milvus storage root path in MinIO/S3, make it the same as your milvus instance
6060

61+
# Local storage path, only used when storageType is "local".
62+
# Should match Milvus localStorage.path. Full path: localPath/bucketName/key
63+
localPath: ""
64+
6165
# Backup storage configs, the storage you want to put the backup data
6266
backupStorageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent)
6367
backupAddress: localhost # Address of MinIO/S3
@@ -72,6 +76,10 @@ minio:
7276
backupRootPath: "backup" # Rootpath to store backup data. Backup data will store to backupBucketName/backupRootPath
7377
backupUseSSL: false # Access to MinIO/S3 with SSL
7478

79+
# Backup local storage path, only used when backupStorageType is "local".
80+
# Defaults to localPath if not set. Full path: backupLocalPath/backupBucketName/key
81+
backupLocalPath: ""
82+
7583
# If you need to back up or restore data between two different storage systems, direct client-side copying is not supported.
7684
# Set this option to true to enable data transfer through Milvus Backup.
7785
# Note: This option will be automatically set to true if `minio.storageType` and `minio.backupStorageType` differ.

‎internal/cfg/cfg.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,9 @@ type MinioConfig struct {
229229
BackupUseIAM Value[bool]
230230
BackupIAMEndpoint Value[string]
231231

232+
LocalPath Value[string]
233+
BackupLocalPath Value[string]
234+
232235
CrossStorage Value[bool]
233236

234237
// MultipartCopyThresholdMiB is the file size threshold above which multipart copy is used.
@@ -272,6 +275,9 @@ func newMinioConfig() MinioConfig {
272275
BackupUseIAM: Value[bool]{Default: false, Keys: []string{"minio.backupUseIAM"}, EnvKeys: []string{"MINIO_BACKUP_USE_IAM"}},
273276
BackupIAMEndpoint: Value[string]{Default: "", Keys: []string{"minio.backupIamEndpoint"}, EnvKeys: []string{"MINIO_BACKUP_IAM_ENDPOINT"}},
274277

278+
LocalPath: Value[string]{Default: "", Keys: []string{"minio.localPath"}, EnvKeys: []string{"MINIO_LOCAL_PATH"}},
279+
BackupLocalPath: Value[string]{Default: "", Keys: []string{"minio.backupLocalPath"}, EnvKeys: []string{"MINIO_BACKUP_LOCAL_PATH"}},
280+
275281
CrossStorage: Value[bool]{Default: false, Keys: []string{"minio.crossStorage"}},
276282

277283
MultipartCopyThresholdMiB: Value[int64]{Default: 500, Keys: []string{"minio.multipartCopyThresholdMiB"}},
@@ -284,6 +290,7 @@ func (c *MinioConfig) Resolve(s *source) error {
284290
&c.StorageType, &c.Address, &c.Port, &c.Region,
285291
&c.AccessKeyID, &c.SecretAccessKey, &c.Token, &c.GcpCredentialJSON,
286292
&c.UseSSL, &c.BucketName, &c.RootPath, &c.UseIAM, &c.IAMEndpoint,
293+
&c.LocalPath,
287294
); err != nil {
288295
return err
289296
}
@@ -301,11 +308,13 @@ func (c *MinioConfig) Resolve(s *source) error {
301308
c.BackupRootPath.Default = cmp.Or(c.BackupRootPath.Default, c.RootPath.Val, "backup")
302309
c.BackupUseIAM.Default = c.BackupUseIAM.Default || c.UseIAM.Val
303310
c.BackupIAMEndpoint.Default = cmp.Or(c.BackupIAMEndpoint.Default, c.IAMEndpoint.Val)
311+
c.BackupLocalPath.Default = cmp.Or(c.BackupLocalPath.Default, c.LocalPath.Val)
304312

305313
return resolve(s,
306314
&c.BackupStorageType, &c.BackupAddress, &c.BackupPort, &c.BackupRegion,
307315
&c.BackupAccessKeyID, &c.BackupSecretAccessKey, &c.BackupToken, &c.BackupGcpCredentialJSON,
308316
&c.BackupUseSSL, &c.BackupBucketName, &c.BackupRootPath, &c.BackupUseIAM, &c.BackupIAMEndpoint,
317+
&c.BackupLocalPath,
309318
&c.CrossStorage,
310319
&c.MultipartCopyThresholdMiB,
311320
)

‎internal/storage/client.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@ type Config struct {
4848

4949
Bucket string
5050

51+
// LocalPath is the base directory for local storage, corresponding to Milvus localStorage.path.
52+
// Only used by LocalClient. The full path is: LocalPath / Bucket / key.
53+
LocalPath string
54+
5155
// MultipartCopyThresholdMiB is the file size threshold above which multipart copy is used.
5256
// Default is 500 MiB if not set. GCP does not support multipart copy.
5357
MultipartCopyThresholdMiB int64

‎internal/storage/factory.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ func BackupStorageConfig(params *cfg.MinioConfig) Config {
4545
Endpoint: ep,
4646
UseSSL: params.BackupUseSSL.Val,
4747
Bucket: params.BackupBucketName.Val,
48+
LocalPath: params.BackupLocalPath.Val,
4849
Credential: newBackupCredential(params),
4950
Region: params.BackupRegion.Val,
5051
MultipartCopyThresholdMiB: params.MultipartCopyThresholdMiB.Val,
@@ -104,6 +105,7 @@ func MilvusStorageConfig(params *cfg.MinioConfig) Config {
104105
UseSSL: params.UseSSL.Val,
105106
Credential: newMilvusCredential(params),
106107
Bucket: params.BucketName.Val,
108+
LocalPath: params.LocalPath.Val,
107109
Region: params.Region.Val,
108110
MultipartCopyThresholdMiB: params.MultipartCopyThresholdMiB.Val,
109111
}

0 commit comments

Comments
 (0)