Skip to content

Commit 178e2f7

Browse files
huanghaoyuanhhyhuanghaoyuanhhy
committed
storage: support local filesystem storage with localPath config
Signed-off-by: huanghaoyuanhhy <haoyuan.huang@zilliz.com>
1 parent 20289ed commit 178e2f7

8 files changed

Lines changed: 390 additions & 80 deletions

File tree

‎.github/workflows/main.yaml‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -836,6 +836,185 @@ jobs:
836836
837837
838838
839+
test-backup-restore-local-storage:
840+
needs: unit-test-go
841+
name: Backup and restore with local storage
842+
runs-on: ubuntu-latest
843+
strategy:
844+
fail-fast: false
845+
matrix:
846+
image_tag: [master-latest]
847+
848+
steps:
849+
- uses: actions/checkout@v6
850+
851+
- name: Set up Python 3.10
852+
uses: actions/setup-python@v6
853+
with:
854+
python-version: '3.10'
855+
cache: pip
856+
857+
- uses: actions/setup-go@v6
858+
name: Set up Go ${{ env.go-version }}
859+
with:
860+
go-version: ${{ env.go-version }}
861+
cache: true
862+
863+
- name: Build
864+
timeout-minutes: 5
865+
shell: bash
866+
run: |
867+
go get
868+
go build
869+
870+
- name: Install dependency
871+
timeout-minutes: 5
872+
working-directory: tests
873+
shell: bash
874+
run: |
875+
pip install -r requirements.txt --trusted-host https://test.pypi.org
876+
877+
- name: Deploy Milvus with local storage
878+
timeout-minutes: 15
879+
shell: bash
880+
working-directory: deployment/standalone
881+
run: |
882+
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag
883+
mkdir -p volumes/milvus
884+
sudo chmod -R 777 volumes
885+
886+
# Create embedEtcd.yaml
887+
cat > embedEtcd.yaml <<ETCD
888+
listen-client-urls: http://0.0.0.0:2379
889+
advertise-client-urls: http://0.0.0.0:2379
890+
quota-backend-bytes: 4294967296
891+
auto-compaction-mode: revision
892+
auto-compaction-retention: '1000'
893+
ETCD
894+
895+
# Create user.yaml for custom config
896+
cat > user.yaml <<USERCFG
897+
log:
898+
level: debug
899+
dataNode:
900+
segment:
901+
insertBufSize: 4096
902+
USERCFG
903+
904+
if [ "${{ matrix.image_tag }}" == "master-latest" ]; then
905+
yq -i '.common.storage.enablev2 = true' user.yaml
906+
yq -i '.common.storage.useLoonFFI = false' user.yaml
907+
fi
908+
cat user.yaml
909+
910+
# Match official standalone docker deployment:
911+
# https://milvus.io/docs/install_standalone-docker.md
912+
sudo docker run -d \
913+
--name milvus-standalone \
914+
--security-opt seccomp:unconfined \
915+
-e ETCD_USE_EMBED=true \
916+
-e ETCD_DATA_DIR=/var/lib/milvus/etcd \
917+
-e ETCD_CONFIG_PATH=/milvus/configs/embedEtcd.yaml \
918+
-e COMMON_STORAGETYPE=local \
919+
-e DEPLOY_MODE=STANDALONE \
920+
-v $(pwd)/volumes/milvus:/var/lib/milvus \
921+
-v $(pwd)/embedEtcd.yaml:/milvus/configs/embedEtcd.yaml \
922+
-v $(pwd)/user.yaml:/milvus/configs/user.yaml \
923+
-p 19530:19530 \
924+
-p 9091:9091 \
925+
-p 2379:2379 \
926+
--health-cmd="curl -f http://localhost:9091/healthz" \
927+
--health-interval=30s \
928+
--health-start-period=90s \
929+
--health-timeout=20s \
930+
--health-retries=3 \
931+
milvusdb/milvus:${tag} \
932+
milvus run standalone
933+
934+
# Wait for healthy
935+
for i in $(seq 1 60); do
936+
status=$(sudo docker inspect --format='{{.State.Health.Status}}' milvus-standalone 2>/dev/null || echo "not ready")
937+
echo "Attempt $i: $status"
938+
if [ "$status" = "healthy" ]; then break; fi
939+
sleep 5
940+
done
941+
sudo docker ps -a
942+
943+
- name: Export container status after deploy
944+
if: ${{ always() }}
945+
shell: bash
946+
working-directory: deployment/standalone
947+
run: |
948+
echo "=== Container Status ==="
949+
sudo docker ps -a || true
950+
echo "=== Standalone Container Logs ==="
951+
sudo docker logs milvus-standalone 2>&1 | tail -100 || true
952+
953+
- name: Configure backup.yaml for local storage
954+
timeout-minutes: 1
955+
shell: bash
956+
run: |
957+
yq -i '.log.level = "debug"' configs/backup.yaml
958+
yq -i '.minio.storageType = "local"' configs/backup.yaml
959+
yq -i '.minio.localPath = "deployment/standalone/volumes/milvus/data"' configs/backup.yaml
960+
yq -i '.minio.bucketName = "a-bucket"' configs/backup.yaml
961+
yq -i '.minio.rootPath = "files"' configs/backup.yaml
962+
yq -i '.minio.backupStorageType = "local"' configs/backup.yaml
963+
yq -i '.minio.backupLocalPath = "deployment/standalone/volumes/backup"' configs/backup.yaml
964+
yq -i '.minio.backupBucketName = "milvus-backup"' configs/backup.yaml
965+
yq -i '.minio.backupRootPath = "backup"' configs/backup.yaml
966+
cat configs/backup.yaml
967+
968+
- name: Prepare data
969+
timeout-minutes: 5
970+
shell: bash
971+
run: |
972+
python example/prepare_data.py
973+
974+
- name: Backup
975+
timeout-minutes: 5
976+
shell: bash
977+
run: |
978+
./milvus-backup check
979+
./milvus-backup list
980+
./milvus-backup create -n my_backup
981+
./milvus-backup list
982+
983+
- name: Restore backup
984+
timeout-minutes: 5
985+
shell: bash
986+
run: |
987+
./milvus-backup restore -n my_backup -s _recover
988+
989+
- name: Verify data
990+
timeout-minutes: 5
991+
shell: bash
992+
run: |
993+
python example/verify_data.py
994+
995+
- name: Delete backup
996+
timeout-minutes: 5
997+
shell: bash
998+
run: |
999+
./milvus-backup delete -n my_backup
1000+
./milvus-backup list
1001+
1002+
- name: Export logs
1003+
if: ${{ always() }}
1004+
shell: bash
1005+
run: |
1006+
mkdir -p /tmp/ci_logs
1007+
sudo docker logs milvus-standalone > /tmp/ci_logs/standalone.log 2>&1 || true
1008+
1009+
- name: Upload logs
1010+
if: ${{ ! success() }}
1011+
uses: actions/upload-artifact@v7
1012+
with:
1013+
name: local-storage-logs-${{ matrix.image_tag }}
1014+
path: |
1015+
/tmp/ci_logs
1016+
./server.log
1017+
8391018
test-backup-restore-api:
8401019
name: Backup and restore api
8411020
runs-on: ubuntu-latest

‎configs/backup-local.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Example config for Milvus Standalone with local storage.
2+
#
3+
# This matches the official docker deployment:
4+
# https://milvus.io/docs/install_standalone-docker.md
5+
#
6+
# The official script runs:
7+
# docker run ... -e COMMON_STORAGETYPE=local -v $(pwd)/volumes/milvus:/var/lib/milvus ...
8+
#
9+
# On the host, Milvus data lives at:
10+
# volumes/milvus/data / <bucketName> / <rootPath> / insert_log / ...
11+
# volumes/milvus/data / a-bucket / files / insert_log / 123 / ...
12+
#
13+
# Set localPath to the HOST path that maps to Milvus localStorage.path.
14+
15+
log:
16+
level: info
17+
console: true
18+
19+
milvus:
20+
address: localhost
21+
port: 19530
22+
23+
minio:
24+
# Milvus storage — match your Milvus minio config
25+
storageType: "local"
26+
localPath: "volumes/milvus/data" # host path to Milvus localStorage.path
27+
bucketName: "a-bucket" # Milvus minio.bucketName (default)
28+
rootPath: "files" # Milvus minio.rootPath (default)
29+
30+
# Backup storage
31+
backupStorageType: "local"
32+
backupLocalPath: "volumes/backup" # separate host directory for backup data
33+
backupBucketName: "milvus-backup"
34+
backupRootPath: "backup"

‎configs/backup.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@ minio:
5858
bucketName: "a-bucket" # Milvus Bucket name in MinIO/S3, make it the same as your milvus instance
5959
rootPath: "files" # Milvus storage root path in MinIO/S3, make it the same as your milvus instance
6060

61+
# Local storage path, only used when storageType is "local".
62+
# Should match Milvus localStorage.path. Full path: localPath/bucketName/key
63+
localPath: ""
64+
6165
# Backup storage configs, the storage you want to put the backup data
6266
backupStorageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent)
6367
backupAddress: localhost # Address of MinIO/S3
@@ -72,6 +76,10 @@ minio:
7276
backupRootPath: "backup" # Rootpath to store backup data. Backup data will store to backupBucketName/backupRootPath
7377
backupUseSSL: false # Access to MinIO/S3 with SSL
7478

79+
# Backup local storage path, only used when backupStorageType is "local".
80+
# Defaults to localPath if not set. Full path: backupLocalPath/backupBucketName/key
81+
backupLocalPath: ""
82+
7583
# If you need to back up or restore data between two different storage systems, direct client-side copying is not supported.
7684
# Set this option to true to enable data transfer through Milvus Backup.
7785
# Note: This option will be automatically set to true if `minio.storageType` and `minio.backupStorageType` differ.

‎internal/cfg/cfg.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,9 @@ type MinioConfig struct {
229229
BackupUseIAM Value[bool]
230230
BackupIAMEndpoint Value[string]
231231

232+
LocalPath Value[string]
233+
BackupLocalPath Value[string]
234+
232235
CrossStorage Value[bool]
233236

234237
// MultipartCopyThresholdMiB is the file size threshold above which multipart copy is used.
@@ -272,6 +275,9 @@ func newMinioConfig() MinioConfig {
272275
BackupUseIAM: Value[bool]{Default: false, Keys: []string{"minio.backupUseIAM"}, EnvKeys: []string{"MINIO_BACKUP_USE_IAM"}},
273276
BackupIAMEndpoint: Value[string]{Default: "", Keys: []string{"minio.backupIamEndpoint"}, EnvKeys: []string{"MINIO_BACKUP_IAM_ENDPOINT"}},
274277

278+
LocalPath: Value[string]{Default: "", Keys: []string{"minio.localPath"}, EnvKeys: []string{"MINIO_LOCAL_PATH"}},
279+
BackupLocalPath: Value[string]{Default: "", Keys: []string{"minio.backupLocalPath"}, EnvKeys: []string{"MINIO_BACKUP_LOCAL_PATH"}},
280+
275281
CrossStorage: Value[bool]{Default: false, Keys: []string{"minio.crossStorage"}},
276282

277283
MultipartCopyThresholdMiB: Value[int64]{Default: 500, Keys: []string{"minio.multipartCopyThresholdMiB"}},
@@ -284,6 +290,7 @@ func (c *MinioConfig) Resolve(s *source) error {
284290
&c.StorageType, &c.Address, &c.Port, &c.Region,
285291
&c.AccessKeyID, &c.SecretAccessKey, &c.Token, &c.GcpCredentialJSON,
286292
&c.UseSSL, &c.BucketName, &c.RootPath, &c.UseIAM, &c.IAMEndpoint,
293+
&c.LocalPath,
287294
); err != nil {
288295
return err
289296
}
@@ -301,11 +308,13 @@ func (c *MinioConfig) Resolve(s *source) error {
301308
c.BackupRootPath.Default = cmp.Or(c.BackupRootPath.Default, c.RootPath.Val, "backup")
302309
c.BackupUseIAM.Default = c.BackupUseIAM.Default || c.UseIAM.Val
303310
c.BackupIAMEndpoint.Default = cmp.Or(c.BackupIAMEndpoint.Default, c.IAMEndpoint.Val)
311+
c.BackupLocalPath.Default = cmp.Or(c.BackupLocalPath.Default, c.LocalPath.Val)
304312

305313
return resolve(s,
306314
&c.BackupStorageType, &c.BackupAddress, &c.BackupPort, &c.BackupRegion,
307315
&c.BackupAccessKeyID, &c.BackupSecretAccessKey, &c.BackupToken, &c.BackupGcpCredentialJSON,
308316
&c.BackupUseSSL, &c.BackupBucketName, &c.BackupRootPath, &c.BackupUseIAM, &c.BackupIAMEndpoint,
317+
&c.BackupLocalPath,
309318
&c.CrossStorage,
310319
&c.MultipartCopyThresholdMiB,
311320
)

‎internal/storage/client.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@ type Config struct {
4848

4949
Bucket string
5050

51+
// LocalPath is the base directory for local storage, corresponding to Milvus localStorage.path.
52+
// Only used by LocalClient. The full path is: LocalPath / Bucket / key.
53+
LocalPath string
54+
5155
// MultipartCopyThresholdMiB is the file size threshold above which multipart copy is used.
5256
// Default is 500 MiB if not set. GCP does not support multipart copy.
5357
MultipartCopyThresholdMiB int64

‎internal/storage/factory.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ func BackupStorageConfig(params *cfg.MinioConfig) Config {
4545
Endpoint: ep,
4646
UseSSL: params.BackupUseSSL.Val,
4747
Bucket: params.BackupBucketName.Val,
48+
LocalPath: params.BackupLocalPath.Val,
4849
Credential: newBackupCredential(params),
4950
Region: params.BackupRegion.Val,
5051
MultipartCopyThresholdMiB: params.MultipartCopyThresholdMiB.Val,
@@ -104,6 +105,7 @@ func MilvusStorageConfig(params *cfg.MinioConfig) Config {
104105
UseSSL: params.UseSSL.Val,
105106
Credential: newMilvusCredential(params),
106107
Bucket: params.BucketName.Val,
108+
LocalPath: params.LocalPath.Val,
107109
Region: params.Region.Val,
108110
MultipartCopyThresholdMiB: params.MultipartCopyThresholdMiB.Val,
109111
}

0 commit comments

Comments
 (0)