Skip to content

Commit 741972a

Browse files
lentitude2tklentitude2tk
authored
bulkImport add objectUrls paramster (#1525)
Signed-off-by: lentitude2tk <xushuang.hu@zilliz.com>
1 parent a7796a1 commit 741972a

5 files changed

Lines changed: 142 additions & 10 deletions

File tree

‎examples/src/main/java/io/milvus/v1/BulkWriterExample.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575
import java.io.IOException;
7676
import java.net.URL;
7777
import java.util.ArrayList;
78+
import java.util.Collections;
7879
import java.util.Iterator;
7980
import java.util.List;
8081
import java.util.Map;
@@ -547,8 +548,9 @@ private void callCloudImport(List<List<String>> batchFiles, String collectionNam
547548
String secretKey = StorageConsts.cloudStorage == CloudStorage.AZURE ? StorageConsts.AZURE_ACCOUNT_KEY : StorageConsts.STORAGE_SECRET_KEY;
548549

549550
System.out.println("\n===================== call cloudImport ====================");
551+
List<String> objectUrls = Lists.newArrayList(objectUrl);
550552
CloudImportRequest bulkImportRequest = CloudImportRequest.builder()
551-
.objectUrl(objectUrl).accessKey(accessKey).secretKey(secretKey)
553+
.objectUrls(Lists.newArrayList(Collections.singleton(objectUrls))).accessKey(accessKey).secretKey(secretKey)
552554
.clusterId(CloudImportConsts.CLUSTER_ID).collectionName(collectionName).partitionName(partitionName)
553555
.apiKey(CloudImportConsts.API_KEY)
554556
.build();
@@ -710,8 +712,9 @@ private Long getCollectionStatistics() {
710712

711713
private static void exampleCloudImport() {
712714
System.out.println("\n===================== import files to cloud vectordb ====================");
715+
List<String> objectUrls = Lists.newArrayList(CloudImportConsts.OBJECT_URL);
713716
CloudImportRequest request = CloudImportRequest.builder()
714-
.objectUrl(CloudImportConsts.OBJECT_URL).accessKey(CloudImportConsts.OBJECT_ACCESS_KEY).secretKey(CloudImportConsts.OBJECT_SECRET_KEY)
717+
.objectUrls(Lists.newArrayList(Collections.singleton(objectUrls))).accessKey(CloudImportConsts.OBJECT_ACCESS_KEY).secretKey(CloudImportConsts.OBJECT_SECRET_KEY)
715718
.clusterId(CloudImportConsts.CLUSTER_ID).collectionName(CloudImportConsts.COLLECTION_NAME).partitionName(CloudImportConsts.PARTITION_NAME)
716719
.apiKey(CloudImportConsts.API_KEY)
717720
.build();

‎examples/src/main/java/io/milvus/v2/bulkwriter/BulkWriterRemoteExample.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -688,8 +688,9 @@ private static Long getCollectionRowCount() {
688688

689689
private static void exampleCloudImport() {
690690
System.out.println("\n===================== import files to cloud vectordb ====================");
691+
List<String> objectUrls = Lists.newArrayList(CloudImportConsts.OBJECT_URL);
691692
CloudImportRequest request = CloudImportRequest.builder()
692-
.objectUrl(CloudImportConsts.OBJECT_URL).accessKey(CloudImportConsts.OBJECT_ACCESS_KEY).secretKey(CloudImportConsts.OBJECT_SECRET_KEY)
693+
.objectUrls(Lists.newArrayList(Collections.singleton(objectUrls))).accessKey(CloudImportConsts.OBJECT_ACCESS_KEY).secretKey(CloudImportConsts.OBJECT_SECRET_KEY)
693694
.clusterId(CloudImportConsts.CLUSTER_ID).collectionName(CloudImportConsts.COLLECTION_NAME).partitionName(CloudImportConsts.PARTITION_NAME)
694695
.apiKey(CloudImportConsts.API_KEY)
695696
.build();

‎sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/request/import_/CloudImportRequest.java‎

Lines changed: 62 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,76 @@
2424
import lombok.NoArgsConstructor;
2525
import lombok.experimental.SuperBuilder;
2626

27+
import java.util.List;
28+
2729
@Data
2830
@SuperBuilder
2931
@AllArgsConstructor
3032
@NoArgsConstructor
33+
/*
34+
If you want to import data into a Zilliz cloud instance and your data is stored in a storage bucket,
35+
you can use this method to import the data from the bucket.
36+
*/
3137
public class CloudImportRequest extends BaseImportRequest {
3238
private static final long serialVersionUID = 6487348610099924813L;
33-
private String objectUrl;
34-
private String accessKey;
35-
private String secretKey;
36-
private String token;
3739
private String clusterId;
40+
41+
/**
42+
* For Free & Serverless deployments: specifying this parameter is not supported.
43+
* For Dedicated deployments: this parameter can be specified; defaults to the "default" database.
44+
*/
3845
private String dbName;
3946
private String collectionName;
47+
48+
/**
49+
* If the collection has partitionKey enabled:
50+
* - The partitionName parameter cannot be specified for import.
51+
* If the collection does not have partitionKey enabled:
52+
* - You may specify partitionName for the import.
53+
* - Defaults to the "default" partition if not specified.
54+
*/
4055
private String partitionName;
56+
57+
/**
58+
* Data import can be configured in multiple ways using `objectUrls`:
59+
* <p>
60+
* 1. Multi-path import (multiple folders or files):
61+
* "objectUrls": [
62+
* ["s3://bucket-name/parquet-folder-1/1.parquet"],
63+
* ["s3://bucket-name/parquet-folder-2/1.parquet"],
64+
* ["s3://bucket-name/parquet-folder-3/"]
65+
* ]
66+
* <p>
67+
* 2. Folder import:
68+
* "objectUrls": [
69+
* ["s3://bucket-name/parquet-folder/"]
70+
* ]
71+
* <p>
72+
* 3. Single file import:
73+
* "objectUrls": [
74+
* ["s3://bucket-name/parquet-folder/1.parquet"]
75+
* ]
76+
*/
77+
private List<List<String>> objectUrls;
78+
79+
/**
80+
* Use `objectUrls` instead for more flexible multi-path configuration.
81+
* <p>
82+
* Folder import:
83+
* "objectUrl": "s3://bucket-name/parquet-folder/"
84+
* <p>
85+
* File import:
86+
* "objectUrl": "s3://bucket-name/parquet-folder/1.parquet"
87+
*/
88+
@Deprecated
89+
private String objectUrl;
90+
91+
/** Specify `accessKey` and `secretKey`; for short-term credentials, also include `token`. */
92+
private String accessKey;
93+
94+
/** Specify `accessKey` and `secretKey`; for short-term credentials, also include `token`. */
95+
private String secretKey;
96+
97+
/** Specify `accessKey` and `secretKey`; for short-term credentials, also include `token`. */
98+
private String token;
4199
}

‎sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/request/import_/MilvusImportRequest.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,42 @@
3030
@SuperBuilder
3131
@AllArgsConstructor
3232
@NoArgsConstructor
33+
/*
34+
If you want to import data into open-source Milvus,
35+
you can use this method to import the data files stored in the bucket where Milvus resides.
36+
*/
3337
public class MilvusImportRequest extends BaseImportRequest {
3438
private static final long serialVersionUID = -1958858397962018740L;
39+
/**
40+
* This parameter can be specified; defaults to the "default" database.
41+
*/
3542
private String dbName;
43+
3644
private String collectionName;
45+
46+
/**
47+
* If the collection has partitionKey enabled:
48+
* - The partitionName parameter cannot be specified for import.
49+
* If the collection does not have partitionKey enabled:
50+
* - You may specify partitionName for the import.
51+
* - Defaults to the "default" partition if not specified.
52+
*/
3753
private String partitionName;
54+
55+
/**
56+
* Data import can be configured in multiple ways using `files`:
57+
* <p>
58+
* 1. Multi-path import (multiple files):
59+
* "files": [
60+
* ["parquet-folder-1/1.parquet"],
61+
* ["parquet-folder-2/1.parquet"],
62+
* ["parquet-folder-3/1.parquet"]
63+
* ]
64+
* <p>
65+
* 2. Single file import:
66+
* "files": [
67+
* ["parquet-folder/1.parquet"]
68+
* ]
69+
*/
3870
private List<List<String>> files;
3971
}

‎sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/request/import_/StageImportRequest.java‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,50 @@
3030
@SuperBuilder
3131
@AllArgsConstructor
3232
@NoArgsConstructor
33+
/*
34+
If you want to import data into a Zilliz cloud instance and your data is stored in a Zilliz stage,
35+
you can use this method to import the data from the stage.
36+
*/
3337
public class StageImportRequest extends BaseImportRequest {
34-
private String stageName;
35-
private List<List<String>> dataPaths;
36-
3738
private String clusterId;
39+
40+
/**
41+
* For Free & Serverless deployments: specifying this parameter is not supported.
42+
* For Dedicated deployments: this parameter can be specified; defaults to the "default" database.
43+
*/
3844
private String dbName;
3945
private String collectionName;
46+
47+
/**
48+
* If the collection has partitionKey enabled:
49+
* - The partitionName parameter cannot be specified for import.
50+
* If the collection does not have partitionKey enabled:
51+
* - You may specify partitionName for the import.
52+
* - Defaults to the "default" partition if not specified.
53+
*/
4054
private String partitionName;
55+
56+
private String stageName;
57+
58+
/**
59+
* Data import can be configured in multiple ways using `dataPaths`:
60+
* <p>
61+
* 1. Multi-path import (multiple folders or files):
62+
* "dataPaths": [
63+
* ["parquet-folder-1/1.parquet"],
64+
* ["parquet-folder-2/1.parquet"],
65+
* ["parquet-folder-3/"]
66+
* ]
67+
* <p>
68+
* 2. Folder import:
69+
* "dataPaths": [
70+
* ["parquet-folder/"]
71+
* ]
72+
* <p>
73+
* 3. Single file import:
74+
* "dataPaths": [
75+
* ["parquet-folder/1.parquet"]
76+
* ]
77+
*/
78+
private List<List<String>> dataPaths;
4179
}

0 commit comments

Comments
 (0)