Skip to content

Commit 5e46e08

Browse files
yhmoyhmo
committed
Add external collection API support
Signed-off-by: yhmo <yihua.mo@zilliz.com>
1 parent 16e5aac commit 5e46e08

17 files changed

+1264
-4
lines changed

‎examples/src/main/java/io/milvus/v2/ExternalTableExample.java‎

Lines changed: 399 additions & 0 deletions
Large diffs are not rendered by default.

‎sdk-core/src/main/java/io/milvus/common/utils/JsonUtils.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import com.google.gson.Gson;
2323
import com.google.gson.GsonBuilder;
2424
import com.google.gson.JsonElement;
25+
import com.google.gson.JsonObject;
26+
import com.google.gson.JsonParser;
2527
import com.google.gson.ToNumberPolicy;
2628
import com.google.gson.reflect.TypeToken;
2729

@@ -72,4 +74,15 @@ public static String toJson(JsonElement jsonElement) {
7274
public static <T> JsonElement toJsonTree(T obj) {
7375
return GSON_INSTANCE.toJsonTree(obj);
7476
}
77+
78+
public static JsonObject parseFromString(String jsonStr) {
79+
if (jsonStr == null || jsonStr.isEmpty()) {
80+
return new JsonObject();
81+
}
82+
return JsonParser.parseString(jsonStr).getAsJsonObject();
83+
}
84+
85+
public static String toJsonString(JsonObject jsonObject) {
86+
return jsonObject != null ? jsonObject.toString() : "";
87+
}
7588
}

‎sdk-core/src/main/java/io/milvus/v2/client/MilvusClientV2.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1210,6 +1210,36 @@ public GetCompactionPlansResp getCompactionPlans(GetCompactionPlansReq request)
12101210
return rpcUtils.retry(() -> utilityService.getCompactionPlans(this.getRpcStub(), request));
12111211
}
12121212

1213+
/**
1214+
* Refresh an external collection from its external data source.
1215+
*
1216+
* @param request refresh external collection request
1217+
* @return RefreshExternalCollectionResp containing the job ID
1218+
*/
1219+
public RefreshExternalCollectionResp refreshExternalCollection(RefreshExternalCollectionReq request) {
1220+
return rpcUtils.retry(() -> utilityService.refreshExternalCollection(this.getRpcStub(), request));
1221+
}
1222+
1223+
/**
1224+
* Get the progress of a refresh external collection job.
1225+
*
1226+
* @param request get refresh progress request containing the job ID
1227+
* @return GetRefreshExternalCollectionProgressResp containing the job info
1228+
*/
1229+
public GetRefreshExternalCollectionProgressResp getRefreshExternalCollectionProgress(GetRefreshExternalCollectionProgressReq request) {
1230+
return rpcUtils.retry(() -> utilityService.getRefreshExternalCollectionProgress(this.getRpcStub(), request));
1231+
}
1232+
1233+
/**
1234+
* List refresh external collection jobs.
1235+
*
1236+
* @param request list refresh jobs request
1237+
* @return ListRefreshExternalCollectionJobsResp containing the list of job infos
1238+
*/
1239+
public ListRefreshExternalCollectionJobsResp listRefreshExternalCollectionJobs(ListRefreshExternalCollectionJobsReq request) {
1240+
return rpcUtils.retry(() -> utilityService.listRefreshExternalCollectionJobs(this.getRpcStub(), request));
1241+
}
1242+
12131243
/**
12141244
* Optimize collection to adjust segment sizes for better query performance.
12151245
*

‎sdk-core/src/main/java/io/milvus/v2/service/collection/CollectionService.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package io.milvus.v2.service.collection;
2121

2222
import io.milvus.common.utils.GTsDict;
23+
import io.milvus.common.utils.JsonUtils;
2324
import io.milvus.grpc.*;
2425
import io.milvus.param.ParamUtils;
2526
import io.milvus.v2.common.IndexParam;
@@ -130,7 +131,9 @@ public Void createCollectionWithSchema(MilvusServiceGrpc.MilvusServiceBlockingSt
130131
CollectionSchema.Builder grpcSchemaBuilder = CollectionSchema.newBuilder()
131132
.setName(collectionName)
132133
.setDescription(request.getDescription())
133-
.setEnableDynamicField(request.getCollectionSchema().isEnableDynamicField());
134+
.setEnableDynamicField(request.getCollectionSchema().isEnableDynamicField())
135+
.setExternalSource(request.getCollectionSchema().getExternalSource())
136+
.setExternalSpec(JsonUtils.toJsonString(request.getCollectionSchema().getExternalSpec()));
134137
List<String> outputFields = new ArrayList<>();
135138
for (CreateCollectionReq.Function function : request.getCollectionSchema().getFunctionList()) {
136139
grpcSchemaBuilder.addFunctions(SchemaUtils.convertToGrpcFunction(function)).build();

‎sdk-core/src/main/java/io/milvus/v2/service/collection/request/AddFieldReq.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ public class AddFieldReq {
5050
private Map<String, String> typeParams;
5151
private Map<String, Object> multiAnalyzerParams; // for multi‑language analyzers
5252

53+
private String externalField; // external field name mapping
54+
5355
private List<FieldSchema> structFields;
5456

5557
AddFieldReq(AddFieldReqBuilder<?> builder) {
@@ -72,6 +74,7 @@ public class AddFieldReq {
7274
this.enableMatch = builder.enableMatch;
7375
this.typeParams = builder.typeParams;
7476
this.multiAnalyzerParams = builder.multiAnalyzerParams;
77+
this.externalField = builder.externalField != null ? builder.externalField : "";
7578
this.structFields = builder.structFields != null ? builder.structFields : new ArrayList<>();
7679
}
7780

@@ -156,6 +159,10 @@ public Map<String, Object> getMultiAnalyzerParams() {
156159
return multiAnalyzerParams;
157160
}
158161

162+
public String getExternalField() {
163+
return externalField;
164+
}
165+
159166
public List<FieldSchema> getStructFields() {
160167
return structFields;
161168
}
@@ -237,6 +244,10 @@ public void setMultiAnalyzerParams(Map<String, Object> multiAnalyzerParams) {
237244
this.multiAnalyzerParams = multiAnalyzerParams;
238245
}
239246

247+
public void setExternalField(String externalField) {
248+
this.externalField = externalField;
249+
}
250+
240251
public void setStructFields(List<FieldSchema> structFields) {
241252
this.structFields = structFields;
242253
}
@@ -263,6 +274,7 @@ public String toString() {
263274
", enableMatch=" + enableMatch +
264275
", typeParams=" + typeParams +
265276
", multiAnalyzerParams=" + multiAnalyzerParams +
277+
", externalField='" + externalField + '\'' +
266278
", structFields=" + structFields +
267279
'}';
268280
}
@@ -287,6 +299,7 @@ public static class AddFieldReqBuilder<T extends AddFieldReqBuilder<T>> {
287299
private Boolean enableMatch;
288300
private Map<String, String> typeParams;
289301
private Map<String, Object> multiAnalyzerParams;
302+
private String externalField;
290303
private List<FieldSchema> structFields;
291304

292305
public T fieldName(String fieldName) {
@@ -385,6 +398,11 @@ public T multiAnalyzerParams(Map<String, Object> multiAnalyzerParams) {
385398
return (T) this;
386399
}
387400

401+
public T externalField(String externalField) {
402+
this.externalField = externalField;
403+
return (T) this;
404+
}
405+
388406
public T structFields(List<CreateCollectionReq.FieldSchema> structFields) {
389407
this.structFields = structFields;
390408
return (T) this;

‎sdk-core/src/main/java/io/milvus/v2/service/collection/request/CreateCollectionReq.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package io.milvus.v2.service.collection.request;
2121

22+
import com.google.gson.JsonObject;
2223
import io.milvus.common.clientenum.FunctionType;
2324
import io.milvus.exception.ParamException;
2425
import io.milvus.v2.common.ConsistencyLevel;
@@ -407,12 +408,16 @@ public static class CollectionSchema {
407408

408409
private boolean enableDynamicField = false;
409410
private List<CreateCollectionReq.Function> functionList = new ArrayList<>();
411+
private String externalSource = "";
412+
private JsonObject externalSpec;
410413

411414
private CollectionSchema(CollectionSchemaBuilder builder) {
412415
this.fieldSchemaList = builder.fieldSchemaList;
413416
this.structFields = builder.structFields;
414417
this.enableDynamicField = builder.enableDynamicField;
415418
this.functionList = builder.functionList;
419+
this.externalSource = builder.externalSource;
420+
this.externalSpec = builder.externalSpec;
416421
}
417422

418423
public CollectionSchema addField(AddFieldReq addFieldReq) {
@@ -479,13 +484,31 @@ public void setFunctionList(List<CreateCollectionReq.Function> functionList) {
479484
this.functionList = functionList;
480485
}
481486

487+
public String getExternalSource() {
488+
return externalSource;
489+
}
490+
491+
public void setExternalSource(String externalSource) {
492+
this.externalSource = externalSource;
493+
}
494+
495+
public JsonObject getExternalSpec() {
496+
return externalSpec;
497+
}
498+
499+
public void setExternalSpec(JsonObject externalSpec) {
500+
this.externalSpec = externalSpec;
501+
}
502+
482503
@Override
483504
public String toString() {
484505
return "CollectionSchema{" +
485506
"fieldSchemaList=" + fieldSchemaList +
486507
", structFields=" + structFields +
487508
", enableDynamicField=" + enableDynamicField +
488509
", functionList=" + functionList +
510+
", externalSource='" + externalSource + '\'' +
511+
", externalSpec=" + externalSpec +
489512
'}';
490513
}
491514

@@ -498,6 +521,8 @@ public static class CollectionSchemaBuilder {
498521
private List<CreateCollectionReq.StructFieldSchema> structFields = new ArrayList<>();
499522
private boolean enableDynamicField = false;
500523
private List<CreateCollectionReq.Function> functionList = new ArrayList<>();
524+
private String externalSource = "";
525+
private JsonObject externalSpec;
501526

502527
private CollectionSchemaBuilder() {
503528
}
@@ -522,6 +547,16 @@ public CollectionSchemaBuilder functionList(List<CreateCollectionReq.Function> f
522547
return this;
523548
}
524549

550+
public CollectionSchemaBuilder externalSource(String externalSource) {
551+
this.externalSource = externalSource;
552+
return this;
553+
}
554+
555+
public CollectionSchemaBuilder externalSpec(JsonObject externalSpec) {
556+
this.externalSpec = externalSpec;
557+
return this;
558+
}
559+
525560
public CollectionSchema build() {
526561
return new CollectionSchema(this);
527562
}
@@ -549,6 +584,7 @@ public static class FieldSchema {
549584
// If a specific field, such as maxLength, has been specified, it will override the corresponding key's value in typeParams.
550585
private Map<String, String> typeParams;
551586
private Map<String, Object> multiAnalyzerParams; // for multi‑language analyzers
587+
private String externalField = ""; // external field name mapping
552588

553589
private FieldSchema(FieldSchemaBuilder builder) {
554590
this.name = builder.name;
@@ -569,6 +605,7 @@ private FieldSchema(FieldSchemaBuilder builder) {
569605
this.enableMatch = builder.enableMatch;
570606
this.typeParams = builder.typeParams;
571607
this.multiAnalyzerParams = builder.multiAnalyzerParams;
608+
this.externalField = builder.externalField;
572609
}
573610

574611
// Getters and Setters
@@ -716,6 +753,14 @@ public void setMultiAnalyzerParams(Map<String, Object> multiAnalyzerParams) {
716753
this.multiAnalyzerParams = multiAnalyzerParams;
717754
}
718755

756+
public String getExternalField() {
757+
return externalField;
758+
}
759+
760+
public void setExternalField(String externalField) {
761+
this.externalField = externalField;
762+
}
763+
719764
@Override
720765
public String toString() {
721766
return "FieldSchema{" +
@@ -737,6 +782,7 @@ public String toString() {
737782
", enableMatch=" + enableMatch +
738783
", typeParams=" + typeParams +
739784
", multiAnalyzerParams=" + multiAnalyzerParams +
785+
", externalField='" + externalField + '\'' +
740786
'}';
741787
}
742788

@@ -763,6 +809,7 @@ public static class FieldSchemaBuilder {
763809
private Boolean enableMatch;
764810
private Map<String, String> typeParams;
765811
private Map<String, Object> multiAnalyzerParams;
812+
private String externalField = "";
766813

767814
private FieldSchemaBuilder() {
768815
}
@@ -857,6 +904,11 @@ public FieldSchemaBuilder multiAnalyzerParams(Map<String, Object> multiAnalyzerP
857904
return this;
858905
}
859906

907+
public FieldSchemaBuilder externalField(String externalField) {
908+
this.externalField = externalField;
909+
return this;
910+
}
911+
860912
public FieldSchema build() {
861913
return new FieldSchema(this);
862914
}

‎sdk-core/src/main/java/io/milvus/v2/service/utility/UtilityService.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package io.milvus.v2.service.utility;
2121

22+
import io.milvus.common.utils.JsonUtils;
2223
import io.milvus.grpc.*;
2324
import io.milvus.v2.common.CompactionPlan;
2425
import io.milvus.v2.common.CompactionState;
@@ -332,4 +333,79 @@ public GetQuerySegmentInfoResp getQuerySegmentInfo(MilvusServiceGrpc.MilvusServi
332333
.segmentInfos(segmentInfos)
333334
.build();
334335
}
336+
337+
public RefreshExternalCollectionResp refreshExternalCollection(MilvusServiceGrpc.MilvusServiceBlockingStub blockingStub,
338+
RefreshExternalCollectionReq request) {
339+
String dbName = request.getDatabaseName();
340+
String collectionName = request.getCollectionName();
341+
String title = String.format("RefreshExternalCollection '%s' in database: '%s'", collectionName, dbName);
342+
343+
RefreshExternalCollectionRequest.Builder builder = RefreshExternalCollectionRequest.newBuilder()
344+
.setCollectionName(collectionName)
345+
.setExternalSource(request.getExternalSource())
346+
.setExternalSpec(JsonUtils.toJsonString(request.getExternalSpec()));
347+
if (StringUtils.isNotEmpty(dbName)) {
348+
builder.setDbName(dbName);
349+
}
350+
351+
RefreshExternalCollectionResponse response = blockingStub.refreshExternalCollection(builder.build());
352+
rpcUtils.handleResponse(title, response.getStatus());
353+
return RefreshExternalCollectionResp.builder()
354+
.jobId(response.getJobId())
355+
.build();
356+
}
357+
358+
public GetRefreshExternalCollectionProgressResp getRefreshExternalCollectionProgress(
359+
MilvusServiceGrpc.MilvusServiceBlockingStub blockingStub,
360+
GetRefreshExternalCollectionProgressReq request) {
361+
String title = String.format("GetRefreshExternalCollectionProgress jobId: %d", request.getJobId());
362+
363+
GetRefreshExternalCollectionProgressRequest grpcRequest = GetRefreshExternalCollectionProgressRequest.newBuilder()
364+
.setJobId(request.getJobId())
365+
.build();
366+
367+
GetRefreshExternalCollectionProgressResponse response = blockingStub.getRefreshExternalCollectionProgress(grpcRequest);
368+
rpcUtils.handleResponse(title, response.getStatus());
369+
return GetRefreshExternalCollectionProgressResp.builder()
370+
.jobInfo(convertJobInfo(response.getJobInfo()))
371+
.build();
372+
}
373+
374+
public ListRefreshExternalCollectionJobsResp listRefreshExternalCollectionJobs(
375+
MilvusServiceGrpc.MilvusServiceBlockingStub blockingStub,
376+
ListRefreshExternalCollectionJobsReq request) {
377+
String dbName = request.getDatabaseName();
378+
String collectionName = request.getCollectionName();
379+
String title = String.format("ListRefreshExternalCollectionJobs '%s' in database: '%s'", collectionName, dbName);
380+
381+
ListRefreshExternalCollectionJobsRequest.Builder builder = ListRefreshExternalCollectionJobsRequest.newBuilder()
382+
.setCollectionName(collectionName);
383+
if (StringUtils.isNotEmpty(dbName)) {
384+
builder.setDbName(dbName);
385+
}
386+
387+
ListRefreshExternalCollectionJobsResponse response = blockingStub.listRefreshExternalCollectionJobs(builder.build());
388+
rpcUtils.handleResponse(title, response.getStatus());
389+
390+
List<io.milvus.v2.service.utility.response.RefreshExternalCollectionJobInfo> jobs = new ArrayList<>();
391+
for (io.milvus.grpc.RefreshExternalCollectionJobInfo job : response.getJobsList()) {
392+
jobs.add(convertJobInfo(job));
393+
}
394+
return ListRefreshExternalCollectionJobsResp.builder()
395+
.jobs(jobs)
396+
.build();
397+
}
398+
399+
private io.milvus.v2.service.utility.response.RefreshExternalCollectionJobInfo convertJobInfo(io.milvus.grpc.RefreshExternalCollectionJobInfo info) {
400+
return io.milvus.v2.service.utility.response.RefreshExternalCollectionJobInfo.builder()
401+
.jobId(info.getJobId())
402+
.collectionName(info.getCollectionName())
403+
.state(info.getState().name())
404+
.progress((int) info.getProgress())
405+
.reason(info.getReason())
406+
.externalSource(info.getExternalSource())
407+
.startTime(info.getStartTime())
408+
.endTime(info.getEndTime())
409+
.build();
410+
}
335411
}

0 commit comments

Comments
 (0)