Add external collection API support

sdk-core/src/main/java/io/milvus/common/utils/JsonUtils.java

@@ -22,6 +22,8 @@
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
 import com.google.gson.ToNumberPolicy;
 import com.google.gson.reflect.TypeToken;
 
@@ -72,4 +74,15 @@ public static String toJson(JsonElement jsonElement) {
     public static <T> JsonElement toJsonTree(T obj) {
         return GSON_INSTANCE.toJsonTree(obj);
     }
+
+    public static JsonObject parseFromString(String jsonStr) {
+        if (jsonStr == null || jsonStr.isEmpty()) {
+            return new JsonObject();
+        }
+        return JsonParser.parseString(jsonStr).getAsJsonObject();
+    }
+
+    public static String toJsonString(JsonObject jsonObject) {
+        return jsonObject != null ? jsonObject.toString() : "";
+    }
 }

sdk-core/src/main/java/io/milvus/v2/client/MilvusClientV2.java

@@ -1210,6 +1210,36 @@ public GetCompactionPlansResp getCompactionPlans(GetCompactionPlansReq request)
         return rpcUtils.retry(() -> utilityService.getCompactionPlans(this.getRpcStub(), request));
     }
 
+    /**
+     * Refresh an external collection from its external data source.
+     *
+     * @param request refresh external collection request
+     * @return RefreshExternalCollectionResp containing the job ID
+     */
+    public RefreshExternalCollectionResp refreshExternalCollection(RefreshExternalCollectionReq request) {
+        return rpcUtils.retry(() -> utilityService.refreshExternalCollection(this.getRpcStub(), request));
+    }
+
+    /**
+     * Get the progress of a refresh external collection job.
+     *
+     * @param request get refresh progress request containing the job ID
+     * @return GetRefreshExternalCollectionProgressResp containing the job info
+     */
+    public GetRefreshExternalCollectionProgressResp getRefreshExternalCollectionProgress(GetRefreshExternalCollectionProgressReq request) {
+        return rpcUtils.retry(() -> utilityService.getRefreshExternalCollectionProgress(this.getRpcStub(), request));
+    }
+
+    /**
+     * List refresh external collection jobs.
+     *
+     * @param request list refresh jobs request
+     * @return ListRefreshExternalCollectionJobsResp containing the list of job infos
+     */
+    public ListRefreshExternalCollectionJobsResp listRefreshExternalCollectionJobs(ListRefreshExternalCollectionJobsReq request) {
+        return rpcUtils.retry(() -> utilityService.listRefreshExternalCollectionJobs(this.getRpcStub(), request));
+    }
+
     /**
      * Optimize collection to adjust segment sizes for better query performance.
      *

sdk-core/src/main/java/io/milvus/v2/service/collection/CollectionService.java

@@ -20,6 +20,7 @@
 package io.milvus.v2.service.collection;
 
 import io.milvus.common.utils.GTsDict;
+import io.milvus.common.utils.JsonUtils;
 import io.milvus.grpc.*;
 import io.milvus.param.ParamUtils;
 import io.milvus.v2.common.IndexParam;
@@ -130,7 +131,9 @@ public Void createCollectionWithSchema(MilvusServiceGrpc.MilvusServiceBlockingSt
         CollectionSchema.Builder grpcSchemaBuilder = CollectionSchema.newBuilder()
                 .setName(collectionName)
                 .setDescription(request.getDescription())
-                .setEnableDynamicField(request.getCollectionSchema().isEnableDynamicField());
+                .setEnableDynamicField(request.getCollectionSchema().isEnableDynamicField())
+                .setExternalSource(request.getCollectionSchema().getExternalSource())
+                .setExternalSpec(JsonUtils.toJsonString(request.getCollectionSchema().getExternalSpec()));
         List<String> outputFields = new ArrayList<>();
         for (CreateCollectionReq.Function function : request.getCollectionSchema().getFunctionList()) {
             grpcSchemaBuilder.addFunctions(SchemaUtils.convertToGrpcFunction(function)).build();

sdk-core/src/main/java/io/milvus/v2/service/collection/request/AddFieldReq.java

@@ -50,6 +50,8 @@ public class AddFieldReq {
     private Map<String, String> typeParams;
     private Map<String, Object> multiAnalyzerParams; // for multi‑language analyzers
 
+    private String externalField; // external field name mapping
+
     private List<FieldSchema> structFields;
 
     AddFieldReq(AddFieldReqBuilder<?> builder) {
@@ -72,6 +74,7 @@ public class AddFieldReq {
         this.enableMatch = builder.enableMatch;
         this.typeParams = builder.typeParams;
         this.multiAnalyzerParams = builder.multiAnalyzerParams;
+        this.externalField = builder.externalField != null ? builder.externalField : "";
         this.structFields = builder.structFields != null ? builder.structFields : new ArrayList<>();
     }
 
@@ -156,6 +159,10 @@ public Map<String, Object> getMultiAnalyzerParams() {
         return multiAnalyzerParams;
     }
 
+    public String getExternalField() {
+        return externalField;
+    }
+
     public List<FieldSchema> getStructFields() {
         return structFields;
     }
@@ -237,6 +244,10 @@ public void setMultiAnalyzerParams(Map<String, Object> multiAnalyzerParams) {
         this.multiAnalyzerParams = multiAnalyzerParams;
     }
 
+    public void setExternalField(String externalField) {
+        this.externalField = externalField;
+    }
+
     public void setStructFields(List<FieldSchema> structFields) {
         this.structFields = structFields;
     }
@@ -263,6 +274,7 @@ public String toString() {
                 ", enableMatch=" + enableMatch +
                 ", typeParams=" + typeParams +
                 ", multiAnalyzerParams=" + multiAnalyzerParams +
+                ", externalField='" + externalField + '\'' +
                 ", structFields=" + structFields +
                 '}';
     }
@@ -287,6 +299,7 @@ public static class AddFieldReqBuilder<T extends AddFieldReqBuilder<T>> {
         private Boolean enableMatch;
         private Map<String, String> typeParams;
         private Map<String, Object> multiAnalyzerParams;
+        private String externalField;
         private List<FieldSchema> structFields;
 
         public T fieldName(String fieldName) {
@@ -385,6 +398,11 @@ public T multiAnalyzerParams(Map<String, Object> multiAnalyzerParams) {
             return (T) this;
         }
 
+        public T externalField(String externalField) {
+            this.externalField = externalField;
+            return (T) this;
+        }
+
         public T structFields(List<CreateCollectionReq.FieldSchema> structFields) {
             this.structFields = structFields;
             return (T) this;

sdk-core/src/main/java/io/milvus/v2/service/collection/request/CreateCollectionReq.java

@@ -19,6 +19,7 @@
 
 package io.milvus.v2.service.collection.request;
 
+import com.google.gson.JsonObject;
 import io.milvus.common.clientenum.FunctionType;
 import io.milvus.exception.ParamException;
 import io.milvus.v2.common.ConsistencyLevel;
@@ -407,12 +408,16 @@ public static class CollectionSchema {
 
         private boolean enableDynamicField = false;
         private List<CreateCollectionReq.Function> functionList = new ArrayList<>();
+        private String externalSource = "";
+        private JsonObject externalSpec;
 
         private CollectionSchema(CollectionSchemaBuilder builder) {
             this.fieldSchemaList = builder.fieldSchemaList;
             this.structFields = builder.structFields;
             this.enableDynamicField = builder.enableDynamicField;
             this.functionList = builder.functionList;
+            this.externalSource = builder.externalSource;
+            this.externalSpec = builder.externalSpec;
         }
 
         public CollectionSchema addField(AddFieldReq addFieldReq) {
@@ -479,13 +484,31 @@ public void setFunctionList(List<CreateCollectionReq.Function> functionList) {
             this.functionList = functionList;
         }
 
+        public String getExternalSource() {
+            return externalSource;
+        }
+
+        public void setExternalSource(String externalSource) {
+            this.externalSource = externalSource;
+        }
+
+        public JsonObject getExternalSpec() {
+            return externalSpec;
+        }
+
+        public void setExternalSpec(JsonObject externalSpec) {
+            this.externalSpec = externalSpec;
+        }
+
         @Override
         public String toString() {
             return "CollectionSchema{" +
                     "fieldSchemaList=" + fieldSchemaList +
                     ", structFields=" + structFields +
                     ", enableDynamicField=" + enableDynamicField +
                     ", functionList=" + functionList +
+                    ", externalSource='" + externalSource + '\'' +
+                    ", externalSpec=" + externalSpec +
                     '}';
         }
 
@@ -498,6 +521,8 @@ public static class CollectionSchemaBuilder {
             private List<CreateCollectionReq.StructFieldSchema> structFields = new ArrayList<>();
             private boolean enableDynamicField = false;
             private List<CreateCollectionReq.Function> functionList = new ArrayList<>();
+            private String externalSource = "";
+            private JsonObject externalSpec;
 
             private CollectionSchemaBuilder() {
             }
@@ -522,6 +547,16 @@ public CollectionSchemaBuilder functionList(List<CreateCollectionReq.Function> f
                 return this;
             }
 
+            public CollectionSchemaBuilder externalSource(String externalSource) {
+                this.externalSource = externalSource;
+                return this;
+            }
+
+            public CollectionSchemaBuilder externalSpec(JsonObject externalSpec) {
+                this.externalSpec = externalSpec;
+                return this;
+            }
+
             public CollectionSchema build() {
                 return new CollectionSchema(this);
             }
@@ -549,6 +584,7 @@ public static class FieldSchema {
         // If a specific field, such as maxLength, has been specified, it will override the corresponding key's value in typeParams.
         private Map<String, String> typeParams;
         private Map<String, Object> multiAnalyzerParams; // for multi‑language analyzers
+        private String externalField = ""; // external field name mapping
 
         private FieldSchema(FieldSchemaBuilder builder) {
             this.name = builder.name;
@@ -569,6 +605,7 @@ private FieldSchema(FieldSchemaBuilder builder) {
             this.enableMatch = builder.enableMatch;
             this.typeParams = builder.typeParams;
             this.multiAnalyzerParams = builder.multiAnalyzerParams;
+            this.externalField = builder.externalField;
         }
 
         // Getters and Setters
@@ -716,6 +753,14 @@ public void setMultiAnalyzerParams(Map<String, Object> multiAnalyzerParams) {
             this.multiAnalyzerParams = multiAnalyzerParams;
         }
 
+        public String getExternalField() {
+            return externalField;
+        }
+
+        public void setExternalField(String externalField) {
+            this.externalField = externalField;
+        }
+
         @Override
         public String toString() {
             return "FieldSchema{" +
@@ -737,6 +782,7 @@ public String toString() {
                     ", enableMatch=" + enableMatch +
                     ", typeParams=" + typeParams +
                     ", multiAnalyzerParams=" + multiAnalyzerParams +
+                    ", externalField='" + externalField + '\'' +
                     '}';
         }
 
@@ -763,6 +809,7 @@ public static class FieldSchemaBuilder {
             private Boolean enableMatch;
             private Map<String, String> typeParams;
             private Map<String, Object> multiAnalyzerParams;
+            private String externalField = "";
 
             private FieldSchemaBuilder() {
             }
@@ -857,6 +904,11 @@ public FieldSchemaBuilder multiAnalyzerParams(Map<String, Object> multiAnalyzerP
                 return this;
             }
 
+            public FieldSchemaBuilder externalField(String externalField) {
+                this.externalField = externalField;
+                return this;
+            }
+
             public FieldSchema build() {
                 return new FieldSchema(this);
             }

sdk-core/src/main/java/io/milvus/v2/service/utility/UtilityService.java

@@ -19,6 +19,7 @@
 
 package io.milvus.v2.service.utility;
 
+import io.milvus.common.utils.JsonUtils;
 import io.milvus.grpc.*;
 import io.milvus.v2.common.CompactionPlan;
 import io.milvus.v2.common.CompactionState;
@@ -332,4 +333,79 @@ public GetQuerySegmentInfoResp getQuerySegmentInfo(MilvusServiceGrpc.MilvusServi
                 .segmentInfos(segmentInfos)
                 .build();
     }
+
+    public RefreshExternalCollectionResp refreshExternalCollection(MilvusServiceGrpc.MilvusServiceBlockingStub blockingStub,
+                                                                    RefreshExternalCollectionReq request) {
+        String dbName = request.getDatabaseName();
+        String collectionName = request.getCollectionName();
+        String title = String.format("RefreshExternalCollection '%s' in database: '%s'", collectionName, dbName);
+
+        RefreshExternalCollectionRequest.Builder builder = RefreshExternalCollectionRequest.newBuilder()
+                .setCollectionName(collectionName)
+                .setExternalSource(request.getExternalSource())
+                .setExternalSpec(JsonUtils.toJsonString(request.getExternalSpec()));
+        if (StringUtils.isNotEmpty(dbName)) {
+            builder.setDbName(dbName);
+        }
+
+        RefreshExternalCollectionResponse response = blockingStub.refreshExternalCollection(builder.build());
+        rpcUtils.handleResponse(title, response.getStatus());
+        return RefreshExternalCollectionResp.builder()
+                .jobId(response.getJobId())
+                .build();
+    }
+
+    public GetRefreshExternalCollectionProgressResp getRefreshExternalCollectionProgress(
+            MilvusServiceGrpc.MilvusServiceBlockingStub blockingStub,
+            GetRefreshExternalCollectionProgressReq request) {
+        String title = String.format("GetRefreshExternalCollectionProgress jobId: %d", request.getJobId());
+
+        GetRefreshExternalCollectionProgressRequest grpcRequest = GetRefreshExternalCollectionProgressRequest.newBuilder()
+                .setJobId(request.getJobId())
+                .build();
+
+        GetRefreshExternalCollectionProgressResponse response = blockingStub.getRefreshExternalCollectionProgress(grpcRequest);
+        rpcUtils.handleResponse(title, response.getStatus());
+        return GetRefreshExternalCollectionProgressResp.builder()
+                .jobInfo(convertJobInfo(response.getJobInfo()))
+                .build();
+    }
+
+    public ListRefreshExternalCollectionJobsResp listRefreshExternalCollectionJobs(
+            MilvusServiceGrpc.MilvusServiceBlockingStub blockingStub,
+            ListRefreshExternalCollectionJobsReq request) {
+        String dbName = request.getDatabaseName();
+        String collectionName = request.getCollectionName();
+        String title = String.format("ListRefreshExternalCollectionJobs '%s' in database: '%s'", collectionName, dbName);
+
+        ListRefreshExternalCollectionJobsRequest.Builder builder = ListRefreshExternalCollectionJobsRequest.newBuilder()
+                .setCollectionName(collectionName);
+        if (StringUtils.isNotEmpty(dbName)) {
+            builder.setDbName(dbName);
+        }
+
+        ListRefreshExternalCollectionJobsResponse response = blockingStub.listRefreshExternalCollectionJobs(builder.build());
+        rpcUtils.handleResponse(title, response.getStatus());
+
+        List<io.milvus.v2.service.utility.response.RefreshExternalCollectionJobInfo> jobs = new ArrayList<>();
+        for (io.milvus.grpc.RefreshExternalCollectionJobInfo job : response.getJobsList()) {
+            jobs.add(convertJobInfo(job));
+        }
+        return ListRefreshExternalCollectionJobsResp.builder()
+                .jobs(jobs)
+                .build();
+    }
+
+    private io.milvus.v2.service.utility.response.RefreshExternalCollectionJobInfo convertJobInfo(io.milvus.grpc.RefreshExternalCollectionJobInfo info) {
+        return io.milvus.v2.service.utility.response.RefreshExternalCollectionJobInfo.builder()
+                .jobId(info.getJobId())
+                .collectionName(info.getCollectionName())
+                .state(info.getState().name())
+                .progress((int) info.getProgress())
+                .reason(info.getReason())
+                .externalSource(info.getExternalSource())
+                .startTime(info.getStartTime())
+                .endTime(info.getEndTime())
+                .build();
+    }
 }