Skip to content

Fix a bug of nullable Array type #1366

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sre-ci-robot merged 1 commit into from
Apr 9, 2025
Merged

Fix a bug of nullable Array type #1366

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ services:

standalone:
container_name: milvus-javasdk-test-standalone
image: milvusdb/milvus:v2.5.4
image: milvusdb/milvus:v2.5.8
command: ["milvus", "run", "standalone"]
environment:
ETCD_ENDPOINTS: etcd:2379
Expand Down Expand Up @@ -77,7 +77,7 @@ services:

standaloneslave:
container_name: milvus-javasdk-test-slave-standalone
image: milvusdb/milvus:v2.5.7
image: milvusdb/milvus:v2.5.8
command: ["milvus", "run", "standalone"]
environment:
ETCD_ENDPOINTS: etcdslave:2379
Expand Down
12 changes: 12 additions & 0 deletions examples/src/main/java/io/milvus/v1/NullAndDefaultExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ private static void queryWithExpr(MilvusClient client, String expr) {
.addOutField("nullable_test")
.addOutField("default_test")
.addOutField("nullable_default")
.addOutField("nullable_array")
.build());
QueryResultsWrapper queryWrapper = new QueryResultsWrapper(queryRet.getData());
System.out.println("\nQuery with expression: " + expr);
Expand Down Expand Up @@ -83,6 +84,14 @@ public static void main(String[] args) {
.withMaxLength(64)
.withDefaultValue("I am default value")
.withNullable(true)
.build(),
FieldType.newBuilder()
.withName("nullable_array")
.withDataType(DataType.Array)
.withElementType(DataType.VarChar)
.withMaxCapacity(10)
.withMaxLength(100)
.withNullable(true)
.build()
);

Expand Down Expand Up @@ -130,6 +139,9 @@ public static void main(String[] args) {
row.addProperty("nullable_test", i);
} else {
row.add("nullable_test", JsonNull.INSTANCE);

List<String> arr = Arrays.asList("A", "B", "C");
row.add("nullable_array", gson.toJsonTree(arr));
}

// some values are default value
Expand Down
13 changes: 12 additions & 1 deletion examples/src/main/java/io/milvus/v2/NullAndDefaultExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ private static void queryWithExpr(MilvusClientV2 client, String expr) {
QueryResp queryRet = client.query(QueryReq.builder()
.collectionName(COLLECTION_NAME)
.filter(expr)
.outputFields(Arrays.asList("nullable_test", "default_test", "nullable_default"))
.outputFields(Arrays.asList("nullable_test", "default_test", "nullable_default", "nullable_array"))
.build());
System.out.println("\nQuery with expression: " + expr);
List<QueryResp.QueryResult> records = queryRet.getQueryResults();
Expand Down Expand Up @@ -81,6 +81,14 @@ public static void main(String[] args) {
.isNullable(true)
.defaultValue("I am default value")
.build());
collectionSchema.addField(AddFieldReq.builder()
.fieldName("nullable_array")
.dataType(DataType.Array)
.elementType(DataType.VarChar)
.maxCapacity(10)
.maxLength(100)
.isNullable(true)
.build());

List<IndexParam> indexes = new ArrayList<>();
indexes.add(IndexParam.builder()
Expand Down Expand Up @@ -111,6 +119,9 @@ public static void main(String[] args) {
row.addProperty("nullable_test", i);
} else {
row.add("nullable_test", JsonNull.INSTANCE);

List<String> arr = Arrays.asList("A", "B", "C");
row.add("nullable_array", gson.toJsonTree(arr));
}

// some values are default value
Expand Down
14 changes: 7 additions & 7 deletions sdk-core/src/main/java/io/milvus/param/ParamUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1329,40 +1329,40 @@ private static ScalarField genScalarField(DataType dataType, List<?> objects) {
case UNRECOGNIZED:
throw new ParamException("Cannot support this dataType:" + dataType);
case Int64: {
List<Long> longs = objects.stream().map(p -> (Long) p).collect(Collectors.toList());
List<Long> longs = objects.stream().map(p -> (p == null) ? null : (Long) p).collect(Collectors.toList());
LongArray longArray = LongArray.newBuilder().addAllData(longs).build();
return ScalarField.newBuilder().setLongData(longArray).build();
}
case Int32:
case Int16:
case Int8: {
List<Integer> integers = objects.stream().map(p -> p instanceof Short ? ((Short) p).intValue() : (Integer) p).collect(Collectors.toList());
List<Integer> integers = objects.stream().map(p -> (p == null) ? null : (p instanceof Short ? ((Short) p).intValue() : (Integer) p)).collect(Collectors.toList());
IntArray intArray = IntArray.newBuilder().addAllData(integers).build();
return ScalarField.newBuilder().setIntData(intArray).build();
}
case Bool: {
List<Boolean> booleans = objects.stream().map(p -> (Boolean) p).collect(Collectors.toList());
List<Boolean> booleans = objects.stream().map(p -> (p == null) ? null : (Boolean) p).collect(Collectors.toList());
BoolArray boolArray = BoolArray.newBuilder().addAllData(booleans).build();
return ScalarField.newBuilder().setBoolData(boolArray).build();
}
case Float: {
List<Float> floats = objects.stream().map(p -> (Float) p).collect(Collectors.toList());
List<Float> floats = objects.stream().map(p -> (p == null) ? null : (Float) p).collect(Collectors.toList());
FloatArray floatArray = FloatArray.newBuilder().addAllData(floats).build();
return ScalarField.newBuilder().setFloatData(floatArray).build();
}
case Double: {
List<Double> doubles = objects.stream().map(p -> (Double) p).collect(Collectors.toList());
List<Double> doubles = objects.stream().map(p -> (p == null) ? null : (Double) p).collect(Collectors.toList());
DoubleArray doubleArray = DoubleArray.newBuilder().addAllData(doubles).build();
return ScalarField.newBuilder().setDoubleData(doubleArray).build();
}
case String:
case VarChar: {
List<String> strings = objects.stream().map(p -> (String) p).collect(Collectors.toList());
List<String> strings = objects.stream().map(p -> (p == null) ? null : (String) p).collect(Collectors.toList());
StringArray stringArray = StringArray.newBuilder().addAllData(strings).build();
return ScalarField.newBuilder().setStringData(stringArray).build();
}
case JSON: {
List<ByteString> byteStrings = objects.stream().map(p -> ByteString.copyFromUtf8(p.toString()))
List<ByteString> byteStrings = objects.stream().map(p -> (p == null) ? null : ByteString.copyFromUtf8(p.toString()))
.collect(Collectors.toList());
JSONArray jsonArray = JSONArray.newBuilder().addAllData(byteStrings).build();
return ScalarField.newBuilder().setJsonData(jsonArray).build();
Expand Down
20 changes: 13 additions & 7 deletions sdk-core/src/main/java/io/milvus/response/FieldDataWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,13 +251,6 @@ private List<?> getFieldDataInternal() throws IllegalResponseException {
return packData;
}
case Array:
List<List<?>> array = new ArrayList<>();
ArrayArray arrArray = fieldData.getScalars().getArrayData();
for (int i = 0; i < arrArray.getDataCount(); i++) {
ScalarField scalar = arrArray.getData(i);
array.add(getScalarData(arrArray.getElementType(), scalar, null));
}
return array;
case Int64:
case Int32:
case Int16:
Expand Down Expand Up @@ -308,6 +301,19 @@ private List<?> getScalarData(DataType dt, ScalarField scalar, List<Boolean> val
case JSON:
List<ByteString> dataList = scalar.getJsonData().getDataList();
return dataList.stream().map(ByteString::toStringUtf8).collect(Collectors.toList());
case Array:
List<List<?>> array = new ArrayList<>();
ArrayArray arrArray = fieldData.getScalars().getArrayData();
boolean nullable = validData != null && validData.size() == arrArray.getDataCount();
for (int i = 0; i < arrArray.getDataCount(); i++) {
if (nullable && validData.get(i) == Boolean.FALSE) {
array.add(null);
} else {
ScalarField rowData = arrArray.getData(i);
array.add(getScalarData(arrArray.getElementType(), rowData, null));
}
}
return array;
default:
return new ArrayList<>();
}
Expand Down
50 changes: 34 additions & 16 deletions sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@

import java.nio.ByteBuffer;
import java.util.*;
import java.util.function.Function;

@Testcontainers(disabledWithoutDocker = true)
class MilvusClientV2DockerTest {
Expand Down Expand Up @@ -1890,6 +1891,13 @@ void testNullableAndDefaultValue() {
.isNullable(Boolean.TRUE)
.maxLength(100)
.build());
collectionSchema.addField(AddFieldReq.builder()
.fieldName("arr")
.dataType(DataType.Array)
.elementType(DataType.Int32)
.isNullable(Boolean.TRUE)
.maxCapacity(100)
.build());

List<IndexParam> indexParams = new ArrayList<>();
indexParams.add(IndexParam.builder()
Expand Down Expand Up @@ -1918,7 +1926,11 @@ void testNullableAndDefaultValue() {
} else {
// row.add("flag", JsonNull.INSTANCE);
row.addProperty("desc", "AAA");

List<Integer> arr = Arrays.asList(5, 6);
row.add("arr", JsonUtils.toJsonTree(arr));
}

data.add(row);
}

Expand All @@ -1928,26 +1940,38 @@ void testNullableAndDefaultValue() {
.build());
Assertions.assertEquals(10, insertResp.getInsertCnt());

Function<Map<String, Object>, Void> checkFunc =
entity -> {
long id = (long)entity.get("id");
if (id%2 == 0) {
Assertions.assertEquals((int)id, entity.get("flag"));
Assertions.assertNull(entity.get("desc"));
Assertions.assertNull(entity.get("arr"));
} else {
Assertions.assertEquals(10, entity.get("flag"));
Assertions.assertEquals("AAA", entity.get("desc"));
Object obj = entity.get("arr");
Assertions.assertInstanceOf(List.class, obj);
List<Integer> arr = (List<Integer>)obj;
Assertions.assertEquals(2, arr.size());
Assertions.assertEquals(5, arr.get(0));
Assertions.assertEquals(6, arr.get(1));
}
return null;
};
// query
QueryResp queryResp = client.query(QueryReq.builder()
.collectionName(randomCollectionName)
.filter("id >= 0")
.outputFields(Arrays.asList("desc", "flag"))
.outputFields(Arrays.asList("desc", "flag", "arr"))
.consistencyLevel(ConsistencyLevel.STRONG)
.build());
List<QueryResp.QueryResult> queryResults = queryResp.getQueryResults();
Assertions.assertEquals(10, queryResults.size());
System.out.println("Query results:");
for (QueryResp.QueryResult result : queryResults) {
Map<String, Object> entity = result.getEntity();
long id = (long)entity.get("id");
if (id%2 == 0) {
Assertions.assertEquals((int)id, entity.get("flag"));
Assertions.assertNull(entity.get("desc"));
} else {
Assertions.assertEquals(10, entity.get("flag"));
Assertions.assertEquals("AAA", entity.get("desc"));
}
checkFunc.apply(entity);
System.out.println(result);
}

Expand All @@ -1968,13 +1992,7 @@ void testNullableAndDefaultValue() {
for (SearchResp.SearchResult result : firstResults) {
long id = (long)result.getId();
Map<String, Object> entity = result.getEntity();
if (id%2 == 0) {
Assertions.assertEquals((int)id, entity.get("flag"));
Assertions.assertNull(entity.get("desc"));
} else {
Assertions.assertEquals(10, entity.get("flag"));
Assertions.assertEquals("AAA", entity.get("desc"));
}
checkFunc.apply(entity);
System.out.println(result);
}
}
Expand Down
Loading