Skip to content

Commit 878aa4e

Browse files
yhmoyhmo
committed
Fix a bug of nullable Array type
Signed-off-by: yhmo <yihua.mo@zilliz.com>
1 parent 28dc79b commit 878aa4e

5 files changed

Lines changed: 78 additions & 31 deletions

File tree

‎examples/src/main/java/io/milvus/v1/NullAndDefaultExample.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ private static void queryWithExpr(MilvusClient client, String expr) {
3636
.addOutField("nullable_test")
3737
.addOutField("default_test")
3838
.addOutField("nullable_default")
39+
.addOutField("nullable_array")
3940
.build());
4041
QueryResultsWrapper queryWrapper = new QueryResultsWrapper(queryRet.getData());
4142
System.out.println("\nQuery with expression: " + expr);
@@ -83,6 +84,14 @@ public static void main(String[] args) {
8384
.withMaxLength(64)
8485
.withDefaultValue("I am default value")
8586
.withNullable(true)
87+
.build(),
88+
FieldType.newBuilder()
89+
.withName("nullable_array")
90+
.withDataType(DataType.Array)
91+
.withElementType(DataType.VarChar)
92+
.withMaxCapacity(10)
93+
.withMaxLength(100)
94+
.withNullable(true)
8695
.build()
8796
);
8897

@@ -130,6 +139,9 @@ public static void main(String[] args) {
130139
row.addProperty("nullable_test", i);
131140
} else {
132141
row.add("nullable_test", JsonNull.INSTANCE);
142+
143+
List<String> arr = Arrays.asList("A", "B", "C");
144+
row.add("nullable_array", gson.toJsonTree(arr));
133145
}
134146

135147
// some values are default value

‎examples/src/main/java/io/milvus/v2/NullAndDefaultExample.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ private static void queryWithExpr(MilvusClientV2 client, String expr) {
2828
QueryResp queryRet = client.query(QueryReq.builder()
2929
.collectionName(COLLECTION_NAME)
3030
.filter(expr)
31-
.outputFields(Arrays.asList("nullable_test", "default_test", "nullable_default"))
31+
.outputFields(Arrays.asList("nullable_test", "default_test", "nullable_default", "nullable_array"))
3232
.build());
3333
System.out.println("\nQuery with expression: " + expr);
3434
List<QueryResp.QueryResult> records = queryRet.getQueryResults();
@@ -81,6 +81,14 @@ public static void main(String[] args) {
8181
.isNullable(true)
8282
.defaultValue("I am default value")
8383
.build());
84+
collectionSchema.addField(AddFieldReq.builder()
85+
.fieldName("nullable_array")
86+
.dataType(DataType.Array)
87+
.elementType(DataType.VarChar)
88+
.maxCapacity(10)
89+
.maxLength(100)
90+
.isNullable(true)
91+
.build());
8492

8593
List<IndexParam> indexes = new ArrayList<>();
8694
indexes.add(IndexParam.builder()
@@ -111,6 +119,9 @@ public static void main(String[] args) {
111119
row.addProperty("nullable_test", i);
112120
} else {
113121
row.add("nullable_test", JsonNull.INSTANCE);
122+
123+
List<String> arr = Arrays.asList("A", "B", "C");
124+
row.add("nullable_array", gson.toJsonTree(arr));
114125
}
115126

116127
// some values are default value

‎sdk-core/src/main/java/io/milvus/param/ParamUtils.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1329,40 +1329,40 @@ private static ScalarField genScalarField(DataType dataType, List<?> objects) {
13291329
case UNRECOGNIZED:
13301330
throw new ParamException("Cannot support this dataType:" + dataType);
13311331
case Int64: {
1332-
List<Long> longs = objects.stream().map(p -> (Long) p).collect(Collectors.toList());
1332+
List<Long> longs = objects.stream().map(p -> (p == null) ? null : (Long) p).collect(Collectors.toList());
13331333
LongArray longArray = LongArray.newBuilder().addAllData(longs).build();
13341334
return ScalarField.newBuilder().setLongData(longArray).build();
13351335
}
13361336
case Int32:
13371337
case Int16:
13381338
case Int8: {
1339-
List<Integer> integers = objects.stream().map(p -> p instanceof Short ? ((Short) p).intValue() : (Integer) p).collect(Collectors.toList());
1339+
List<Integer> integers = objects.stream().map(p -> (p == null) ? null : (p instanceof Short ? ((Short) p).intValue() : (Integer) p)).collect(Collectors.toList());
13401340
IntArray intArray = IntArray.newBuilder().addAllData(integers).build();
13411341
return ScalarField.newBuilder().setIntData(intArray).build();
13421342
}
13431343
case Bool: {
1344-
List<Boolean> booleans = objects.stream().map(p -> (Boolean) p).collect(Collectors.toList());
1344+
List<Boolean> booleans = objects.stream().map(p -> (p == null) ? null : (Boolean) p).collect(Collectors.toList());
13451345
BoolArray boolArray = BoolArray.newBuilder().addAllData(booleans).build();
13461346
return ScalarField.newBuilder().setBoolData(boolArray).build();
13471347
}
13481348
case Float: {
1349-
List<Float> floats = objects.stream().map(p -> (Float) p).collect(Collectors.toList());
1349+
List<Float> floats = objects.stream().map(p -> (p == null) ? null : (Float) p).collect(Collectors.toList());
13501350
FloatArray floatArray = FloatArray.newBuilder().addAllData(floats).build();
13511351
return ScalarField.newBuilder().setFloatData(floatArray).build();
13521352
}
13531353
case Double: {
1354-
List<Double> doubles = objects.stream().map(p -> (Double) p).collect(Collectors.toList());
1354+
List<Double> doubles = objects.stream().map(p -> (p == null) ? null : (Double) p).collect(Collectors.toList());
13551355
DoubleArray doubleArray = DoubleArray.newBuilder().addAllData(doubles).build();
13561356
return ScalarField.newBuilder().setDoubleData(doubleArray).build();
13571357
}
13581358
case String:
13591359
case VarChar: {
1360-
List<String> strings = objects.stream().map(p -> (String) p).collect(Collectors.toList());
1360+
List<String> strings = objects.stream().map(p -> (p == null) ? null : (String) p).collect(Collectors.toList());
13611361
StringArray stringArray = StringArray.newBuilder().addAllData(strings).build();
13621362
return ScalarField.newBuilder().setStringData(stringArray).build();
13631363
}
13641364
case JSON: {
1365-
List<ByteString> byteStrings = objects.stream().map(p -> ByteString.copyFromUtf8(p.toString()))
1365+
List<ByteString> byteStrings = objects.stream().map(p -> (p == null) ? null : ByteString.copyFromUtf8(p.toString()))
13661366
.collect(Collectors.toList());
13671367
JSONArray jsonArray = JSONArray.newBuilder().addAllData(byteStrings).build();
13681368
return ScalarField.newBuilder().setJsonData(jsonArray).build();

‎sdk-core/src/main/java/io/milvus/response/FieldDataWrapper.java‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -251,13 +251,6 @@ private List<?> getFieldDataInternal() throws IllegalResponseException {
251251
return packData;
252252
}
253253
case Array:
254-
List<List<?>> array = new ArrayList<>();
255-
ArrayArray arrArray = fieldData.getScalars().getArrayData();
256-
for (int i = 0; i < arrArray.getDataCount(); i++) {
257-
ScalarField scalar = arrArray.getData(i);
258-
array.add(getScalarData(arrArray.getElementType(), scalar, null));
259-
}
260-
return array;
261254
case Int64:
262255
case Int32:
263256
case Int16:
@@ -308,6 +301,19 @@ private List<?> getScalarData(DataType dt, ScalarField scalar, List<Boolean> val
308301
case JSON:
309302
List<ByteString> dataList = scalar.getJsonData().getDataList();
310303
return dataList.stream().map(ByteString::toStringUtf8).collect(Collectors.toList());
304+
case Array:
305+
List<List<?>> array = new ArrayList<>();
306+
ArrayArray arrArray = fieldData.getScalars().getArrayData();
307+
boolean nullable = validData != null && validData.size() == arrArray.getDataCount();
308+
for (int i = 0; i < arrArray.getDataCount(); i++) {
309+
if (nullable && validData.get(i) == Boolean.FALSE) {
310+
array.add(null);
311+
} else {
312+
ScalarField rowData = arrArray.getData(i);
313+
array.add(getScalarData(arrArray.getElementType(), rowData, null));
314+
}
315+
}
316+
return array;
311317
default:
312318
return new ArrayList<>();
313319
}

‎sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java‎

Lines changed: 34 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070

7171
import java.nio.ByteBuffer;
7272
import java.util.*;
73+
import java.util.function.Function;
7374

7475
@Testcontainers(disabledWithoutDocker = true)
7576
class MilvusClientV2DockerTest {
@@ -1892,6 +1893,13 @@ void testNullableAndDefaultValue() {
18921893
.isNullable(Boolean.TRUE)
18931894
.maxLength(100)
18941895
.build());
1896+
collectionSchema.addField(AddFieldReq.builder()
1897+
.fieldName("arr")
1898+
.dataType(DataType.Array)
1899+
.elementType(DataType.Int32)
1900+
.isNullable(Boolean.TRUE)
1901+
.maxCapacity(100)
1902+
.build());
18951903

18961904
List<IndexParam> indexParams = new ArrayList<>();
18971905
indexParams.add(IndexParam.builder()
@@ -1920,7 +1928,11 @@ void testNullableAndDefaultValue() {
19201928
} else {
19211929
// row.add("flag", JsonNull.INSTANCE);
19221930
row.addProperty("desc", "AAA");
1931+
1932+
List<Integer> arr = Arrays.asList(5, 6);
1933+
row.add("arr", JsonUtils.toJsonTree(arr));
19231934
}
1935+
19241936
data.add(row);
19251937
}
19261938

@@ -1930,26 +1942,38 @@ void testNullableAndDefaultValue() {
19301942
.build());
19311943
Assertions.assertEquals(10, insertResp.getInsertCnt());
19321944

1945+
Function<Map<String, Object>, Void> checkFunc =
1946+
entity -> {
1947+
long id = (long)entity.get("id");
1948+
if (id%2 == 0) {
1949+
Assertions.assertEquals((int)id, entity.get("flag"));
1950+
Assertions.assertNull(entity.get("desc"));
1951+
Assertions.assertNull(entity.get("arr"));
1952+
} else {
1953+
Assertions.assertEquals(10, entity.get("flag"));
1954+
Assertions.assertEquals("AAA", entity.get("desc"));
1955+
Object obj = entity.get("arr");
1956+
Assertions.assertInstanceOf(List.class, obj);
1957+
List<Integer> arr = (List<Integer>)obj;
1958+
Assertions.assertEquals(2, arr.size());
1959+
Assertions.assertEquals(5, arr.get(0));
1960+
Assertions.assertEquals(6, arr.get(1));
1961+
}
1962+
return null;
1963+
};
19331964
// query
19341965
QueryResp queryResp = client.query(QueryReq.builder()
19351966
.collectionName(randomCollectionName)
19361967
.filter("id >= 0")
1937-
.outputFields(Arrays.asList("desc", "flag"))
1968+
.outputFields(Arrays.asList("desc", "flag", "arr"))
19381969
.consistencyLevel(ConsistencyLevel.STRONG)
19391970
.build());
19401971
List<QueryResp.QueryResult> queryResults = queryResp.getQueryResults();
19411972
Assertions.assertEquals(10, queryResults.size());
19421973
System.out.println("Query results:");
19431974
for (QueryResp.QueryResult result : queryResults) {
19441975
Map<String, Object> entity = result.getEntity();
1945-
long id = (long)entity.get("id");
1946-
if (id%2 == 0) {
1947-
Assertions.assertEquals((int)id, entity.get("flag"));
1948-
Assertions.assertNull(entity.get("desc"));
1949-
} else {
1950-
Assertions.assertEquals(10, entity.get("flag"));
1951-
Assertions.assertEquals("AAA", entity.get("desc"));
1952-
}
1976+
checkFunc.apply(entity);
19531977
System.out.println(result);
19541978
}
19551979

@@ -1970,13 +1994,7 @@ void testNullableAndDefaultValue() {
19701994
for (SearchResp.SearchResult result : firstResults) {
19711995
long id = (long)result.getId();
19721996
Map<String, Object> entity = result.getEntity();
1973-
if (id%2 == 0) {
1974-
Assertions.assertEquals((int)id, entity.get("flag"));
1975-
Assertions.assertNull(entity.get("desc"));
1976-
} else {
1977-
Assertions.assertEquals(10, entity.get("flag"));
1978-
Assertions.assertEquals("AAA", entity.get("desc"));
1979-
}
1997+
checkFunc.apply(entity);
19801998
System.out.println(result);
19811999
}
19822000
}

0 commit comments

Comments
 (0)