diff --git a/docker-compose.yml b/docker-compose.yml index d05757f33..85b6fdcef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,7 +77,7 @@ services: standaloneslave: container_name: milvus-javasdk-test-slave-standalone - image: milvusdb/milvus:v2.5.8 + image: milvusdb/milvus:v2.5.11 command: ["milvus", "run", "standalone"] environment: ETCD_ENDPOINTS: etcdslave:2379 diff --git a/pom.xml b/pom.xml index f445466bc..793d54bbb 100644 --- a/pom.xml +++ b/pom.xml @@ -99,13 +99,13 @@ 3.1.3 1.1.0 5.10.1 - 2.10.1 + 2.13.1 1.9.10 4.11.0 1.19.8 2.12.0 32.1.3-jre - 2.21.1 + 2.38.0 3.0.24 diff --git a/sdk-core/src/main/java/io/milvus/v2/client/MilvusClientV2.java b/sdk-core/src/main/java/io/milvus/v2/client/MilvusClientV2.java index bc5d35e45..168bdb488 100644 --- a/sdk-core/src/main/java/io/milvus/v2/client/MilvusClientV2.java +++ b/sdk-core/src/main/java/io/milvus/v2/client/MilvusClientV2.java @@ -382,7 +382,7 @@ public Boolean getLoadState(GetLoadStateReq request) { /** * Get information of all replicas from a collection. * - * @param request {@link DescribeReplicasReq} + * @param request describe replicas request */ public DescribeReplicasResp describeReplicas(DescribeReplicasReq request) { return rpcUtils.retry(()->collectionService.describeReplicas(this.getRpcStub(), request)); @@ -528,8 +528,8 @@ public SearchResp hybridSearch(HybridSearchReq request) { * Get queryIterator based on scalar field(s) filtered by boolean expression. * Note that the order of the returned entities cannot be guaranteed. * - * @param request {@link QueryIteratorReq} - * @return {status:result code,data: QueryIterator} + * @param request query iterator request + * @return QueryIterator */ public QueryIterator queryIterator(QueryIteratorReq request) { return rpcUtils.retry(()->vectorService.queryIterator(this.getRpcStub(), request)); @@ -538,8 +538,8 @@ public QueryIterator queryIterator(QueryIteratorReq request) { /** * Get searchIterator based on a vector field. Use expression to do filtering before search. * - * @param request {@link SearchIteratorReq} - * @return {status:result code, data: SearchIterator} + * @param request search iterator request + * @return SearchIterator */ public SearchIterator searchIterator(SearchIteratorReq request) { return rpcUtils.retry(()->vectorService.searchIterator(this.getRpcStub(), request)); @@ -548,13 +548,24 @@ public SearchIterator searchIterator(SearchIteratorReq request) { /** * Get searchIteratorV2 based on a vector field. Use expression to do filtering before search. * - * @param request {@link SearchIteratorReqV2} - * @return {status:result code, data: SearchIteratorV2} + * @param request search iterator request V2 + * @return SearchIteratorV2 */ public SearchIteratorV2 searchIteratorV2(SearchIteratorReqV2 request) { return rpcUtils.retry(()->vectorService.searchIteratorV2(this.getRpcStub(), request)); } + /** + * Run analyzer. Return result tokens of analysis. + * Milvus server supports this interface from v2.5.11 + * + * @param request run analyzer request + * @return RunAnalyzerResp + */ + public RunAnalyzerResp runAnalyzer(RunAnalyzerReq request) { + return rpcUtils.retry(()->vectorService.runAnalyzer(this.getRpcStub(), request)); + } + ///////////////////////////////////////////////////////////////////////////////////////////// // Partition Operations ///////////////////////////////////////////////////////////////////////////////////////////// diff --git a/sdk-core/src/main/java/io/milvus/v2/service/vector/VectorService.java b/sdk-core/src/main/java/io/milvus/v2/service/vector/VectorService.java index 45ebf803d..862419049 100644 --- a/sdk-core/src/main/java/io/milvus/v2/service/vector/VectorService.java +++ b/sdk-core/src/main/java/io/milvus/v2/service/vector/VectorService.java @@ -19,7 +19,9 @@ package io.milvus.v2.service.vector; +import com.google.protobuf.ByteString; import io.milvus.common.utils.GTsDict; +import io.milvus.common.utils.JsonUtils; import io.milvus.exception.ParamException; import io.milvus.grpc.*; import io.milvus.orm.iterator.*; @@ -303,4 +305,51 @@ public GetResp get(MilvusServiceGrpc.MilvusServiceBlockingStub blockingStub, Get .getResults(queryResp.getQueryResults()) .build(); } + + public RunAnalyzerResp runAnalyzer(MilvusServiceGrpc.MilvusServiceBlockingStub blockingStub, RunAnalyzerReq request) { + String title = "RunAnalyzer"; + if (request.getTexts().isEmpty()) { + throw new MilvusClientException(ErrorCode.INVALID_PARAMS, "Texts list is empty."); + } + + RunAnalyzerRequest.Builder builder = RunAnalyzerRequest.newBuilder(); + List byteStrings = new ArrayList<>(); + for (String text : request.getTexts()) { + byteStrings.add(ByteString.copyFrom(text.getBytes())); + } + + String params = JsonUtils.toJson(request.getAnalyzerParams()); + System.out.println(params); + RunAnalyzerRequest runRequest = builder.addAllPlaceholder(byteStrings) + .setAnalyzerParams(params) + .setWithDetail(request.getWithDetail()) + .setWithHash(request.getWithHash()) + .build(); + RunAnalyzerResponse response = blockingStub.runAnalyzer(runRequest); + rpcUtils.handleResponse(title, response.getStatus()); + + List toResults = new ArrayList<>(); + List results = response.getResultsList(); + results.forEach((item)->{ + List toTokens = new ArrayList<>(); + List tokens = item.getTokensList(); + tokens.forEach((token)->{ + toTokens.add(RunAnalyzerResp.AnalyzerToken.builder() + .token(token.getToken()) + .startOffset(token.getStartOffset()) + .endOffset(token.getEndOffset()) + .position(token.getPosition()) + .positionLength(token.getPositionLength()) + .hash(token.getHash() & 0xFFFFFFFFL) + .build()); + }); + toResults.add(RunAnalyzerResp.AnalyzerResult.builder() + .tokens(toTokens) + .build()); + }); + + return RunAnalyzerResp.builder() + .results(toResults) + .build(); + } } diff --git a/sdk-core/src/main/java/io/milvus/v2/service/vector/request/RunAnalyzerReq.java b/sdk-core/src/main/java/io/milvus/v2/service/vector/request/RunAnalyzerReq.java new file mode 100644 index 000000000..e0bb184f9 --- /dev/null +++ b/sdk-core/src/main/java/io/milvus/v2/service/vector/request/RunAnalyzerReq.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.milvus.v2.service.vector.request; + +import lombok.Builder; +import lombok.Data; +import lombok.experimental.SuperBuilder; + +import java.util.*; + +@Data +@SuperBuilder +public class RunAnalyzerReq { + @Builder.Default + private List texts = new ArrayList<>(); + @Builder.Default + private Map analyzerParams = new HashMap<>(); + @Builder.Default + private Boolean withDetail = Boolean.FALSE; + @Builder.Default + private Boolean withHash = Boolean.FALSE; +} diff --git a/sdk-core/src/main/java/io/milvus/v2/service/vector/response/RunAnalyzerResp.java b/sdk-core/src/main/java/io/milvus/v2/service/vector/response/RunAnalyzerResp.java new file mode 100644 index 000000000..ea448b805 --- /dev/null +++ b/sdk-core/src/main/java/io/milvus/v2/service/vector/response/RunAnalyzerResp.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.milvus.v2.service.vector.response; + +import lombok.Builder; +import lombok.Data; +import lombok.experimental.SuperBuilder; + +import java.util.ArrayList; +import java.util.List; + +@Data +@SuperBuilder +public class RunAnalyzerResp { + @Builder.Default + List results = new ArrayList<>(); + + @Data + @SuperBuilder + public static final class AnalyzerResult { + @Builder.Default + List tokens = new ArrayList<>(); + } + + @Data + @SuperBuilder + public static final class AnalyzerToken { + private String token; + private Long startOffset; + private Long endOffset; + private Long position; + private Long positionLength; + private Long hash; + } +} diff --git a/sdk-core/src/test/java/io/milvus/client/MilvusClientDockerTest.java b/sdk-core/src/test/java/io/milvus/client/MilvusClientDockerTest.java index c452acca3..b2710385a 100644 --- a/sdk-core/src/test/java/io/milvus/client/MilvusClientDockerTest.java +++ b/sdk-core/src/test/java/io/milvus/client/MilvusClientDockerTest.java @@ -75,7 +75,7 @@ class MilvusClientDockerTest { private static final TestUtils utils = new TestUtils(DIMENSION); @Container - private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.5.8"); + private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.5.11"); @BeforeAll public static void setUp() { diff --git a/sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java b/sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java index 5c6e3f6b7..8e4a9c969 100644 --- a/sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java +++ b/sdk-core/src/test/java/io/milvus/v2/client/MilvusClientV2DockerTest.java @@ -81,7 +81,7 @@ class MilvusClientV2DockerTest { private static final TestUtils utils = new TestUtils(DIMENSION); @Container - private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.5.8"); + private static final MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.5.11"); @BeforeAll public static void setUp() { @@ -2377,4 +2377,76 @@ void testReplica() { Assertions.assertFalse(replica.getLeaderAddress().isEmpty()); Assertions.assertNotEquals(0L, replica.getLeaderID()); } + + @Test + void testRunAnalyzer() { + List texts = new ArrayList<>(); + texts.add("Analyzers (tokenizers) for multi languages"); + texts.add("2.5 to take advantage of enhancements and fixes!"); + + Map analyzerParams = new HashMap<>(); + analyzerParams.put("tokenizer", "standard"); + analyzerParams.put("filter", + Arrays.asList("lowercase", + new HashMap() {{ + put("type", "stop"); + put("stop_words", Arrays.asList("to", "of", "for", "the")); + }})); + + RunAnalyzerResp resp = client.runAnalyzer(RunAnalyzerReq.builder() + .texts(texts) + .analyzerParams(analyzerParams) + .withDetail(true) + .withHash(true) + .build()); + + List results = resp.getResults(); + Assertions.assertEquals(texts.size(), results.size()); + + { + List tokens1 = Arrays.asList("analyzers", "tokenizers", "multi", "languages"); + List startOffset1 = Arrays.asList(0L, 11L, 27L, 33L); + List endOffset1 = Arrays.asList(9L, 21L, 32L, 42L); + List position1 = Arrays.asList(0L, 1L, 3L, 4L); + List positionLen1 = Arrays.asList(1L, 1L, 1L, 1L); + List hash1 = Arrays.asList(1356745679L, 4089107865L, 3314631429L, 2698072953L); + + List outTokens1 = results.get(0).getTokens(); + System.out.printf("%d tokens%n", outTokens1.size()); + Assertions.assertEquals(tokens1.size(), outTokens1.size()); + for (int i = 0; i < outTokens1.size(); i++) { + RunAnalyzerResp.AnalyzerToken token = outTokens1.get(i); + System.out.println(token); + Assertions.assertEquals(tokens1.get(i), token.getToken()); + Assertions.assertEquals(startOffset1.get(i), token.getStartOffset()); + Assertions.assertEquals(endOffset1.get(i), token.getEndOffset()); + Assertions.assertEquals(position1.get(i), token.getPosition()); + Assertions.assertEquals(positionLen1.get(i), token.getPositionLength()); + Assertions.assertEquals(hash1.get(i), token.getHash()); + } + } + + { + List tokens2 = Arrays.asList("2", "5", "take", "advantage", "enhancements", "and", "fixes"); + List startOffset2 = Arrays.asList(0L, 2L, 7L, 12L, 25L, 38L, 42L); + List endOffset2 = Arrays.asList(1L, 3L, 11L, 21L, 37L, 41L, 47L); + List position2 = Arrays.asList(0L, 1L, 3L, 4L, 6L, 7L, 8L); + List positionLen2 = Arrays.asList(1L, 1L, 1L, 1L, 1L, 1L, 1L); + List hash2 = Arrays.asList(450215437L, 2226203566L, 937258619L, 697180577L, 3403941281L, 133536621L, 488262645L); + + List outTokens2 = results.get(1).getTokens(); + System.out.printf("%d tokens%n", outTokens2.size()); + Assertions.assertEquals(tokens2.size(), outTokens2.size()); + for (int i = 0; i < outTokens2.size(); i++) { + RunAnalyzerResp.AnalyzerToken token = outTokens2.get(i); + System.out.println(token); + Assertions.assertEquals(tokens2.get(i), token.getToken()); + Assertions.assertEquals(startOffset2.get(i), token.getStartOffset()); + Assertions.assertEquals(endOffset2.get(i), token.getEndOffset()); + Assertions.assertEquals(position2.get(i), token.getPosition()); + Assertions.assertEquals(positionLen2.get(i), token.getPositionLength()); + Assertions.assertEquals(hash2.get(i), token.getHash()); + } + } + } }