Skip to content

Fix a regression caused by timestamp refine #1467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
yhmo wants to merge 1 commit into from
Closed

Fix a regression caused by timestamp refine #1467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions sdk-core/src/main/java/io/milvus/common/utils/GTsDict.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,19 +52,19 @@ public static String CombineCollectionName(String databaseName, String collectio

private ConcurrentMap<String, Long> tsDict = new ConcurrentHashMap<>();

public void updateCollectionTs(String collectionName, long ts) {
public void updateCollectionTs(String name, long ts) {
// If the collection name exists, use its value to compare to the input ts,
// only when the input ts is larger than the existing value, replace it with the input ts.
// If the collection name doesn't exist, directly set the input value.
tsDict.compute(collectionName, (key, value) -> (value == null) ? ts : ((ts > value) ? ts : value));
tsDict.compute(name, (key, value) -> (value == null) ? ts : ((ts > value) ? ts : value));
}

public Long getCollectionTs(String collectionName) {
return tsDict.get(collectionName);
public Long getCollectionTs(String name) {
return tsDict.get(name);
}

public void removeCollectionTs(String collectionName) {
tsDict.remove(collectionName);
public void removeCollectionTs(String name) {
tsDict.remove(name);
}

public void cleanAllCollectionTs() {
Expand Down
41 changes: 25 additions & 16 deletions sdk-core/src/main/java/io/milvus/param/ParamUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -849,14 +849,16 @@ public static void compatibleSearchParams(Map<String, Object> searchParams, Sear

@SuppressWarnings("unchecked")
public static SearchRequest convertSearchParam(@NonNull SearchParam requestParam) throws ParamException {
String dbName = requestParam.getDatabaseName();
String collectionName = requestParam.getCollectionName();
SearchRequest.Builder builder = SearchRequest.newBuilder()
.setCollectionName(requestParam.getCollectionName());
.setCollectionName(collectionName);

if (!requestParam.getPartitionNames().isEmpty()) {
requestParam.getPartitionNames().forEach(builder::addPartitionNames);
}
if (StringUtils.isNotEmpty(requestParam.getDatabaseName())) {
builder.setDbName(requestParam.getDatabaseName());
if (StringUtils.isNotEmpty(dbName)) {
builder.setDbName(dbName);
}

// prepare target vectors
Expand Down Expand Up @@ -946,7 +948,7 @@ public static SearchRequest convertSearchParam(@NonNull SearchParam requestParam
builder.setDsl(requestParam.getExpr());
}

long guaranteeTimestamp = getGuaranteeTimestamp(requestParam.getConsistencyLevel(), requestParam.getCollectionName());
long guaranteeTimestamp = getGuaranteeTimestamp(requestParam.getConsistencyLevel(), dbName, collectionName);
builder.setTravelTimestamp(requestParam.getTravelTimestamp()); // deprecated
builder.setGuaranteeTimestamp(guaranteeTimestamp);

Expand Down Expand Up @@ -1010,14 +1012,16 @@ public static SearchRequest convertAnnSearchParam(@NonNull AnnSearchParam annSea
}

public static HybridSearchRequest convertHybridSearchParam(@NonNull HybridSearchParam requestParam) throws ParamException {
String dbName = requestParam.getDatabaseName();
String collectionName = requestParam.getCollectionName();
HybridSearchRequest.Builder builder = HybridSearchRequest.newBuilder()
.setCollectionName(requestParam.getCollectionName());
.setCollectionName(collectionName);

if (!requestParam.getPartitionNames().isEmpty()) {
requestParam.getPartitionNames().forEach(builder::addPartitionNames);
}
if (StringUtils.isNotEmpty(requestParam.getDatabaseName())) {
builder.setDbName(requestParam.getDatabaseName());
if (StringUtils.isNotEmpty(dbName)) {
builder.setDbName(dbName);
}

for (AnnSearchParam req : requestParam.getSearchRequests()) {
Expand Down Expand Up @@ -1063,7 +1067,7 @@ public static HybridSearchRequest convertHybridSearchParam(@NonNull HybridSearch
requestParam.getOutFields().forEach(builder::addOutputFields);
}

long guaranteeTimestamp = getGuaranteeTimestamp(requestParam.getConsistencyLevel(), requestParam.getCollectionName());
long guaranteeTimestamp = getGuaranteeTimestamp(requestParam.getConsistencyLevel(), dbName, collectionName);
builder.setGuaranteeTimestamp(guaranteeTimestamp);

if (requestParam.getConsistencyLevel() == null) {
Expand All @@ -1076,18 +1080,20 @@ public static HybridSearchRequest convertHybridSearchParam(@NonNull HybridSearch
}

public static QueryRequest convertQueryParam(@NonNull QueryParam requestParam) {
String dbName = requestParam.getDatabaseName();
String collectionName = requestParam.getCollectionName();
boolean useDefaultConsistency = (requestParam.getConsistencyLevel() == null);
long guaranteeTimestamp = getGuaranteeTimestamp(requestParam.getConsistencyLevel(), requestParam.getCollectionName());
long guaranteeTimestamp = getGuaranteeTimestamp(requestParam.getConsistencyLevel(), dbName, collectionName);
QueryRequest.Builder builder = QueryRequest.newBuilder()
.setCollectionName(requestParam.getCollectionName())
.setCollectionName(collectionName)
.addAllPartitionNames(requestParam.getPartitionNames())
.addAllOutputFields(requestParam.getOutFields())
.setExpr(requestParam.getExpr())
.setTravelTimestamp(requestParam.getTravelTimestamp())
.setGuaranteeTimestamp(guaranteeTimestamp);

if (StringUtils.isNotEmpty(requestParam.getDatabaseName())) {
builder.setDbName(requestParam.getDatabaseName());
if (StringUtils.isNotEmpty(dbName)) {
builder.setDbName(dbName);
}

// a new parameter from v2.2.9, if user didn't specify consistency level, set this parameter to true
Expand Down Expand Up @@ -1124,17 +1130,20 @@ public static QueryRequest convertQueryParam(@NonNull QueryParam requestParam) {
return builder.build();
}

private static long getGuaranteeTimestamp(ConsistencyLevelEnum consistencyLevel, String collectionName){
private static long getGuaranteeTimestamp(ConsistencyLevelEnum consistencyLevel, String dbName, String collectionName){
if(consistencyLevel == null){
Long ts = GTsDict.getInstance().getCollectionTs(collectionName);
String key = GTsDict.CombineCollectionName(dbName, collectionName);
Long ts = GTsDict.getInstance().getCollectionTs(key);
return (ts == null) ? 1L : ts;
}
switch (consistencyLevel){
case STRONG:
return 0L;
case SESSION:
Long ts = GTsDict.getInstance().getCollectionTs(collectionName);
case SESSION: {
String key = GTsDict.CombineCollectionName(dbName, collectionName);
Long ts = GTsDict.getInstance().getCollectionTs(key);
return (ts == null) ? 1L : ts;
}
case BOUNDED:
return 2L; // let server side to determine the bounded time
default:
Expand Down
26 changes: 23 additions & 3 deletions sdk-core/src/main/java/io/milvus/v2/service/vector/request/AnnSearchReq.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,20 @@ public class AnnSearchReq {
@Deprecated
private int topK = 0;
@Builder.Default
private long limit = 0L;
private long limit = 0L; // deprecated, replaced by limit
@Builder.Default
private String expr = "";
@Deprecated
private String expr = ""; // deprecated, replaced by filter
@Builder.Default
private String filter = "";
private List<BaseVector> vectors;
private String params;

@Builder.Default
private IndexParam.MetricType metricType = null;

public static abstract class AnnSearchReqBuilder<C extends AnnSearchReq, B extends AnnSearchReq.AnnSearchReqBuilder<C, B>> {
// topK is deprecated, topK and limit must be the same value
// topK is deprecated replaced by limit, topK and limit must be the same value
public B topK(int val) {
this.topK$value = val;
this.topK$set = true;
Expand All @@ -61,5 +64,22 @@ public B limit(long val) {
this.limit$set = true;
return self();
}

// expr is deprecated replaced by filter, expr and filter must be the same value
public B expr(String val) {
this.expr$value = val;
this.expr$set = true;
this.filter$value = val;
this.filter$set = true;
return self();
}

public B filter(String val) {
this.expr$value = val;
this.expr$set = true;
this.filter$value = val;
this.filter$set = true;
return self();
}
}
}
44 changes: 29 additions & 15 deletions sdk-core/src/main/java/io/milvus/v2/utils/VectorUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,17 @@
public class VectorUtils {

public QueryRequest ConvertToGrpcQueryRequest(QueryReq request){
String dbName = request.getDatabaseName();
String collectionName = request.getCollectionName();
long guaranteeTimestamp = getGuaranteeTimestamp(request.getConsistencyLevel(), dbName, collectionName);
QueryRequest.Builder builder = QueryRequest.newBuilder()
.setCollectionName(request.getCollectionName())
.setCollectionName(collectionName)
.addAllPartitionNames(request.getPartitionNames())
.addAllOutputFields(request.getOutputFields())
.setGuaranteeTimestamp(guaranteeTimestamp)
.setExpr(request.getFilter());
if (StringUtils.isNotEmpty(request.getDatabaseName())) {
builder.setDbName(request.getDatabaseName());
if (StringUtils.isNotEmpty(dbName)) {
builder.setDbName(dbName);
}

if (request.getFilter() != null && !request.getFilter().isEmpty()) {
Expand Down Expand Up @@ -94,17 +98,20 @@ public QueryRequest ConvertToGrpcQueryRequest(QueryReq request){

}

private static long getGuaranteeTimestamp(ConsistencyLevel consistencyLevel, String collectionName){
private static long getGuaranteeTimestamp(ConsistencyLevel consistencyLevel, String dbName, String collectionName){
if(consistencyLevel == null){
Long ts = GTsDict.getInstance().getCollectionTs(collectionName);
String key = GTsDict.CombineCollectionName(dbName, collectionName);
Long ts = GTsDict.getInstance().getCollectionTs(key);
return (ts == null) ? 1L : ts;
}
switch (consistencyLevel){
case STRONG:
return 0L;
case SESSION:
Long ts = GTsDict.getInstance().getCollectionTs(collectionName);
return (ts == null) ? 1L : ts;
case SESSION: {
String key = GTsDict.CombineCollectionName(dbName, collectionName);
Long ts = GTsDict.getInstance().getCollectionTs(key);
return (ts == null) ? 1L : ts;
}
case BOUNDED:
return 2L; // let server side to determine the bounded time
default:
Expand Down Expand Up @@ -138,14 +145,16 @@ private static ByteString convertPlaceholder(List<Object> data, PlaceholderType
}

public SearchRequest ConvertToGrpcSearchRequest(SearchReq request) {
String dbName = request.getDatabaseName();
String collectionName = request.getCollectionName();
SearchRequest.Builder builder = SearchRequest.newBuilder()
.setCollectionName(request.getCollectionName());
.setCollectionName(collectionName);
if (!request.getPartitionNames().isEmpty()) {
request.getPartitionNames().forEach(builder::addPartitionNames);
}

if (StringUtils.isNotEmpty(request.getDatabaseName())) {
builder.setDbName(request.getDatabaseName());
if (StringUtils.isNotEmpty(dbName)) {
builder.setDbName(dbName);
}

// prepare target, the input could be vectors or string list for doc-in-doc-out
Expand Down Expand Up @@ -259,7 +268,7 @@ public SearchRequest ConvertToGrpcSearchRequest(SearchReq request) {
}
builder.setGuaranteeTimestamp(guaranteeTimestamp);
} else {
long guaranteeTimestamp = getGuaranteeTimestamp(request.getConsistencyLevel(), request.getCollectionName());
long guaranteeTimestamp = getGuaranteeTimestamp(request.getConsistencyLevel(), dbName, collectionName);
builder.setGuaranteeTimestamp(guaranteeTimestamp);
}

Expand Down Expand Up @@ -442,14 +451,16 @@ public static SearchRequest convertAnnSearchParam(@NonNull AnnSearchReq annSearc
}

public HybridSearchRequest ConvertToGrpcHybridSearchRequest(HybridSearchReq request) {
String dbName = request.getDatabaseName();
String collectionName = request.getCollectionName();
HybridSearchRequest.Builder builder = HybridSearchRequest.newBuilder()
.setCollectionName(request.getCollectionName());
.setCollectionName(collectionName);

if (request.getPartitionNames() != null && !request.getPartitionNames().isEmpty()) {
request.getPartitionNames().forEach(builder::addPartitionNames);
}
if (StringUtils.isNotEmpty(request.getDatabaseName())) {
builder.setDbName(request.getDatabaseName());
if (StringUtils.isNotEmpty(dbName)) {
builder.setDbName(dbName);
}

if (request.getSearchRequests() == null || request.getSearchRequests().isEmpty()) {
Expand Down Expand Up @@ -505,6 +516,9 @@ public HybridSearchRequest ConvertToGrpcHybridSearchRequest(HybridSearchReq requ
request.getOutFields().forEach(builder::addOutputFields);
}

long guaranteeTimestamp = getGuaranteeTimestamp(request.getConsistencyLevel(), dbName, collectionName);
builder.setGuaranteeTimestamp(guaranteeTimestamp);

if (request.getConsistencyLevel() == null) {
builder.setUseDefaultConsistency(true);
} else {
Expand Down
Loading
Loading