Skip to content

Support NGRAM and BoostRanker #1640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sre-ci-robot merged 1 commit into from
Oct 11, 2025
Merged

Support NGRAM and BoostRanker #1640

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions examples/src/main/java/io/milvus/v2/HybridSearchExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public class HybridSearchExample {
private static final String SPARSE_VECTOR_FIELD = "sparse_vector";
private static final IndexParam.MetricType SPARSE_VECTOR_METRIC = IndexParam.MetricType.IP;

private void createCollection() {
private static void createCollection() {
client.dropCollection(DropCollectionReq.builder()
.collectionName(COLLECTION_NAME)
.build());
Expand Down Expand Up @@ -147,7 +147,7 @@ private void createCollection() {
System.out.println("Collection created");
}

private void insertData() {
private static void insertData() {
long idCount = 0;
int rowCount = 10000;
// Insert entities by rows
Expand All @@ -167,19 +167,20 @@ private void insertData() {
.collectionName(COLLECTION_NAME)
.data(rows)
.build());

System.out.printf("%d entities inserted by rows\n", rowCount);
printRowCount();
}

private void hybridSearch() {
private static void printRowCount() {
// Get row count, set ConsistencyLevel.STRONG to sync the data to query node so that data is visible
QueryResp countR = client.query(QueryReq.builder()
.collectionName(COLLECTION_NAME)
.outputFields(Collections.singletonList("count(*)"))
.consistencyLevel(ConsistencyLevel.STRONG)
.build());
System.out.printf("%d rows persisted\n", (long)countR.getQueryResults().get(0).getEntity().get("count(*)"));
}

private static void hybridSearch() {
// Search on multiple vector fields
int NQ = 2;
List<BaseVector> floatVectors = new ArrayList<>();
Expand Down Expand Up @@ -229,19 +230,18 @@ private void hybridSearch() {
}
}

private void dropCollection() {
private static void dropCollection() {
client.dropCollection(DropCollectionReq.builder()
.collectionName(COLLECTION_NAME)
.build());
System.out.println("Collection dropped");
}

public static void main(String[] args) {
io.milvus.v2.HybridSearchExample example = new io.milvus.v2.HybridSearchExample();
example.createCollection();
example.insertData();
example.hybridSearch();
example.dropCollection();
createCollection();
insertData();
hybridSearch();
dropCollection();

client.close();
}
Expand Down
29 changes: 28 additions & 1 deletion examples/src/main/java/io/milvus/v2/JsonFieldExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,32 @@ public static void main(String[] args) {
.metricType(IndexParam.MetricType.COSINE)
.build());

// Create INVERTED index for a specific entry of JSON field
// Index for JSON field is supported from milvus v2.5.7 and fully supported in v2.5.13+
// Read the doc for more info: https://milvus.io/docs/json-indexing.md
Map<String,Object> p1 = new HashMap<>();
p1.put("json_path", "metadata[\"flags\"]");
p1.put("json_cast_type", "array_double");
indexes.add(IndexParam.builder()
.fieldName(JSON_FIELD)
.indexType(IndexParam.IndexType.INVERTED)
.extraParams(p1)
.build());

// Create NGRAM index for a specific entry of JSON field
// NGRAM index for JSON field is supported from milvus v2.6.2
// Read the doc for more info: https://milvus.io/docs/ngram.md
Map<String,Object> p2 = new HashMap<>();
p2.put("json_path","metadata[\"path\"]");
p2.put("json_cast_type", "varchar");
p2.put("min_gram", 3);
p2.put("max_gram", 5);
indexes.add(IndexParam.builder()
.fieldName(JSON_FIELD)
.indexType(IndexParam.IndexType.NGRAM)
.extraParams(p2)
.build());

CreateCollectionReq requestCreate = CreateCollectionReq.builder()
.collectionName(COLLECTION_NAME)
.collectionSchema(collectionSchema)
Expand All @@ -121,7 +147,7 @@ public static void main(String[] args) {
// Note: for JSON field, always construct a real JsonObject
// don't use row.addProperty(JSON_FIELD, strContent) since the value is treated as a string, not a JsonObject
JsonObject metadata = new JsonObject();
metadata.addProperty("path", String.format("\\root/abc/path_%d", i));
metadata.addProperty("path", String.format("\\root/abc_%d/path_%d", i, i));
metadata.addProperty("size", i);
if (i%7 == 0) {
metadata.addProperty("special", true);
Expand Down Expand Up @@ -197,6 +223,7 @@ public static void main(String[] args) {
queryWithExpr(client, "JSON_CONTAINS(metadata[\"flags\"], 9)");
queryWithExpr(client, "JSON_CONTAINS_ANY(metadata[\"flags\"], [8, 9, 10])");
queryWithExpr(client, "JSON_CONTAINS_ALL(metadata[\"flags\"], [8, 9, 10])");
queryWithExpr(client, "metadata[\"path\"] LIKE \"%c_5%\"");
queryWithExpr(client, "dynamic1 < 2.0");

client.close();
Expand Down
267 changes: 267 additions & 0 deletions examples/src/main/java/io/milvus/v2/RankerExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,267 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package io.milvus.v2;

import com.google.gson.JsonObject;
import io.milvus.common.clientenum.FunctionType;
import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.common.ConsistencyLevel;
import io.milvus.v2.common.DataType;
import io.milvus.v2.common.IndexParam;
import io.milvus.v2.service.collection.request.AddFieldReq;
import io.milvus.v2.service.collection.request.CreateCollectionReq;
import io.milvus.v2.service.collection.request.DropCollectionReq;
import io.milvus.v2.service.vector.request.FunctionScore;
import io.milvus.v2.service.vector.request.InsertReq;
import io.milvus.v2.service.vector.request.QueryReq;
import io.milvus.v2.service.vector.request.SearchReq;
import io.milvus.v2.service.vector.request.data.EmbeddedText;
import io.milvus.v2.service.vector.request.ranker.BoostRanker;
import io.milvus.v2.service.vector.request.ranker.DecayRanker;
import io.milvus.v2.service.vector.response.QueryResp;
import io.milvus.v2.service.vector.response.SearchResp;

import java.util.*;

public class RankerExample {
private static final MilvusClientV2 client;

static {
ConnectConfig config = ConnectConfig.builder()
.uri("http://localhost:19530")
.build();
client = new MilvusClientV2(config);
}

private static final String COLLECTION_NAME = "java_sdk_example_ranker_v2";
private static final String NAME_FIELD = "name";
private static final String BIRTH_YEAR_FIELD = "birth_year";
private static final String LIFESPAN_FIELD = "lifespan";
private static final String SPARSE_VECTOR_FIELD = "sparse_vector";

private static class Person {
public String name;
public int fromYear;
public int toYear;
public Person(String name, int from, int to) {
this.name = name;
this.fromYear = from;
this.toYear = to;
}
}

private static List<Person> genData() {
List<Person> persons = new ArrayList<>();
persons.add(new Person("Isaac Newton", 1643, 1727));
persons.add(new Person("Albert Einstein", 1879, 1955));
persons.add(new Person("Marie Curie", 1867, 1934));
persons.add(new Person("Charles Darwin", 1809, 1882));
persons.add(new Person("Galileo Galilei", 1564, 1642));
persons.add(new Person("Nikola Tesla", 1856, 1943));
persons.add(new Person("James Clerk Maxwell", 1831, 1879));
persons.add(new Person("Thomas Edison", 1847, 1931));
persons.add(new Person("Alexander Fleming", 1881, 1955));
persons.add(new Person("Louis Pasteur", 1822, 1895));
persons.add(new Person("Werner Heisenberg", 1901, 1976));
persons.add(new Person("Stephen Hawking", 1942, 2018));
persons.add(new Person("Dmitri Mendeleev", 1834, 1907));
persons.add(new Person("Max Planck", 1858, 1947));
persons.add(new Person("Niels Bohr", 1885, 1962));
persons.add(new Person("Richard Feynman", 1918, 1988));
persons.add(new Person("Carl Sagan", 1934, 1996));
persons.add(new Person("Francis Crick", 1916, 2004));
persons.add(new Person("Rosalind Franklin", 1920, 1958));
persons.add(new Person("Edwin Hubble", 1889, 1953));
persons.add(new Person("Linus Pauling", 1901, 1994));
persons.add(new Person("Alan Turing", 1912, 1954));
persons.add(new Person("Guglielmo Marconi", 1874, 1937));
persons.add(new Person("Michael Faraday", 1791, 1867));
persons.add(new Person("Enrico Fermi", 1901, 1954));
persons.add(new Person("Johannes Kepler", 1571, 1630));
persons.add(new Person("Edwin Schrödinger", 1887, 1961));
persons.add(new Person("Werner von Braun", 1912, 1977));
persons.add(new Person("Albert Hofmann", 1906, 2008));
persons.add(new Person("Robert Oppenheimer", 1904, 1967));
persons.add(new Person("Edwin Land", 1909, 1991));
persons.add(new Person("Rachel Carson", 1907, 1964));
persons.add(new Person("Ernest Rutherford", 1871, 1937));
persons.add(new Person("Hans Geiger", 1882, 1945));
persons.add(new Person("John Bardeen", 1908, 1991));
persons.add(new Person("George Washington Carver", 1864, 1943));
return persons;
}

private static void createCollection() {
client.dropCollection(DropCollectionReq.builder()
.collectionName(COLLECTION_NAME)
.build());

// Create collection
CreateCollectionReq.CollectionSchema collectionSchema = CreateCollectionReq.CollectionSchema.builder()
.build();
collectionSchema.addField(AddFieldReq.builder()
.fieldName(NAME_FIELD)
.dataType(DataType.VarChar)
.isPrimaryKey(Boolean.TRUE)
.maxLength(1024)
.enableAnalyzer(true)
.build());
collectionSchema.addField(AddFieldReq.builder()
.fieldName(BIRTH_YEAR_FIELD)
.dataType(DataType.Int64)
.build());
collectionSchema.addField(AddFieldReq.builder()
.fieldName(LIFESPAN_FIELD)
.dataType(DataType.Int8)
.build());
collectionSchema.addField(AddFieldReq.builder()
.fieldName(SPARSE_VECTOR_FIELD)
.dataType(DataType.SparseFloatVector)
.build());

collectionSchema.addFunction(CreateCollectionReq.Function.builder()
.functionType(FunctionType.BM25)
.name("function_bm25")
.inputFieldNames(Collections.singletonList(NAME_FIELD))
.outputFieldNames(Collections.singletonList(SPARSE_VECTOR_FIELD))
.build());

List<IndexParam> indexes = new ArrayList<>();
indexes.add(IndexParam.builder()
.fieldName(SPARSE_VECTOR_FIELD)
.indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)
.metricType(IndexParam.MetricType.BM25)
.build());

CreateCollectionReq requestCreate = CreateCollectionReq.builder()
.collectionName(COLLECTION_NAME)
.collectionSchema(collectionSchema)
.indexParams(indexes)
.consistencyLevel(ConsistencyLevel.BOUNDED)
.build();
client.createCollection(requestCreate);
System.out.println("Collection created");
}

private static void insertData() {
List<JsonObject> rows = new ArrayList<>();
List<Person> data = genData();
for (Person person : data) {
JsonObject row = new JsonObject();
row.addProperty(NAME_FIELD, person.name);
row.addProperty(BIRTH_YEAR_FIELD, person.fromYear);
row.addProperty(LIFESPAN_FIELD, person.toYear - person.fromYear);
rows.add(row);
}

client.insert(InsertReq.builder()
.collectionName(COLLECTION_NAME)
.data(rows)
.build());
printRowCount();
}

private static void printRowCount() {
// Get row count, set ConsistencyLevel.STRONG to sync the data to query node so that data is visible
QueryResp countR = client.query(QueryReq.builder()
.collectionName(COLLECTION_NAME)
.outputFields(Collections.singletonList("count(*)"))
.consistencyLevel(ConsistencyLevel.STRONG)
.build());
System.out.printf("%d rows persisted\n", (long)countR.getQueryResults().get(0).getEntity().get("count(*)"));
}

private static void dropCollection() {
client.dropCollection(DropCollectionReq.builder()
.collectionName(COLLECTION_NAME)
.build());
System.out.println("Collection dropped");
}

private static void searchWithRanker(String text, CreateCollectionReq.Function rankerFunction) {
System.out.println("\n=============================================================");
SearchReq.SearchReqBuilder builder = SearchReq.builder()
.collectionName(COLLECTION_NAME)
.data(Collections.singletonList(new EmbeddedText(text)))
.limit(100)
.outputFields(Arrays.asList(BIRTH_YEAR_FIELD, LIFESPAN_FIELD));

if (rankerFunction != null) {
builder.functionScore(FunctionScore.builder()
.addFunction(rankerFunction)
.build());
System.out.printf("Search text '%s' with ranker '%s'\n\n", text, rankerFunction.getName());
} else {
System.out.printf("Search text '%s' without ranker\n\n", text);
}

// The text is tokenized inside server and turned into a sparse embedding to compare with the vector field
SearchResp searchResp = client.search(builder.build());
List<List<SearchResp.SearchResult>> searchResults = searchResp.getSearchResults();
for (List<SearchResp.SearchResult> results : searchResults) {
for (SearchResp.SearchResult result : results) {
System.out.println(result);
}
}
}

private static void searchWithoutRanker(String text) {
searchWithRanker(text, null);
}

public static void main(String[] args) {
createCollection();
insertData();

// Search scientists with name or surname
String scientists = "Albert, Charles, Darwin and Edwin";
searchWithoutRanker(scientists);

// Search scientists with name or surname
// Rerank the results by linear decay, the scores are rearranged according to the birth years
// Read the doc for more info: https://milvus.io/docs/decay-ranker-overview.md
// The scientist whose birth year is close to 1900 will get a high score
DecayRanker decay = DecayRanker.builder()
.name("birth_year_linear_decay")
.inputFieldNames(Collections.singletonList(BIRTH_YEAR_FIELD))
.function("linear")
.origin(1900)
.scale(50)
.offset(0)
.decay(0.1)
.build();
searchWithRanker(scientists, decay);

// Search scientists with name or surname
// Rerank the results by boost, the scores are rearranged according to the birth years
// Read the doc for more info: https://milvus.io/docs/boost-ranker.md
// The scientist whose lifespan is between 60 and 70 will get a high score
BoostRanker boost = BoostRanker.builder()
.name("boost_on_lifespan")
.filter(String.format("%s > 60 and %s < 70", LIFESPAN_FIELD, LIFESPAN_FIELD))
.weight(5.0f)
.build();
searchWithRanker(scientists, boost);

dropCollection();
client.close();
}
}
Loading
Loading