Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
6 changes: 0 additions & 6 deletions examples/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,6 @@
<artifactId>tensorflow-core-platform</artifactId>
<version>0.5.0</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.22</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
Expand Down
9 changes: 6 additions & 3 deletions examples/src/main/java/io/milvus/v1/ArrayFieldExample.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@
import io.milvus.param.index.CreateIndexParam;
import io.milvus.response.QueryResultsWrapper;

import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;

public class ArrayFieldExample {
private static final String COLLECTION_NAME = "java_sdk_example_array_v1";
Expand Down Expand Up @@ -141,7 +144,7 @@ public static void main(String[] args) {
List<String> strArray = new ArrayList<>();
int capacity = random.nextInt(5) + 5;
for (int k = 0; k < capacity; k++) {
intArray.add((i+k)%100);
intArray.add((i + k) % 100);
strArray.add(String.format("string-%d-%d", i, k));
}
intArrArray.add(intArray);
Expand Down Expand Up @@ -186,7 +189,7 @@ public static void main(String[] args) {
.withConsistencyLevel(ConsistencyLevelEnum.STRONG)
.build());
QueryResultsWrapper queryWrapper = new QueryResultsWrapper(queryRet.getData());
System.out.printf("%d rows in collection\n", (long)queryWrapper.getFieldWrapper("count(*)").getFieldData().get(0));
System.out.printf("%d rows in collection\n", (long) queryWrapper.getFieldWrapper("count(*)").getFieldData().get(0));

// Query by filtering expression
queryWithExpr(client, "array_int32[0] == 99");
Expand Down
4 changes: 2 additions & 2 deletions examples/src/main/java/io/milvus/v1/BinaryVectorExample.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public class BinaryVectorExample {
private static final String VECTOR_FIELD = "vector";

private static final Integer VECTOR_DIM = 128;


public static void main(String[] args) {
// Connect to Milvus server. Replace the "localhost" and port with your Milvus server address.
Expand Down Expand Up @@ -172,7 +172,7 @@ public static void main(String[] args) {
System.out.printf("The result of No.%d target vector:\n", i);
for (SearchResultsWrapper.IDScore score : scores) {
System.out.println(score);
ByteBuffer vector = (ByteBuffer)score.get(VECTOR_FIELD);
ByteBuffer vector = (ByteBuffer) score.get(VECTOR_FIELD);
CommonUtils.printBinaryVector(vector);
}
if (scores.get(0).getLongID() != k) {
Expand Down
29 changes: 5 additions & 24 deletions examples/src/main/java/io/milvus/v1/BulkWriterExample.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,7 @@
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import io.milvus.bulkwriter.BulkWriter;
import io.milvus.bulkwriter.LocalBulkWriter;
import io.milvus.bulkwriter.LocalBulkWriterParam;
import io.milvus.bulkwriter.RemoteBulkWriter;
import io.milvus.bulkwriter.RemoteBulkWriterParam;
import io.milvus.bulkwriter.*;
import io.milvus.bulkwriter.common.clientenum.BulkFileType;
import io.milvus.bulkwriter.common.clientenum.CloudStorage;
import io.milvus.bulkwriter.common.utils.GeneratorUtils;
Expand All @@ -50,19 +46,8 @@
import io.milvus.grpc.DataType;
import io.milvus.grpc.GetCollectionStatisticsResponse;
import io.milvus.grpc.QueryResults;
import io.milvus.param.ConnectParam;
import io.milvus.param.IndexType;
import io.milvus.param.MetricType;
import io.milvus.param.R;
import io.milvus.param.RpcStatus;
import io.milvus.param.collection.CollectionSchemaParam;
import io.milvus.param.collection.CreateCollectionParam;
import io.milvus.param.collection.DropCollectionParam;
import io.milvus.param.collection.FieldType;
import io.milvus.param.collection.FlushParam;
import io.milvus.param.collection.GetCollectionStatisticsParam;
import io.milvus.param.collection.HasCollectionParam;
import io.milvus.param.collection.LoadCollectionParam;
import io.milvus.param.*;
import io.milvus.param.collection.*;
import io.milvus.param.dml.QueryParam;
import io.milvus.param.index.CreateIndexParam;
import io.milvus.response.GetCollStatResponseWrapper;
Expand All @@ -74,11 +59,7 @@
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.TimeUnit;


Expand Down Expand Up @@ -596,7 +577,7 @@ private void callCloudImport(List<List<String>> batchFiles, String collectionNam

/**
* @param collectionSchema collection info
* @param dropIfExist if collection already exist, will drop firstly and then create again
* @param dropIfExist if collection already exist, will drop firstly and then create again
*/
private void createCollection(String collectionName, CollectionSchemaParam collectionSchema, boolean dropIfExist) {
System.out.println("\n===================== create collection ====================");
Expand Down
16 changes: 10 additions & 6 deletions examples/src/main/java/io/milvus/v1/ClientPoolExample.java
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,10 @@
import io.milvus.response.QueryResultsWrapper;

import java.time.Duration;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class ClientPoolExample {
public static String serverUri = "http://localhost:19530";
Expand All @@ -51,6 +54,7 @@ private static void printKeyClientNumber(MilvusClientV1Pool pool, String key) {
System.out.printf("Key '%s': %d idle clients and %d active clients%n",
key, pool.getIdleClientNumber(key), pool.getActiveClientNumber(key));
}

private static void printClientNumber(MilvusClientV1Pool pool) {
System.out.println("======================================================================");
System.out.printf("Total %d idle clients and %d active clients%n",
Expand Down Expand Up @@ -158,7 +162,7 @@ public static Thread runInsertThread(MilvusClientV1Pool pool, String dbName, int
Gson gson = new Gson();
for (int i = 0; i < repeatRequests; i++) {
MilvusClient client = null;
while(client == null) {
while (client == null) {
try {
// getClient() might exceeds the borrowMaxWaitMillis and throw exception
// retry to call until it return a client
Expand Down Expand Up @@ -201,7 +205,7 @@ public static Thread runSearchThread(MilvusClientV1Pool pool, String dbName, int
Thread t = new Thread(() -> {
for (int i = 0; i < repeatRequests; i++) {
MilvusClient client = null;
while(client == null) {
while (client == null) {
try {
// getClient() might exceeds the borrowMaxWaitMillis and throw exception
// retry to call until it return a client
Expand Down Expand Up @@ -253,7 +257,7 @@ public static void verifyRowCount(MilvusClientV1Pool pool, long expectedCount) {
.withConsistencyLevel(ConsistencyLevelEnum.STRONG)
.build());
QueryResultsWrapper queryWrapper = new QueryResultsWrapper(queryRet.getData());
long rowCount = (long)queryWrapper.getFieldWrapper("count(*)").getFieldData().get(0);
long rowCount = (long) queryWrapper.getFieldWrapper("count(*)").getFieldData().get(0);
System.out.printf("%d rows persisted in collection '%s' of database '%s'%n",
rowCount, CollectionName, dbName);
if (rowCount != expectedCount) {
Expand Down Expand Up @@ -359,15 +363,15 @@ public static void main(String[] args) throws InterruptedException {
printClientNumber(pool);

// check row count of each collection, there are threadCount*repeatRequests rows were inserted by multiple threads
verifyRowCount(pool, threadCount*repeatRequests);
verifyRowCount(pool, threadCount * repeatRequests);
// drop collections
dropCollections(pool);
// drop databases, only after database is empty, it is able to be dropped
dropDatabases(pool);

long end = System.currentTimeMillis();
System.out.printf("%d insert requests and %d search requests finished in %.3f seconds%n",
threadCount*repeatRequests*3, threadCount*repeatRequests*3, (end-start)*0.001);
threadCount * repeatRequests * 3, threadCount * repeatRequests * 3, (end - start) * 0.001);

printClientNumber(pool);
pool.clear(); // clear idle clients
Expand Down
29 changes: 14 additions & 15 deletions examples/src/main/java/io/milvus/v1/CommonUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

import io.milvus.common.utils.Float16Utils;
import io.milvus.param.R;

import org.tensorflow.Tensor;
import org.tensorflow.ndarray.Shape;
import org.tensorflow.ndarray.buffer.ByteDataBuffer;
Expand Down Expand Up @@ -69,7 +68,7 @@ public static List<List<Float>> generateFloatVectors(int dimension, int count) {
public static List<List<Float>> generateFixFloatVectors(int dimension, int count) {
List<List<Float>> vectors = new ArrayList<>();
for (int n = 0; n < count; ++n) {
List<Float> vector = generateFloatVector(dimension, (float)n);
List<Float> vector = generateFloatVector(dimension, (float) n);
vectors.add(vector);
}
return vectors;
Expand All @@ -87,7 +86,7 @@ public static void compareFloatVectors(List<Float> vec1, List<Float> vec2) {
}
}

/////////////////////////////////////////////////////////////////////////////////////////////////////
/// //////////////////////////////////////////////////////////////////////////////////////////////////
public static ByteBuffer generateBinaryVector(int dimension) {
Random ran = new Random();
int byteCount = dimension / 8;
Expand Down Expand Up @@ -117,7 +116,7 @@ public static void printBinaryVector(ByteBuffer vector) {
System.out.println();
}

/////////////////////////////////////////////////////////////////////////////////////////////////////
/// //////////////////////////////////////////////////////////////////////////////////////////////////
public static TBfloat16 genTensorflowBF16Vector(int dimension) {
Random ran = new Random();
float[] array = new float[dimension];
Expand All @@ -131,7 +130,7 @@ public static TBfloat16 genTensorflowBF16Vector(int dimension) {
public static List<TBfloat16> genTensorflowBF16Vectors(int dimension, int count) {
List<TBfloat16> vectors = new ArrayList<>();
for (int n = 0; n < count; ++n) {
TBfloat16 vector = genTensorflowBF16Vector(dimension);
TBfloat16 vector = genTensorflowBF16Vector(dimension);
vectors.add(vector);
}

Expand All @@ -140,7 +139,7 @@ public static List<TBfloat16> genTensorflowBF16Vectors(int dimension, int count)

public static ByteBuffer encodeTensorBF16Vector(TBfloat16 vector) {
ByteDataBuffer tensorBuf = vector.asRawTensor().data();
ByteBuffer buf = ByteBuffer.allocate((int)tensorBuf.size());
ByteBuffer buf = ByteBuffer.allocate((int) tensorBuf.size());
for (long i = 0; i < tensorBuf.size(); i++) {
buf.put(tensorBuf.getByte(i));
}
Expand All @@ -157,10 +156,10 @@ public static List<ByteBuffer> encodeTensorBF16Vectors(List<TBfloat16> vectors)
}

public static TBfloat16 decodeBF16VectorToTensor(ByteBuffer buf) {
if (buf.limit()%2 != 0) {
if (buf.limit() % 2 != 0) {
return null;
}
int dim = buf.limit()/2;
int dim = buf.limit() / 2;
ByteDataBuffer bf = DataBuffers.of(buf.array());
return Tensor.of(TBfloat16.class, Shape.of(dim), bf);
}
Expand Down Expand Up @@ -197,7 +196,7 @@ public static List<TFloat16> genTensorflowFP16Vectors(int dimension, int count)

public static ByteBuffer encodeTensorFP16Vector(TFloat16 vector) {
ByteDataBuffer tensorBuf = vector.asRawTensor().data();
ByteBuffer buf = ByteBuffer.allocate((int)tensorBuf.size());
ByteBuffer buf = ByteBuffer.allocate((int) tensorBuf.size());
for (long i = 0; i < tensorBuf.size(); i++) {
buf.put(tensorBuf.getByte(i));
}
Expand All @@ -214,10 +213,10 @@ public static List<ByteBuffer> encodeTensorFP16Vectors(List<TFloat16> vectors) {
}

public static TFloat16 decodeFP16VectorToTensor(ByteBuffer buf) {
if (buf.limit()%2 != 0) {
if (buf.limit() % 2 != 0) {
return null;
}
int dim = buf.limit()/2;
int dim = buf.limit() / 2;
ByteDataBuffer bf = DataBuffers.of(buf.array());
return Tensor.of(TFloat16.class, Shape.of(dim), bf);
}
Expand All @@ -231,7 +230,7 @@ public static List<Float> decodeFP16VectorToFloat(ByteBuffer buf) {
return vector;
}

/////////////////////////////////////////////////////////////////////////////////////////////////////
/// //////////////////////////////////////////////////////////////////////////////////////////////////
public static ByteBuffer encodeFloat16Vector(List<Float> originVector, boolean bfloat16) {
if (bfloat16) {
return Float16Utils.f32VectorToBf16Buffer(originVector);
Expand Down Expand Up @@ -274,7 +273,7 @@ public static List<ByteBuffer> generateFloat16Vectors(int dimension, int count,
return vectors;
}

/////////////////////////////////////////////////////////////////////////////////////////////////////
/// //////////////////////////////////////////////////////////////////////////////////////////////////
public static ByteBuffer generateInt8Vector(int dimension) {
Random ran = new Random();
int byteCount = dimension;
Expand All @@ -295,13 +294,13 @@ public static List<ByteBuffer> generateInt8Vectors(int dimension, int count) {
return vectors;
}

/////////////////////////////////////////////////////////////////////////////////////////////////////
/// //////////////////////////////////////////////////////////////////////////////////////////////////
public static SortedMap<Long, Float> generateSparseVector() {
Random ran = new Random();
SortedMap<Long, Float> sparse = new TreeMap<>();
int dim = ran.nextInt(10) + 10;
while (sparse.size() < dim) {
sparse.put((long)ran.nextInt(1000000), ran.nextFloat());
sparse.put((long) ran.nextInt(1000000), ran.nextFloat());
}
return sparse;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ private static List<SearchResultsWrapper.IDScore> search(String collectionName,
.withCollectionName(collectionName)
.withVectorFieldName("vector")
.withFloatVectors(Collections.singletonList(CommonUtils.generateFloatVector(VECTOR_DIM)))
.withLimit((long)topK)
.withLimit((long) topK)
.withMetricType(MetricType.L2)
.build());
CommonUtils.handleResponseStatus(searchR);
Expand Down Expand Up @@ -187,7 +187,7 @@ private static void testSessionLevel() throws ClassNotFoundException, NoSuchMeth
row.add("vector", gson.toJsonTree(vector));

// insert by a MilvusClient
String clientName1 = String.format("client_%d", i%10);
String clientName1 = String.format("client_%d", i % 10);
MilvusClient client1 = pool.getClient(clientName1);
client1.insert(InsertParam.newBuilder()
.withCollectionName(collectionName)
Expand All @@ -198,7 +198,7 @@ private static void testSessionLevel() throws ClassNotFoundException, NoSuchMeth

// search by another MilvusClient, use the just inserted vector to search
// the returned item is expected to be the just inserted item
String clientName2 = String.format("client_%d", i%10+1);
String clientName2 = String.format("client_%d", i % 10 + 1);
MilvusClient client2 = pool.getClient(clientName2);
R<SearchResults> searchR = client2.search(SearchParam.newBuilder()
.withCollectionName(collectionName)
Expand Down
22 changes: 15 additions & 7 deletions examples/src/main/java/io/milvus/v1/Float16VectorExample.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,19 @@
import com.google.gson.JsonObject;
import io.milvus.client.MilvusServiceClient;
import io.milvus.common.clientenum.ConsistencyLevelEnum;
import io.milvus.grpc.*;
import io.milvus.grpc.DataType;
import io.milvus.grpc.MutationResult;
import io.milvus.grpc.QueryResults;
import io.milvus.grpc.SearchResults;
import io.milvus.param.*;
import io.milvus.param.collection.*;
import io.milvus.param.dml.*;
import io.milvus.param.index.*;
import io.milvus.response.*;
import io.milvus.param.dml.InsertParam;
import io.milvus.param.dml.QueryParam;
import io.milvus.param.dml.SearchParam;
import io.milvus.param.index.CreateIndexParam;
import io.milvus.response.FieldDataWrapper;
import io.milvus.response.QueryResultsWrapper;
import io.milvus.response.SearchResultsWrapper;
import org.tensorflow.types.TBfloat16;
import org.tensorflow.types.TFloat16;

Expand All @@ -42,6 +49,7 @@ public class Float16VectorExample {
private static final Integer VECTOR_DIM = 128;

private static final MilvusServiceClient milvusClient;

static {
// Connect to Milvus server. Replace the "localhost" and port with your Milvus server address.
milvusClient = new MilvusServiceClient(ConnectParam.newBuilder()
Expand Down Expand Up @@ -179,7 +187,7 @@ private static void testFloat16(boolean bfloat16) {
// Ensure the returned top1 item's ID should be equal to target vector's ID
for (int i = 0; i < 10; i++) {
Random ran = new Random();
int k = ran.nextInt(batchRowCount*2);
int k = ran.nextInt(batchRowCount * 2);
ByteBuffer targetVector = encodedVectors.get(k);
SearchParam.Builder builder = SearchParam.newBuilder()
.withCollectionName(COLLECTION_NAME)
Expand Down Expand Up @@ -208,7 +216,7 @@ private static void testFloat16(boolean bfloat16) {
firstScore.getLongID(), k));
}

ByteBuffer outputBuf = (ByteBuffer)firstScore.get(VECTOR_FIELD);
ByteBuffer outputBuf = (ByteBuffer) firstScore.get(VECTOR_FIELD);
if (!outputBuf.equals(targetVector)) {
throw new RuntimeException(String.format("The output vector is not equal to target vector: ID %d", k));
}
Expand All @@ -229,7 +237,7 @@ private static void testFloat16(boolean bfloat16) {
// Retrieve some data and verify the output
for (int i = 0; i < 10; i++) {
Random ran = new Random();
int k = ran.nextInt(batchRowCount*2);
int k = ran.nextInt(batchRowCount * 2);
R<QueryResults> queryR = milvusClient.query(QueryParam.newBuilder()
.withCollectionName(COLLECTION_NAME)
.withExpr(String.format("id == %d", k))
Expand Down
2 changes: 1 addition & 1 deletion examples/src/main/java/io/milvus/v1/GeneralExample.java
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public class GeneralExample {
ConnectParam connectParam = ConnectParam.newBuilder()
.withHost("localhost")
.withPort(19530)
.withAuthorization("root","Milvus")
.withAuthorization("root", "Milvus")
.build();
RetryParam retryParam = RetryParam.newBuilder()
.withMaxRetryTimes(3)
Expand Down
Loading
Loading