Skip to content

Commit 9607596

Browse files
authored
Merge pull request #5 from Tencent/dev/aleronwang
2.5.5
2 parents f4cc004 + 03dad05 commit 9607596

23 files changed

Lines changed: 2769 additions & 2593 deletions

CHANGELOG.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Changelog
2+
3+
## v2.5.5
4+
* feat: Support new IndexType `IVF_RABITQ`.
5+
* feat: Support new FieldType Double and Int64.
6+
7+
## v2.5.2
8+
* feat: support embedding service interface `/ai/service/embedding`
9+
10+
## v2.5.0
11+
* feat: support full-text search interface `/document/fullTextSearch`
12+
* feat: support rebuilding sparse vector indexes via `/rebuild/index` interface
13+
* feat: support vector index type modification via `/index/modifyVectorIndex` interface (BIN_FLAT not supported)
14+
* feat: support rebuilding vector/sparse vector indexes with user defined CPU resource usage
15+

LICENSE

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@
187187
same "printed page" as the copyright notice for easier
188188
identification within third-party archives.
189189

190-
Copyright (C) 2017-2019 Tencent.
190+
Copyright (C) 2017-2019 THL A29 Limited, a Tencent company.
191191
All rights reserved.
192192

193193
Licensed under the Apache License, Version 2.0 (the "License");
@@ -251,4 +251,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
251251
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
252252
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
253253
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
254-
SOFTWARE.
254+
SOFTWARE.

README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,14 @@ You can use **Apache Maven** or **Gradle**/**Grails** to download the SDK.
2424
<dependency>
2525
<groupId>com.tencent.tcvectordb</groupId>
2626
<artifactId>vectordatabase-sdk-java</artifactId>
27-
<version>2.5.4</version>
27+
<version>2.5.5</version>
2828
</dependency>
2929
```
3030

3131
- Gradle/Grails
3232

3333
```gradle
34-
compile 'com.tencent.tcvectordb:vectordatabase-sdk-java:2.5.4'
34+
compile 'com.tencent.tcvectordb:vectordatabase-sdk-java:2.5.5'
3535
```
3636

3737
### Examples

tcvdb_text/pom.xml

Lines changed: 55 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,36 @@
66

77
<groupId>com.tencent.tcvectordb</groupId>
88
<artifactId>tcvdb-text</artifactId>
9-
<version>1.0.2</version>
9+
<version>1.0.3</version>
1010
<packaging>jar</packaging>
1111
<name>tcvdb-text</name>
1212

13+
<description>
14+
tencent vectordatabase text java sdk.
15+
</description>
16+
<url>https://cloud.tencent.com/product/vdb</url>
17+
<licenses>
18+
<license>
19+
<name>The Apache License, Version 2.0</name>
20+
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
21+
</license>
22+
</licenses>
23+
<developers>
24+
<developer>
25+
<id>tcvdb_text</id>
26+
<name>yihaoan</name>
27+
<email>yihaoan@tencent.com</email>
28+
</developer>
29+
</developers>
30+
<scm>
31+
<connection>scm:git:https://github.com/Tencent/vectordatabase-sdk-java.git
32+
</connection>
33+
<developerConnection>
34+
scm:git:https://github.com/Tencent/vectordatabase-sdk-java.git
35+
</developerConnection>
36+
<url>https://github.com/Tencent/vectordatabase-sdk-java</url>
37+
</scm>
38+
1339
<properties>
1440
<maven.compiler.source>1.8</maven.compiler.source>
1541
<maven.compiler.target>1.8</maven.compiler.target>
@@ -57,14 +83,14 @@
5783
</configuration>
5884
</plugin>
5985
<plugin>
60-
<groupId>org.sonatype.plugins</groupId>
61-
<artifactId>nexus-staging-maven-plugin</artifactId>
62-
<version>1.6.13</version>
86+
<groupId>org.sonatype.central</groupId>
87+
<artifactId>central-publishing-maven-plugin</artifactId>
88+
<version>0.8.0</version>
6389
<extensions>true</extensions>
6490
<configuration>
65-
<serverId>ossrh</serverId>
66-
<nexusUrl>https://oss.sonatype.org</nexusUrl>
67-
<autoReleaseAfterClose>true</autoReleaseAfterClose>
91+
<publishingServerId>ossrh</publishingServerId>
92+
<autoPublish>true</autoPublish>
93+
<waitUntil>published</waitUntil>
6894
</configuration>
6995
</plugin>
7096
<plugin>
@@ -80,6 +106,28 @@
80106
</execution>
81107
</executions>
82108
</plugin>
109+
<plugin>
110+
<groupId>org.apache.maven.plugins</groupId>
111+
<artifactId>maven-javadoc-plugin</artifactId>
112+
<version>3.2.0</version>
113+
<executions>
114+
<execution>
115+
<id>attach-javadoc</id>
116+
<phase>verify</phase>
117+
<goals>
118+
<!--生成javadoc的jar -->
119+
<goal>jar</goal>
120+
<!--生成javadoc的html -->
121+
<goal>javadoc</goal>
122+
</goals>
123+
<configuration>
124+
<!--不显示javadoc警告-->
125+
<additionalOptions>-Xdoclint:none</additionalOptions>
126+
<additionalJOption>-Xdoclint:none</additionalJOption>
127+
</configuration>
128+
</execution>
129+
</executions>
130+
</plugin>
83131
<plugin>
84132
<groupId>org.apache.maven.plugins</groupId>
85133
<artifactId>maven-gpg-plugin</artifactId>

tcvdb_text/src/main/java/com/tencent/tcvdbtext/encoder/SparseVectorBm25Encoder.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.apache.commons.lang3.tuple.Pair;
3737

3838
import java.io.*;
39+
import java.nio.charset.Charset;
3940
import java.util.*;
4041
import java.util.stream.Collectors;
4142

@@ -363,6 +364,15 @@ public void setStopWords(String stopWordsFile) {
363364
this.tokenizer.setStopWords(stopWordsFile);
364365
}
365366

367+
/**
368+
* set stop words file
369+
* @param stopWordsFile
370+
* @param charset stop words file charset, example: StandardCharsets.UTF_8
371+
*/
372+
public void setStopWords(String stopWordsFile, Charset charset) {
373+
this.tokenizer.setStopWords(stopWordsFile);
374+
}
375+
366376
// build模式
367377
public static class Builder {
368378
private BaseTokenizer tokenizer;

tcvdb_text/src/main/java/com/tencent/tcvdbtext/example.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import com.tencent.tcvdbtext.tokenizer.JiebaTokenizer;
3333

3434
import java.io.File;
35+
import java.nio.charset.StandardCharsets;
3536
import java.util.Arrays;
3637
import java.util.List;
3738

@@ -114,7 +115,8 @@ public static void stopWord(){
114115
String projectAbsolutePath = projectDirectory.getAbsolutePath();
115116
String path = projectAbsolutePath.replace("target/classes", "") +
116117
"src/main/resources/data/user_stopwords.txt";
117-
encoder.setStopWords(path);
118+
// 支持其他编码格式,默认UTF-8
119+
// encoder.setStopWords(path, StandardCharsets.UTF_8);
118120
encoder.setEnableStopWords(true);
119121
System.out.println(encoder.getTokenizer().tokenize("什么是腾讯云向量数据库。"));
120122

tcvdb_text/src/main/java/com/tencent/tcvdbtext/tokenizer/BaseTokenizer.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929

3030
import com.tencent.tcvdbtext.hash.BaseHash;
3131

32+
import java.nio.charset.Charset;
3233
import java.util.HashMap;
3334
import java.util.List;
3435
import java.util.Map;
@@ -128,4 +129,6 @@ public void setStopWords(Set<String> stopWords) {
128129
public abstract void setLowerCase(Boolean lowerCase);
129130

130131
public abstract void setStopWords(String stopWordsFile);
132+
133+
public abstract void setStopWords(String stopWordsFile, Charset charset);
131134
}

tcvdb_text/src/main/java/com/tencent/tcvdbtext/tokenizer/JiebaTokenizer.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import com.tencent.tcvdbtext.hash.BaseHash;
3535
import com.tencent.tcvdbtext.hash.Mm3BaseHash;
3636

37+
import java.nio.charset.Charset;
3738
import java.nio.file.Paths;
3839
import java.util.List;
3940
import java.util.Set;
@@ -88,6 +89,13 @@ public void setStopWords(String stopWordsFile) {
8889
}
8990
}
9091

92+
@Override
93+
public void setStopWords(String stopWordsFile, Charset charset) {
94+
if (!stopWordsFile.isEmpty()) {
95+
this.stopWords = StopWords.getStopWordsFromFilePath(stopWordsFile, charset);
96+
}
97+
}
98+
9199
@Override
92100
public List<String> tokenize(String sentence) {
93101
if(sentence.isEmpty()){

tcvdb_text/src/main/java/com/tencent/tcvdbtext/tokenizer/StopWords.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
package com.tencent.tcvdbtext.tokenizer;
2929

3030
import java.io.*;
31+
import java.nio.charset.Charset;
3132
import java.nio.charset.StandardCharsets;
3233
import java.util.HashSet;
3334
import java.util.Set;
@@ -78,4 +79,28 @@ public static Set<String> getStopWordsFromFilePath(String filePath) {
7879
}
7980
return lines;
8081
}
82+
83+
public static Set<String> getStopWordsFromFilePath(String filePath, Charset charset) {
84+
Set<String> lines = new HashSet<>();
85+
InputStream inputStream = null;
86+
try {
87+
inputStream = new FileInputStream(filePath);
88+
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, charset));
89+
String line;
90+
while ((line = reader.readLine()) != null) {
91+
lines.add(line);
92+
}
93+
} catch (IOException e) {
94+
e.printStackTrace();
95+
}finally {
96+
if(inputStream != null){
97+
try {
98+
inputStream.close();
99+
} catch (IOException e) {
100+
e.printStackTrace();
101+
}
102+
}
103+
}
104+
return lines;
105+
}
81106
}

tcvdb_text/src/main/java/com/tencent/tcvdbtext/util/JsonUtils.java

Lines changed: 0 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,12 @@ private JsonUtils() {
1717

1818
private static final String DATE_FORMAT_STR_ISO8601_CH = "yyyy-MM-dd HH:mm:ss";
1919
private static final ObjectMapper DESERIALIZE_IGNORE_KEY_MAPPER = new ObjectMapper();
20-
private static final ObjectMapper PARAMS_DESERIALIZE_MAPPER = new ObjectMapper();
2120
private static final ObjectMapper SERIALIZE_MAPPER = new ObjectMapper();
2221

2322

2423
static {
2524
DESERIALIZE_IGNORE_KEY_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
2625
DESERIALIZE_IGNORE_KEY_MAPPER.setDateFormat(new SimpleDateFormat(DATE_FORMAT_STR_ISO8601_CH));
27-
PARAMS_DESERIALIZE_MAPPER.setDateFormat(new SimpleDateFormat(DATE_FORMAT_STR_ISO8601_CH));
28-
29-
SimpleModule module = new SimpleModule();
30-
PARAMS_DESERIALIZE_MAPPER.registerModule(module);
3126
}
3227

3328
/**
@@ -60,45 +55,4 @@ public static <T> String toJsonString(T obj) {
6055
"can't serialize value=%s, type=%s", obj, obj.getClass().getName()));
6156
}
6257
}
63-
64-
/**
65-
* convert java object to JsonNode
66-
*
67-
* @param obj java object
68-
* @return {@link JsonNode}
69-
*/
70-
public static JsonNode toJsonNode(Object obj) {
71-
return SERIALIZE_MAPPER.valueToTree(obj);
72-
}
73-
74-
/**
75-
* parse json string to JsonNode
76-
*
77-
* @param jsonStr json string
78-
* @return {@link JsonNode}
79-
*/
80-
public static JsonNode parseToJsonNode(String jsonStr) {
81-
try {
82-
return SERIALIZE_MAPPER.readTree(jsonStr);
83-
} catch (JsonProcessingException e) {
84-
throw new ParamException(String.format(
85-
"can't parse content=%s", jsonStr));
86-
}
87-
}
88-
89-
/**
90-
* @param jsonStr json {@link String}
91-
* @param clz {@link TypeReference}
92-
* @param <T> result type
93-
* @return {@link T}
94-
*/
95-
public static <T> T collectionDeserializer(String jsonStr, TypeReference<T> clz) {
96-
try {
97-
return PARAMS_DESERIALIZE_MAPPER.readValue(jsonStr, clz);
98-
} catch (JsonProcessingException e) {
99-
System.out.println(e);
100-
throw new ParamException(String.format(
101-
"can't parse content=%s", jsonStr));
102-
}
103-
}
10458
}

0 commit comments

Comments
 (0)