Skip to content

Commit 0efed6e

Browse files
StriftStrift
committed
Add embedders settings
1 parent 6fc2b51 commit 0efed6e

4 files changed

Lines changed: 324 additions & 1 deletion

File tree

‎src/main/java/com/meilisearch/sdk/Index.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1249,4 +1249,41 @@ public SimilarDocumentsResults searchSimilarDocuments(SimilarDocumentRequest que
12491249
query,
12501250
SimilarDocumentsResults.class);
12511251
}
1252+
1253+
/**
1254+
* Gets the embedders settings of the index
1255+
*
1256+
* @return a Map that contains all embedders settings
1257+
* @throws MeilisearchException if an error occurs
1258+
* @see <a href="https://www.meilisearch.com/docs/reference/api/settings#get-embedders">API
1259+
* specification</a>
1260+
*/
1261+
public Map<String, Embedders> getEmbeddersSettings() throws MeilisearchException {
1262+
return this.settingsHandler.getEmbedders(this.uid);
1263+
}
1264+
1265+
/**
1266+
* Updates the embedders settings of the index
1267+
*
1268+
* @param embedders a Map that contains the new embedders settings
1269+
* @return TaskInfo instance
1270+
* @throws MeilisearchException if an error occurs
1271+
* @see <a href="https://www.meilisearch.com/docs/reference/api/settings#update-embedders">API
1272+
* specification</a>
1273+
*/
1274+
public TaskInfo updateEmbeddersSettings(Map<String, Embedders> embedders) throws MeilisearchException {
1275+
return this.settingsHandler.updateEmbedders(this.uid, embedders);
1276+
}
1277+
1278+
/**
1279+
* Resets the embedders settings of the index
1280+
*
1281+
* @return TaskInfo instance
1282+
* @throws MeilisearchException if an error occurs
1283+
* @see <a href="https://www.meilisearch.com/docs/reference/api/settings#reset-embedders">API
1284+
* specification</a>
1285+
*/
1286+
public TaskInfo resetEmbeddersSettings() throws MeilisearchException {
1287+
return this.settingsHandler.resetEmbedders(this.uid);
1288+
}
12521289
}

‎src/main/java/com/meilisearch/sdk/SettingsHandler.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.meilisearch.sdk.exceptions.MeilisearchException;
44
import com.meilisearch.sdk.http.URLBuilder;
5+
import com.meilisearch.sdk.model.Embedders;
56
import com.meilisearch.sdk.model.Faceting;
67
import com.meilisearch.sdk.model.LocalizedAttribute;
78
import com.meilisearch.sdk.model.Pagination;
@@ -769,4 +770,46 @@ public TaskInfo resetNonSeparatorTokensSettings(String uid) {
769770
return httpClient.delete(
770771
settingsPath(uid).addSubroute("non-separator-tokens").getURL(), TaskInfo.class);
771772
}
773+
774+
/**
775+
* Gets the embedders settings of the index
776+
*
777+
* @param uid Index identifier
778+
* @return a Map that contains all embedders settings
779+
* @throws MeilisearchException if an error occurs
780+
*/
781+
Map<String, Embedders> getEmbedders(String uid) throws MeilisearchException {
782+
return httpClient.get(
783+
settingsPath(uid).addSubroute("embedders").getURL(),
784+
Map.class,
785+
String.class,
786+
Embedders.class);
787+
}
788+
789+
/**
790+
* Updates the embedders settings of the index
791+
*
792+
* @param uid Index identifier
793+
* @param embedders a Map that contains the new embedders settings
794+
* @return TaskInfo instance
795+
* @throws MeilisearchException if an error occurs
796+
*/
797+
TaskInfo updateEmbedders(String uid, Map<String, Embedders> embedders) throws MeilisearchException {
798+
return httpClient.patch(
799+
settingsPath(uid).addSubroute("embedders").getURL(),
800+
embedders == null ? httpClient.jsonHandler.encode(embedders) : embedders,
801+
TaskInfo.class);
802+
}
803+
804+
/**
805+
* Resets the embedders settings of the index
806+
*
807+
* @param uid Index identifier
808+
* @return TaskInfo instance
809+
* @throws MeilisearchException if an error occurs
810+
*/
811+
TaskInfo resetEmbedders(String uid) throws MeilisearchException {
812+
return httpClient.delete(
813+
settingsPath(uid).addSubroute("embedders").getURL(), TaskInfo.class);
814+
}
772815
}

‎src/main/java/com/meilisearch/sdk/model/Embedders.java‎

Lines changed: 89 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,110 @@
22

33
import lombok.*;
44
import lombok.experimental.Accessors;
5+
import java.util.Map;
6+
import com.fasterxml.jackson.annotation.JsonInclude;
57

68
@Builder
79
@AllArgsConstructor(access = AccessLevel.PACKAGE)
810
@Getter
911
@Setter
1012
@Accessors(chain = true)
13+
@JsonInclude(JsonInclude.Include.NON_NULL)
1114
public class Embedders {
15+
/**
16+
* Source of the embedder.
17+
* Accepts: ollama, rest, openAI, huggingFace and userProvided
18+
*/
1219
protected EmbedderSource source;
13-
protected String url;
20+
21+
/**
22+
* API key for authentication with the embedder service.
23+
* Optional: Only applicable for openAi, ollama, and rest sources.
24+
*/
1425
protected String apiKey;
26+
27+
/**
28+
* Model to use for generating embeddings.
29+
* Optional: Only applicable for ollama, openAI, and huggingFace sources.
30+
*/
1531
protected String model;
32+
33+
/**
34+
* Template for document embedding.
35+
* Optional.
36+
*/
1637
protected String documentTemplate;
38+
39+
/**
40+
* Dimensions of the embedding vectors.
41+
* Optional: Only applicable for openAi, huggingFace, ollama, and rest sources.
42+
*/
1743
protected Integer dimensions;
44+
45+
/**
46+
* Distribution configuration.
47+
* Optional.
48+
*/
49+
protected String distribution;
50+
51+
/**
52+
* Request configuration.
53+
* Mandatory only when using rest embedder, optional otherwise.
54+
*/
55+
protected Map<String, Object> request;
56+
57+
/**
58+
* Response configuration.
59+
* Mandatory only when using rest embedder, optional otherwise.
60+
*/
61+
protected Map<String, Object> response;
62+
63+
/**
64+
* Maximum bytes for document template.
65+
* Optional.
66+
*/
67+
protected Integer documentTemplateMaxBytes;
68+
69+
/**
70+
* Revision identifier.
71+
* Optional: Only applicable for huggingFace.
72+
*/
1873
protected String revision;
74+
75+
/**
76+
* HTTP headers.
77+
* Optional: Only applicable for rest.
78+
*/
79+
protected Map<String, String> headers;
80+
81+
/**
82+
* Whether to use binary quantization.
83+
* Optional.
84+
*/
85+
protected Boolean binaryQuantized;
86+
87+
/**
88+
* URL for the embedder service.
89+
* Optional.
90+
*/
91+
protected String url;
92+
93+
/**
94+
* Input fields for the embedder.
95+
* Optional.
96+
*/
1997
protected String[] inputField;
98+
99+
/**
100+
* Type of input for the embedder.
101+
* Optional.
102+
*/
20103
protected EmbedderInputType inputType;
104+
105+
/**
106+
* Query for the embedder.
107+
* Optional.
108+
*/
21109
protected String query;
22110

23111
public Embedders() {}

‎src/test/java/com/meilisearch/integration/SettingsTest.java‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
import com.meilisearch.integration.classes.AbstractIT;
1717
import com.meilisearch.integration.classes.TestData;
1818
import com.meilisearch.sdk.Index;
19+
import com.meilisearch.sdk.model.Embedders;
20+
import com.meilisearch.sdk.model.EmbedderSource;
1921
import com.meilisearch.sdk.model.FacetSortValue;
2022
import com.meilisearch.sdk.model.Faceting;
2123
import com.meilisearch.sdk.model.LocalizedAttribute;
@@ -1465,4 +1467,157 @@ public void testResetNonSeparatorTokensSettings() throws Exception {
14651467
nonSeparatorTokensAfterReset, is(arrayWithSize(initialNonSeparatorTokens.length)));
14661468
assertThat(nonSeparatorTokensAfterReset, is(equalTo(initialNonSeparatorTokens)));
14671469
}
1470+
1471+
@Test
1472+
@DisplayName("Test get embedders settings by uid")
1473+
public void testGetEmbeddersSettings() throws Exception {
1474+
Index index = createIndex("testGetEmbeddersSettings");
1475+
Settings initialSettings = index.getSettings();
1476+
Map<String, Embedders> initialEmbedders = index.getEmbeddersSettings();
1477+
1478+
assertThat(initialEmbedders, is(equalTo(initialSettings.getEmbedders())));
1479+
}
1480+
1481+
@Test
1482+
@DisplayName("Test update embedders settings")
1483+
public void testUpdateEmbeddersSettings() throws Exception {
1484+
Index index = createIndex("testUpdateEmbeddersSettings");
1485+
Map<String, Embedders> initialEmbedders = index.getEmbeddersSettings();
1486+
1487+
// Create new embedders settings
1488+
HashMap<String, Embedders> newEmbedders = new HashMap<>();
1489+
1490+
// Test OpenAI embedder with apiKey and model
1491+
Embedders openAiEmbedder = new Embedders()
1492+
.setSource(EmbedderSource.OPEN_AI)
1493+
.setApiKey("test-api-key")
1494+
.setModel("text-embedding-ada-002")
1495+
.setDimensions(1536)
1496+
.setDocumentTemplate("OpenAI document: {{document}}")
1497+
.setDocumentTemplateMaxBytes(8000)
1498+
.setBinaryQuantized(true);
1499+
1500+
// Test HuggingFace embedder with model and revision
1501+
Embedders huggingFaceEmbedder = new Embedders()
1502+
.setSource(EmbedderSource.HUGGING_FACE)
1503+
.setModel("sentence-transformers/all-MiniLM-L6-v2")
1504+
.setRevision("main")
1505+
.setDistribution("uniform");
1506+
1507+
// Test REST embedder with request and response
1508+
Map<String, Object> request = new HashMap<>();
1509+
request.put("url", "https://api.example.com/embeddings");
1510+
request.put("method", "POST");
1511+
1512+
Map<String, Object> response = new HashMap<>();
1513+
response.put("embeddingsPath", "embeddings");
1514+
1515+
Map<String, String> headers = new HashMap<>();
1516+
headers.put("Authorization", "Bearer test-token");
1517+
1518+
Embedders restEmbedder = new Embedders()
1519+
.setSource(EmbedderSource.REST)
1520+
.setApiKey("test-rest-key")
1521+
.setRequest(request)
1522+
.setResponse(response)
1523+
.setHeaders(headers)
1524+
.setDimensions(384);
1525+
1526+
// Test Ollama embedder
1527+
Embedders ollamaEmbedder = new Embedders()
1528+
.setSource(EmbedderSource.OLLAMA)
1529+
.setModel("llama2")
1530+
.setApiKey("test-ollama-key")
1531+
.setDimensions(4096);
1532+
1533+
// Test UserProvided embedder
1534+
Embedders userProvidedEmbedder = new Embedders()
1535+
.setSource(EmbedderSource.USER_PROVIDED)
1536+
.setDimensions(768);
1537+
1538+
// Add all embedders to the map
1539+
newEmbedders.put("openai", openAiEmbedder);
1540+
newEmbedders.put("huggingface", huggingFaceEmbedder);
1541+
newEmbedders.put("rest", restEmbedder);
1542+
newEmbedders.put("ollama", ollamaEmbedder);
1543+
newEmbedders.put("user", userProvidedEmbedder);
1544+
1545+
// Update settings
1546+
index.waitForTask(index.updateEmbeddersSettings(newEmbedders).getTaskUid());
1547+
Map<String, Embedders> updatedEmbedders = index.getEmbeddersSettings();
1548+
1549+
// Verify results
1550+
assertThat(updatedEmbedders.size(), is(equalTo(5)));
1551+
1552+
// Check OpenAI embedder
1553+
Embedders retrievedOpenAi = updatedEmbedders.get("openai");
1554+
assertThat(retrievedOpenAi.getSource(), is(equalTo(EmbedderSource.OPEN_AI)));
1555+
assertThat(retrievedOpenAi.getApiKey(), is(equalTo("test-api-key")));
1556+
assertThat(retrievedOpenAi.getModel(), is(equalTo("text-embedding-ada-002")));
1557+
assertThat(retrievedOpenAi.getDimensions(), is(equalTo(1536)));
1558+
assertThat(retrievedOpenAi.getDocumentTemplate(), is(equalTo("OpenAI document: {{document}}")));
1559+
assertThat(retrievedOpenAi.getDocumentTemplateMaxBytes(), is(equalTo(8000)));
1560+
assertThat(retrievedOpenAi.getBinaryQuantized(), is(equalTo(true)));
1561+
1562+
// Check HuggingFace embedder
1563+
Embedders retrievedHf = updatedEmbedders.get("huggingface");
1564+
assertThat(retrievedHf.getSource(), is(equalTo(EmbedderSource.HUGGING_FACE)));
1565+
assertThat(retrievedHf.getModel(), is(equalTo("sentence-transformers/all-MiniLM-L6-v2")));
1566+
assertThat(retrievedHf.getRevision(), is(equalTo("main")));
1567+
assertThat(retrievedHf.getDistribution(), is(equalTo("uniform")));
1568+
1569+
// Check REST embedder
1570+
Embedders retrievedRest = updatedEmbedders.get("rest");
1571+
assertThat(retrievedRest.getSource(), is(equalTo(EmbedderSource.REST)));
1572+
assertThat(retrievedRest.getApiKey(), is(equalTo("test-rest-key")));
1573+
assertThat(retrievedRest.getRequest(), is(notNullValue()));
1574+
assertThat(retrievedRest.getResponse(), is(notNullValue()));
1575+
assertThat(retrievedRest.getHeaders(), is(notNullValue()));
1576+
assertThat(retrievedRest.getDimensions(), is(equalTo(384)));
1577+
1578+
// Check Ollama embedder
1579+
Embedders retrievedOllama = updatedEmbedders.get("ollama");
1580+
assertThat(retrievedOllama.getSource(), is(equalTo(EmbedderSource.OLLAMA)));
1581+
assertThat(retrievedOllama.getModel(), is(equalTo("llama2")));
1582+
assertThat(retrievedOllama.getApiKey(), is(equalTo("test-ollama-key")));
1583+
assertThat(retrievedOllama.getDimensions(), is(equalTo(4096)));
1584+
1585+
// Check UserProvided embedder
1586+
Embedders retrievedUser = updatedEmbedders.get("user");
1587+
assertThat(retrievedUser.getSource(), is(equalTo(EmbedderSource.USER_PROVIDED)));
1588+
assertThat(retrievedUser.getDimensions(), is(equalTo(768)));
1589+
}
1590+
1591+
@Test
1592+
@DisplayName("Test reset embedders settings")
1593+
public void testResetEmbeddersSettings() throws Exception {
1594+
Index index = createIndex("testResetEmbeddersSettings");
1595+
Map<String, Embedders> initialEmbedders = index.getEmbeddersSettings();
1596+
1597+
// Create new embedders settings
1598+
HashMap<String, Embedders> newEmbedders = new HashMap<>();
1599+
Embedders embedder = new Embedders()
1600+
.setSource(EmbedderSource.USER_PROVIDED)
1601+
.setDimensions(768);
1602+
newEmbedders.put("test", embedder);
1603+
1604+
// Update settings
1605+
index.waitForTask(index.updateEmbeddersSettings(newEmbedders).getTaskUid());
1606+
Map<String, Embedders> updatedEmbedders = index.getEmbeddersSettings();
1607+
1608+
// Reset settings
1609+
index.waitForTask(index.resetEmbeddersSettings().getTaskUid());
1610+
Map<String, Embedders> resetEmbedders = index.getEmbeddersSettings();
1611+
1612+
// Verify results
1613+
assertThat(updatedEmbedders.size(), is(equalTo(1)));
1614+
assertThat(updatedEmbedders.containsKey("test"), is(true));
1615+
1616+
// After reset, should be back to initial state
1617+
if (initialEmbedders == null) {
1618+
assertThat(resetEmbedders == null || resetEmbedders.isEmpty(), is(true));
1619+
} else {
1620+
assertThat(resetEmbedders.size(), is(equalTo(initialEmbedders.size())));
1621+
}
1622+
}
14681623
}

0 commit comments

Comments
 (0)