Skip to content

Commit 41c2b21

Browse files
krus210krus210
committed
vectorstores/duckdb: add DuckDB vectorstore implementation
1 parent 8fea3de commit 41c2b21

18 files changed

Lines changed: 1368 additions & 115 deletions

‎examples/duckdb-vectorstore-example/duckdb_vectorstore_example.go‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"log"
7+
8+
"github.com/tmc/langchaingo/embeddings"
9+
"github.com/tmc/langchaingo/llms/openai"
10+
"github.com/tmc/langchaingo/schema"
11+
"github.com/tmc/langchaingo/vectorstores"
12+
duckdbstore "github.com/tmc/langchaingo/vectorstores/duckdb"
13+
)
14+
15+
func main() {
16+
// Create an embeddings client using the OpenAI API. Requires environment variable OPENAI_API_KEY to be set.
17+
llm, err := openai.New()
18+
if err != nil {
19+
log.Fatal(err)
20+
}
21+
22+
e, err := embeddings.NewEmbedder(llm)
23+
if err != nil {
24+
log.Fatal(err)
25+
}
26+
27+
// Create a new DuckDB store. An empty connection URL creates an in-memory database.
28+
ctx := context.Background()
29+
store, err := duckdbstore.New(
30+
ctx,
31+
duckdbstore.WithConnectionURL(""),
32+
duckdbstore.WithEmbedder(e),
33+
duckdbstore.WithVectorDimensions(1536),
34+
)
35+
if err != nil {
36+
log.Fatal(err)
37+
}
38+
defer store.Close()
39+
40+
// Add documents to the DuckDB store.
41+
_, err = store.AddDocuments(ctx, []schema.Document{
42+
{
43+
PageContent: "Tokyo",
44+
Metadata: map[string]any{
45+
"population": 38,
46+
"area": 2190,
47+
},
48+
},
49+
{
50+
PageContent: "Paris",
51+
Metadata: map[string]any{
52+
"population": 11,
53+
"area": 105,
54+
},
55+
},
56+
{
57+
PageContent: "London",
58+
Metadata: map[string]any{
59+
"population": 9.5,
60+
"area": 1572,
61+
},
62+
},
63+
{
64+
PageContent: "Santiago",
65+
Metadata: map[string]any{
66+
"population": 6.9,
67+
"area": 641,
68+
},
69+
},
70+
{
71+
PageContent: "Buenos Aires",
72+
Metadata: map[string]any{
73+
"population": 15.5,
74+
"area": 203,
75+
},
76+
},
77+
{
78+
PageContent: "Rio de Janeiro",
79+
Metadata: map[string]any{
80+
"population": 13.7,
81+
"area": 1200,
82+
},
83+
},
84+
{
85+
PageContent: "Sao Paulo",
86+
Metadata: map[string]any{
87+
"population": 22.6,
88+
"area": 1523,
89+
},
90+
},
91+
})
92+
if err != nil {
93+
log.Fatal(err)
94+
}
95+
96+
// Search for similar documents.
97+
docs, err := store.SimilaritySearch(ctx, "japan", 1)
98+
if err != nil {
99+
log.Fatal(err)
100+
}
101+
fmt.Println(docs)
102+
103+
// Search for similar documents using score threshold.
104+
docs, err = store.SimilaritySearch(ctx, "only cities in south america", 10, vectorstores.WithScoreThreshold(0.80))
105+
if err != nil {
106+
log.Fatal(err)
107+
}
108+
fmt.Println(docs)
109+
110+
// Search for similar documents using score threshold and metadata filter.
111+
// Note: json_extract_string converts numeric metadata values to strings.
112+
filter := map[string]any{"area": 1523} // Sao Paulo
113+
114+
docs, err = store.SimilaritySearch(ctx, "only cities in south america",
115+
10,
116+
vectorstores.WithScoreThreshold(0.80),
117+
vectorstores.WithFilters(filter),
118+
)
119+
if err != nil {
120+
log.Fatal(err)
121+
}
122+
fmt.Println(docs)
123+
}

‎examples/duckdb-vectorstore-example/go.mod‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module github.com/tmc/langchaingo/examples/duckdb-vectorstore-example
2+
3+
go 1.24.3
4+
5+
require github.com/tmc/langchaingo v0.1.14-pre.4

‎examples/duckdb-vectorstore-example/go.sum‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
github.com/tmc/langchaingo v0.1.14-pre.4/go.mod h1:xGqIATL4itqqEAVwSF5xVh4ZuIP7gOE0dyoqe3quvzw=

‎go.mod‎

Lines changed: 50 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ require github.com/google/uuid v1.6.0
1616

1717
// Testing
1818
require (
19-
github.com/stretchr/testify v1.10.0
19+
github.com/stretchr/testify v1.11.1
2020
github.com/testcontainers/testcontainers-go v0.38.0
2121
github.com/testcontainers/testcontainers-go/modules/chroma v0.37.0
2222
github.com/testcontainers/testcontainers-go/modules/milvus v0.37.0
@@ -53,7 +53,7 @@ require (
5353

5454
// Cloud platforms and AI services
5555
require (
56-
cloud.google.com/go/aiplatform v1.69.0
56+
cloud.google.com/go/aiplatform v1.74.0
5757
cloud.google.com/go/vertexai v0.12.0
5858
github.com/aws/aws-sdk-go-v2 v1.36.3
5959
github.com/aws/aws-sdk-go-v2/config v1.29.4
@@ -62,10 +62,10 @@ require (
6262
github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.24.3
6363
github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2
6464
github.com/aws/smithy-go v1.22.2
65-
golang.org/x/oauth2 v0.30.0
66-
google.golang.org/api v0.218.0
67-
google.golang.org/grpc v1.70.0
68-
google.golang.org/protobuf v1.36.3
65+
golang.org/x/oauth2 v0.32.0
66+
google.golang.org/api v0.230.0
67+
google.golang.org/grpc v1.78.0
68+
google.golang.org/protobuf v1.36.11
6969
)
7070

7171
// Embeddings and ML
@@ -78,6 +78,7 @@ require (
7878
require (
7979
cloud.google.com/go/alloydbconn v1.13.2
8080
cloud.google.com/go/cloudsqlconn v1.14.1
81+
github.com/duckdb/duckdb-go/v2 v2.5.5
8182
github.com/go-sql-driver/mysql v1.8.1
8283
github.com/jackc/pgx/v5 v5.7.2
8384
github.com/mattn/go-sqlite3 v1.14.17
@@ -107,23 +108,23 @@ require (
107108
github.com/google/go-cmp v0.7.0
108109
github.com/nikolalohinski/gonja v1.5.3
109110
github.com/zeebo/blake3 v0.2.4
110-
golang.org/x/sync v0.16.0
111-
golang.org/x/tools v0.35.0
111+
golang.org/x/sync v0.19.0
112+
golang.org/x/tools v0.41.0
112113
sigs.k8s.io/yaml v1.3.0
113114
)
114115

115116
// Indirect dependencies (automatically managed)
116117

117118
// Cloud platforms and AI services - indirect
118119
require (
119-
cloud.google.com/go v0.116.0 // indirect
120+
cloud.google.com/go v0.121.0 // indirect
120121
cloud.google.com/go/ai v0.7.0 // indirect
121122
cloud.google.com/go/alloydb v1.14.0 // indirect
122-
cloud.google.com/go/auth v0.14.0 // indirect
123-
cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect
124-
cloud.google.com/go/compute/metadata v0.6.0 // indirect
125-
cloud.google.com/go/iam v1.2.2 // indirect
126-
cloud.google.com/go/longrunning v0.6.2 // indirect
123+
cloud.google.com/go/auth v0.16.0 // indirect
124+
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
125+
cloud.google.com/go/compute/metadata v0.9.0 // indirect
126+
cloud.google.com/go/iam v1.5.0 // indirect
127+
cloud.google.com/go/longrunning v0.6.6 // indirect
127128
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
128129
github.com/aws/aws-sdk-go-v2/credentials v1.17.57 // indirect
129130
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 // indirect
@@ -139,19 +140,19 @@ require (
139140
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 // indirect
140141
github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 // indirect
141142
github.com/google/s2a-go v0.1.9 // indirect
142-
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
143+
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
143144
github.com/googleapis/gax-go/v2 v2.14.1 // indirect
144145
go.opencensus.io v0.24.0 // indirect
145-
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
146-
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
146+
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
147+
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
147148
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
148-
go.opentelemetry.io/otel v1.36.0 // indirect
149-
go.opentelemetry.io/otel/metric v1.36.0 // indirect
150-
go.opentelemetry.io/otel/trace v1.36.0 // indirect
149+
go.opentelemetry.io/otel v1.38.0 // indirect
150+
go.opentelemetry.io/otel/metric v1.38.0 // indirect
151+
go.opentelemetry.io/otel/trace v1.38.0 // indirect
151152
google.golang.org/appengine v1.6.8 // indirect
152-
google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect
153-
google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
154-
google.golang.org/genproto/googleapis/rpc v0.0.0-20250122153221-138b5a5a4fd4 // indirect
153+
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect
154+
google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda // indirect
155+
google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect
155156
)
156157

157158
// Embeddings and ML - indirect
@@ -262,17 +263,17 @@ require (
262263
github.com/gogo/protobuf v1.3.2 // indirect
263264
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
264265
github.com/golang/protobuf v1.5.4 // indirect
265-
github.com/golang/snappy v0.0.4 // indirect
266-
github.com/google/flatbuffers v23.5.26+incompatible // indirect
266+
github.com/golang/snappy v1.0.0 // indirect
267+
github.com/google/flatbuffers v25.12.19+incompatible // indirect
267268
github.com/google/go-querystring v1.1.0 // indirect
268269
github.com/goph/emperror v0.17.2 // indirect
269270
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
270271
github.com/huandu/xstrings v1.3.3 // indirect
271272
github.com/imdario/mergo v0.3.13 // indirect
272273
github.com/josharian/intern v1.0.0 // indirect
273274
github.com/json-iterator/go v1.1.12 // indirect
274-
github.com/klauspost/compress v1.18.0 // indirect
275-
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
275+
github.com/klauspost/compress v1.18.3 // indirect
276+
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
276277
github.com/kr/pretty v0.3.1 // indirect
277278
github.com/kr/text v0.2.0 // indirect
278279
github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
@@ -291,7 +292,7 @@ require (
291292
github.com/pelletier/go-toml/v2 v2.0.9 // indirect
292293
github.com/pkg/errors v0.9.1 // indirect
293294
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
294-
github.com/rogpeppe/go-internal v1.13.1 // indirect
295+
github.com/rogpeppe/go-internal v1.14.1 // indirect
295296
github.com/rs/zerolog v1.31.0 // indirect
296297
github.com/shirou/gopsutil/v4 v4.25.5 // indirect
297298
github.com/shopspring/decimal v1.2.0 // indirect
@@ -304,12 +305,12 @@ require (
304305
github.com/tklauser/numcpus v0.10.0 // indirect
305306
github.com/yargevad/filepathx v1.0.0 // indirect
306307
github.com/yusufpapurcu/wmi v1.2.4 // indirect
307-
golang.org/x/crypto v0.41.0 // indirect
308-
golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
309-
golang.org/x/net v0.43.0 // indirect
310-
golang.org/x/sys v0.35.0 // indirect
311-
golang.org/x/text v0.28.0 // indirect
312-
golang.org/x/time v0.9.0 // indirect
308+
golang.org/x/crypto v0.47.0 // indirect
309+
golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect
310+
golang.org/x/net v0.49.0 // indirect
311+
golang.org/x/sys v0.40.0 // indirect
312+
golang.org/x/text v0.33.0 // indirect
313+
golang.org/x/time v0.11.0 // indirect
313314
gopkg.in/yaml.v2 v2.4.0 // indirect
314315
gopkg.in/yaml.v3 v3.0.1 // indirect
315316
)
@@ -320,14 +321,23 @@ require (
320321
)
321322

322323
require (
324+
github.com/apache/arrow-go/v18 v18.5.1 // indirect
323325
github.com/beorn7/perks v1.0.1 // indirect
324326
github.com/blang/semver/v4 v4.0.0 // indirect
325327
github.com/cespare/xxhash/v2 v2.3.0 // indirect
326328
github.com/cilium/ebpf v0.11.0 // indirect
327329
github.com/containerd/cgroups/v3 v3.0.3 // indirect
328330
github.com/coreos/go-semver v0.3.0 // indirect
329331
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
332+
github.com/duckdb/duckdb-go-bindings v0.3.3 // indirect
333+
github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.3.3 // indirect
334+
github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.3.3 // indirect
335+
github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.3.3 // indirect
336+
github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.3.3 // indirect
337+
github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.3.3 // indirect
330338
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
339+
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
340+
github.com/goccy/go-json v0.10.5 // indirect
331341
github.com/godbus/dbus/v5 v5.0.4 // indirect
332342
github.com/google/btree v1.1.3 // indirect
333343
github.com/gorilla/websocket v1.5.0 // indirect
@@ -339,6 +349,7 @@ require (
339349
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
340350
github.com/opencontainers/runtime-spec v1.0.2 // indirect
341351
github.com/panjf2000/ants/v2 v2.11.3 // indirect
352+
github.com/pierrec/lz4/v4 v4.1.25 // indirect
342353
github.com/prometheus/client_golang v1.20.5 // indirect
343354
github.com/prometheus/client_model v0.6.1 // indirect
344355
github.com/prometheus/common v0.62.0 // indirect
@@ -354,6 +365,7 @@ require (
354365
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
355366
github.com/x448/float16 v0.8.4 // indirect
356367
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
368+
github.com/zeebo/xxh3 v1.1.0 // indirect
357369
go.etcd.io/bbolt v1.3.11 // indirect
358370
go.etcd.io/etcd/api/v3 v3.5.5 // indirect
359371
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
@@ -364,12 +376,15 @@ require (
364376
go.etcd.io/etcd/server/v3 v3.5.5 // indirect
365377
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect
366378
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect
367-
go.opentelemetry.io/otel/sdk v1.36.0 // indirect
379+
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
368380
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
369381
go.uber.org/atomic v1.11.0 // indirect
370382
go.uber.org/automaxprocs v1.5.3 // indirect
371383
go.uber.org/multierr v1.11.0 // indirect
372384
go.uber.org/zap v1.27.0 // indirect
385+
golang.org/x/mod v0.32.0 // indirect
386+
golang.org/x/telemetry v0.0.0-20260116145544-c6413dc483f5 // indirect
387+
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
373388
gopkg.in/inf.v0 v0.9.1 // indirect
374389
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
375390
k8s.io/apimachinery v0.28.6 // indirect

0 commit comments

Comments
 (0)