feat(valkey): add Valkey indexer and retriever components

.gitignore

@@ -42,6 +42,9 @@ output/*
 .claude/
 .claude-trace
 
+# Kiro files
+.kiro/
+
 # Data files
 *.jsonl
 *.csv

components/indexer/valkey/README.md

@@ -0,0 +1,147 @@
+# Valkey Indexer
+
+A Valkey indexer implementation for [Eino](https://github.com/cloudwego/eino) that implements the `Indexer` interface. This component uses Valkey Hashes to store documents with vector embeddings via the [valkey-glide](https://github.com/valkey-io/valkey-glide) client, enabling vector similarity search capabilities.
+
+## Features
+
+- Implements `github.com/cloudwego/eino/components/indexer.Indexer`
+- Uses Valkey Hashes (HSET) or JSON (JSON.SET) for document storage
+- Pipeline batch execution for high throughput
+- Automatic embedding generation with configurable batch size
+- Custom field mapping via `DocumentToHashes` or `DocumentToJSON` function
+- Configurable key prefix for index partitioning
+- Full Eino callback integration (OnStart, OnEnd, OnError)
+
+## Installation
+
+```bash
+go get github.com/cloudwego/eino-ext/components/indexer/valkey@latest
+```
+
+## Prerequisites
+
+- Valkey 9.1+ with the Search module (e.g., `valkey/valkey-bundle`)
+- [valkey-glide](https://github.com/valkey-io/valkey-glide) Go client (requires CGO and Rust core library)
+
+## Quick Start
+
+```go
+import (
+	"context"
+	"fmt"
+
+	glide "github.com/valkey-io/valkey-glide/go/v2"
+	"github.com/valkey-io/valkey-glide/go/v2/config"
+	"github.com/cloudwego/eino/schema"
+
+	valkeyIndexer "github.com/cloudwego/eino-ext/components/indexer/valkey"
+)
+
+func main() {
+	ctx := context.Background()
+
+	// 1. Create Valkey GLIDE client
+	cfg := config.NewClientConfiguration().
+		WithAddress(&config.NodeAddress{Host: "localhost", Port: 6379})
+	client, _ := glide.NewClient(cfg)
+
+	// 2. Create embedding component (use your preferred embedder)
+	emb := yourEmbedder()
+
+	// 3. Create Valkey indexer
+	indexer, _ := valkeyIndexer.NewIndexer(ctx, &valkeyIndexer.IndexerConfig{
+		Client:    client,
+		KeyPrefix: "doc:",
+		BatchSize: 10,
+		Embedding: emb,
+	})
+
+	// 4. Store documents
+	docs := []*schema.Document{
+		{ID: "1", Content: "Valkey is a high-performance key-value store."},
+		{ID: "2", Content: "Vector search enables semantic similarity queries."},
+	}
+
+	ids, err := indexer.Store(ctx, docs)
+	if err != nil {
+		fmt.Printf("store error: %v\n", err)
+		return
+	}
+	fmt.Printf("stored document IDs: %v\n", ids)
+}
+```
+
+## Configuration
+
+```go
+type IndexerConfig struct {
+    // Required: Valkey GLIDE client (must implement Exec for pipeline batching)
+    Client BatchClient
+
+    // Optional: Key prefix prepended to each hash key
+    // Should match the prefix used in FT.CREATE
+    KeyPrefix string
+
+    // Optional: Storage format - DocumentTypeHash (default) or DocumentTypeJSON
+    DocumentType DocumentType
+
+    // Optional: Custom document-to-hash conversion (Hash mode only)
+    // Default: stores Content with embedding, plus all MetaData fields
+    DocumentToHashes func(ctx context.Context, doc *schema.Document) (*Hashes, error)
+
+    // Optional: Custom document-to-JSON conversion (JSON mode only)
+    // Default: stores content, vector, and metadata fields
+    DocumentToJSON func(ctx context.Context, doc *schema.Document, vector []float64) (map[string]any, error)
+
+    // Optional: Max texts per embedding batch call (default: 10)
+    BatchSize int
+
+    // Required: Embedding method for vectorizing document content
+    Embedding embedding.Embedder
+}
+```
+
+## JSON Document Storage
+
+For JSON storage, use `DocumentTypeJSON`. This requires the Valkey JSON module and an index created with `ON JSON`:
+
+```go
+indexer, _ := valkeyIndexer.NewIndexer(ctx, &valkeyIndexer.IndexerConfig{
+    Client:       client,
+    KeyPrefix:    "jdoc:",
+    DocumentType: valkeyIndexer.DocumentTypeJSON,
+    Embedding:    emb,
+})
+```
+
+Create the index with JSONPath field identifiers:
+```
+FT.CREATE my_json_index ON JSON PREFIX 1 jdoc: SCHEMA
+  $.content AS content TEXT
+  $.vector_content AS vector_content VECTOR HNSW 6 TYPE FLOAT32 DIM 1024 DISTANCE_METRIC COSINE
+```
+
+## Custom Field Mapping
+
+```go
+indexer, _ := valkeyIndexer.NewIndexer(ctx, &valkeyIndexer.IndexerConfig{
+    Client:    client,
+    KeyPrefix: "doc:",
+    DocumentToHashes: func(ctx context.Context, doc *schema.Document) (*valkeyIndexer.Hashes, error) {
+        return &valkeyIndexer.Hashes{
+            Key: doc.ID,
+            Field2Value: map[string]valkeyIndexer.FieldValue{
+                "title": {Value: doc.MetaData["title"]},
+                "body":  {Value: doc.Content, EmbedKey: "body_vector"},
+            },
+        }, nil
+    },
+    Embedding: emb,
+})
+```
+
+## For More Details
+
+- [Eino Documentation](https://www.cloudwego.io/zh/docs/eino/)
+- [Valkey GLIDE Go Client](https://github.com/valkey-io/valkey-glide)
+- [Valkey Search](https://valkey.io/topics/search/)

components/indexer/valkey/consts.go

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2026 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package valkey
+
+const (
+	defaultReturnFieldContent       = "content"
+	defaultReturnFieldVectorContent = "vector_content"
+)

components/indexer/valkey/examples/hash_indexer/hash_indexer.go

@@ -0,0 +1,90 @@
+//go:build ignore
+
+/*
+ * Copyright 2026 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	glide "github.com/valkey-io/valkey-glide/go/v2"
+	"github.com/valkey-io/valkey-glide/go/v2/config"
+
+	"github.com/cloudwego/eino/components/embedding"
+	"github.com/cloudwego/eino/schema"
+
+	vi "github.com/cloudwego/eino-ext/components/indexer/valkey"
+)
+
+// This example demonstrates indexing documents as Valkey Hashes.
+// Prerequisites:
+//   - Valkey 9.1+ with Search module running on localhost:6379
+//   - Create an index first:
+//     FT.CREATE my_index ON HASH PREFIX 1 doc: SCHEMA
+//     content TEXT vector_content VECTOR HNSW 6 TYPE FLOAT32 DIM 1024 DISTANCE_METRIC COSINE
+func main() {
+	ctx := context.Background()
+
+	// 1. Create Valkey GLIDE client
+	cfg := config.NewClientConfiguration().
+		WithAddress(&config.NodeAddress{Host: "localhost", Port: 6379})
+	client, err := glide.NewClient(cfg)
+	if err != nil {
+		panic(fmt.Sprintf("failed to create client: %v", err))
+	}
+	defer client.Close()
+
+	// 2. Create indexer
+	indexer, err := vi.NewIndexer(ctx, &vi.IndexerConfig{
+		Client:       client,
+		KeyPrefix:    "doc:",
+		DocumentType: vi.DocumentTypeHash,
+		BatchSize:    10,
+		Embedding:    &mockEmbedding{dims: 1024}, // replace with real embedder
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	// 3. Store documents
+	docs := []*schema.Document{
+		{ID: "1", Content: "Eiffel Tower: Located in Paris, France"},
+		{ID: "2", Content: "Great Wall of China: One of the greatest wonders of the world"},
+		{ID: "3", Content: "Grand Canyon: Located in Arizona, USA"},
+	}
+
+	ids, err := indexer.Store(ctx, docs)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("Stored %d documents: %v\n", len(ids), ids)
+}
+
+// mockEmbedding is a placeholder - replace with a real embedding implementation.
+type mockEmbedding struct {
+	dims int
+}
+
+func (m *mockEmbedding) EmbedStrings(_ context.Context, texts []string, _ ...embedding.Option) ([][]float64, error) {
+	vectors := make([][]float64, len(texts))
+	for i := range texts {
+		vectors[i] = make([]float64, m.dims)
+	}
+	return vectors, nil
+}

components/indexer/valkey/examples/json_indexer/json_indexer.go

@@ -0,0 +1,90 @@
+//go:build ignore
+
+/*
+ * Copyright 2026 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	glide "github.com/valkey-io/valkey-glide/go/v2"
+	"github.com/valkey-io/valkey-glide/go/v2/config"
+
+	"github.com/cloudwego/eino/components/embedding"
+	"github.com/cloudwego/eino/schema"
+
+	vi "github.com/cloudwego/eino-ext/components/indexer/valkey"
+)
+
+// This example demonstrates indexing documents as Valkey JSON.
+// Prerequisites:
+//   - Valkey 9.1+ with Search and JSON modules running on localhost:6379
+//   - Create an index first:
+//     FT.CREATE my_json_index ON JSON PREFIX 1 jdoc: SCHEMA
+//     $.content AS content TEXT $.vector_content AS vector_content VECTOR HNSW 6
+//     TYPE FLOAT32 DIM 1024 DISTANCE_METRIC COSINE
+func main() {
+	ctx := context.Background()
+
+	// 1. Create Valkey GLIDE client
+	cfg := config.NewClientConfiguration().
+		WithAddress(&config.NodeAddress{Host: "localhost", Port: 6379})
+	client, err := glide.NewClient(cfg)
+	if err != nil {
+		panic(fmt.Sprintf("failed to create client: %v", err))
+	}
+	defer client.Close()
+
+	// 2. Create JSON indexer
+	indexer, err := vi.NewIndexer(ctx, &vi.IndexerConfig{
+		Client:       client,
+		KeyPrefix:    "jdoc:",
+		DocumentType: vi.DocumentTypeJSON,
+		BatchSize:    10,
+		Embedding:    &mockEmbedding{dims: 1024}, // replace with real embedder
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	// 3. Store documents as JSON
+	docs := []*schema.Document{
+		{ID: "1", Content: "Eiffel Tower: Located in Paris, France", MetaData: map[string]any{"category": "europe"}},
+		{ID: "2", Content: "Great Wall of China", MetaData: map[string]any{"category": "asia"}},
+	}
+
+	ids, err := indexer.Store(ctx, docs)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("Stored %d JSON documents: %v\n", len(ids), ids)
+}
+
+// mockEmbedding is a placeholder - replace with a real embedding implementation.
+type mockEmbedding struct {
+	dims int
+}
+
+func (m *mockEmbedding) EmbedStrings(_ context.Context, texts []string, _ ...embedding.Option) ([][]float64, error) {
+	vectors := make([][]float64, len(texts))
+	for i := range texts {
+		vectors[i] = make([]float64, m.dims)
+	}
+	return vectors, nil
+}