Merge pull request #25 from gigapi/feat/metadata

Feat/metadata

Author: akvlad

go.mod

@@ -4,8 +4,9 @@ go 1.24.2
 
 require (
 	github.com/apache/arrow/go/v14 v14.0.2
-	github.com/gigapi/gigapi-config v0.0.6
+	github.com/gigapi/gigapi-config v0.0.7
 	github.com/gigapi/gigapi/v2 v2.0.13
+	github.com/gigapi/metadata v0.0.3
 	github.com/marcboeker/go-duckdb/v2 v2.2.0
 	github.com/spf13/afero v1.12.0
 	github.com/stretchr/testify v1.10.0
@@ -16,7 +17,9 @@ require (
 
 require (
 	github.com/apache/arrow-go/v18 v18.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/duckdb/duckdb-go-bindings v0.1.14 // indirect
 	github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.9 // indirect
 	github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.9 // indirect
@@ -29,15 +32,19 @@ require (
 	github.com/google/flatbuffers v25.1.24+incompatible // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/marcboeker/go-duckdb/arrowmapping v0.0.7 // indirect
 	github.com/marcboeker/go-duckdb/mapping v0.0.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/redis/go-redis/v9 v9.8.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect

go.sum

@@ -6,10 +6,18 @@ github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO
 github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY=
 github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
 github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/duckdb/duckdb-go-bindings v0.1.14 h1:57DCZuuKQ65gRQxFG+XGnqVQtMADKY/noozmCjYs+zE=
 github.com/duckdb/duckdb-go-bindings v0.1.14/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc=
 github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.9 h1:K95YlR28Fb3+n3D6RcBzdznNVGcCnrGaAZqs52JUFOs=
@@ -28,10 +36,12 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
-github.com/gigapi/gigapi-config v0.0.6 h1:WTDJZt70uEbCLwikSKMwWYysZtfp7AoLOLtNiurX6qA=
-github.com/gigapi/gigapi-config v0.0.6/go.mod h1:/hD+d1odWyElSP9++ZPLULXthWHXvx0shNypdbZDYA8=
+github.com/gigapi/gigapi-config v0.0.7 h1:JBygAXojXdTmxhKe2vTSvXoPZU0D7kLWl9+E+PuUQJs=
+github.com/gigapi/gigapi-config v0.0.7/go.mod h1:/hD+d1odWyElSP9++ZPLULXthWHXvx0shNypdbZDYA8=
 github.com/gigapi/gigapi/v2 v2.0.13 h1:m4w2Sx+AQZl+SPS+5oCtCe7G1888/2nF6m+d1vsiHlQ=
 github.com/gigapi/gigapi/v2 v2.0.13/go.mod h1:xoh5uz+EQCK43cI08wMJlyhAZ1368OOPlJjmOr1TGis=
+github.com/gigapi/metadata v0.0.3 h1:foO8cD4JT3HbzGUkpEda7jHgiGXpaK74t4x4cSMxO3o=
+github.com/gigapi/metadata v0.0.3/go.mod h1:LGFPrq9868U3+4S5DdoL1/aU3Ua0mXsRVblXXzCK1i0=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
@@ -48,13 +58,16 @@ github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLK
 github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
@@ -81,6 +94,11 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
 github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
 github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
@@ -90,6 +108,8 @@ github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFu
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI=
+github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@@ -113,6 +133,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=

querier/queryClient.go

@@ -6,6 +6,8 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"github.com/gigapi/gigapi-config/config"
+	"github.com/gigapi/metadata"
 	"log"
 	"os"
 	"path/filepath"
@@ -390,15 +392,20 @@ func (q *QueryClient) enumFolderNoMetadata(path string) ([]string, error) {
 	return res, nil
 }
 
+func getIndex(db, measurement string) (metadata.TableIndex, error) {
+	switch config.Config.Gigapi.Metadata.Type {
+	case "json":
+		return metadata.NewJSONIndex(config.Config.Gigapi.Root, db, measurement), nil
+	case "redis":
+		return metadata.NewRedisIndex(config.Config.Gigapi.Metadata.URL, db, measurement)
+	}
+	return nil, fmt.Errorf("Unsupported index " + config.Config.Gigapi.Metadata.Type)
+}
+
 // Find relevant parquet files based on time range
 func (q *QueryClient) FindRelevantFiles(ctx context.Context, dbName, measurement string,
 	timeRange TimeRange) ([]string, error) {
 	// If no time range specified, get all files
-	if timeRange.Start == nil && timeRange.End == nil {
-		log.Printf("No time range specified, getting all files for %s.%s", dbName, measurement)
-		return q.findAllFiles(ctx, dbName, measurement)
-	}
-
 	var relevantFiles []string
 	// log.Printf("Getting relevant files for %s.%s within time range %v to %v", dbName, measurement,
 	// time.Unix(0, *timeRange.Start), time.Unix(0, *timeRange.End))
@@ -408,65 +415,23 @@ func (q *QueryClient) FindRelevantFiles(ctx context.Context, dbName, measurement
 	}()
 
 	// Convert nanosecond timestamps to time.Time for directory parsing
-	var startDate, endDate time.Time
+	var opts metadata.QueryOptions
 	if timeRange.Start != nil {
-		startDate = time.Unix(0, *timeRange.Start)
-	} else {
-		startDate = time.Unix(0, 0) // Beginning of epoch
+		opts.After = time.Unix(0, *timeRange.Start)
 	}
 
 	if timeRange.End != nil {
-		endDate = time.Unix(0, *timeRange.End)
-	} else {
-		endDate = time.Now() // Current time
+		opts.Before = time.Unix(0, *timeRange.End)
 	}
 
-	// Get all date directories that might contain relevant data
-	// log.Printf("Looking for date directories between %v and %v", startDate, endDate)
-	dateDirectories, err := q.getDateDirectoriesInRange(dbName, measurement, startDate, endDate)
+	idx, err := getIndex(dbName, measurement)
 	if err != nil {
-		log.Printf("Failed to get date directories: %v", err)
 		return nil, err
 	}
-	// log.Printf("Found %d date directories", len(dateDirectories))
-
-	for _, dateDir := range dateDirectories {
-		// For each date directory, get all hour directories
-		datePath := filepath.Join(q.DataDir, dbName, measurement, dateDir)
-		// log.Printf("Processing date directory: %s", datePath)
-
-		hourDirs, err := q.getHourDirectoriesInRange(datePath, startDate, endDate)
-		if err != nil {
-			log.Printf("Failed to get hour directories for %s: %v", datePath, err)
-			continue // Skip this directory on error
-		}
-		// log.Printf("Found %d hour directories in %s", len(hourDirs), dateDir)
-
-		for _, hourDir := range hourDirs {
-			hourPath := filepath.Join(datePath, hourDir)
-			// log.Printf("Processing hour directory: %s", hourPath)
 
-			// Read metadata.json
-			metadataPath := filepath.Join(hourPath, "metadata.json")
-			if _, err := os.Stat(metadataPath); err == nil {
-				// log.Printf("Found metadata.json in %s", hourPath)
-				_relevantFiles, err := q.enumFolderWithMetadata(metadataPath, timeRange)
-				if err == nil {
-					// log.Printf("Found %d files in metadata.json", len(_relevantFiles))
-					relevantFiles = append(relevantFiles, _relevantFiles...)
-					continue
-				}
-				log.Printf("Failed to read metadata.json: %v", err)
-			}
-
-			_relevantFiles, err := q.enumFolderNoMetadata(hourPath)
-			if err == nil {
-				// log.Printf("Found %d files without metadata", len(_relevantFiles))
-				relevantFiles = append(relevantFiles, _relevantFiles...)
-				continue
-			}
-			log.Printf("Failed to enumerate folder: %v", err)
-		}
+	ies, err := idx.GetQuerier().Query(opts)
+	for _, ie := range ies {
+		relevantFiles = append(relevantFiles, filepath.Join(config.Config.Gigapi.Root, dbName, measurement, "data", ie.Path))
 	}
 
 	if len(relevantFiles) == 0 {