77 "encoding/json"
88 "fmt"
99 "github.com/gigapi/gigapi-config/config"
10+ "github.com/gigapi/gigapi/v2/merge/shared"
1011 "github.com/gigapi/metadata"
1112 "log"
1213 "os"
@@ -392,14 +393,31 @@ func (q *QueryClient) enumFolderNoMetadata(path string) ([]string, error) {
392393 return res , nil
393394}
394395
395- func getIndex (db , measurement string ) (metadata.TableIndex , error ) {
396+ func getTableIndex (table * shared.Table ) (metadata.TableIndex , error ) {
397+ layers := make ([]metadata.Layer , len (config .Config .Gigapi .Layers ))
398+ for i , l := range config .Config .Gigapi .Layers {
399+ layers [i ] = metadata.Layer {
400+ URL : l .URL ,
401+ Name : l .Name ,
402+ Type : l .Type ,
403+ TTLSec : int32 (l .TTL .Seconds ()),
404+ }
405+ }
396406 switch config .Config .Gigapi .Metadata .Type {
397407 case "json" :
398- return metadata .NewJSONIndex (config .Config .Gigapi .Root , db , measurement ), nil
408+ return metadata .NewJSONIndex (
409+ config .Config .Gigapi .Root ,
410+ table .Database ,
411+ table .Name ,
412+ layers )
399413 case "redis" :
400- return metadata .NewRedisIndex (config .Config .Gigapi .Metadata .URL , db , measurement )
414+ return metadata .NewRedisIndex (
415+ config .Config .Gigapi .Metadata .URL ,
416+ table .Database ,
417+ table .Name ,
418+ layers )
401419 }
402- return nil , fmt .Errorf ("Unsupported index " + config .Config .Gigapi .Metadata .Type )
420+ return nil , fmt .Errorf ("unknown metadata type: %q" , config .Config .Gigapi .Metadata .Type )
403421}
404422
405423// Find relevant parquet files based on time range
@@ -424,14 +442,21 @@ func (q *QueryClient) FindRelevantFiles(ctx context.Context, dbName, measurement
424442 opts .Before = time .Unix (0 , * timeRange .End )
425443 }
426444
427- idx , err := getIndex (dbName , measurement )
445+ idx , err := getTableIndex (& shared.Table {
446+ Database : dbName ,
447+ Name : measurement ,
448+ })
428449 if err != nil {
429450 return nil , err
430451 }
431452
432453 ies , err := idx .GetQuerier ().Query (opts )
454+ layersMap := make (map [string ]string )
455+ for _ , l := range config .Config .Gigapi .Layers {
456+ layersMap [l .Name ] = strings .TrimPrefix (l .URL , "file://" )
457+ }
433458 for _ , ie := range ies {
434- relevantFiles = append (relevantFiles , filepath .Join (config . Config . Gigapi . Root , dbName , measurement , "data" , ie .Path ))
459+ relevantFiles = append (relevantFiles , filepath .Join (layersMap [ ie . Layer ] , dbName , measurement , "data" , ie .Path ))
435460 }
436461
437462 if len (relevantFiles ) == 0 {
@@ -616,6 +641,25 @@ func (q *QueryClient) getHourDirectoriesInRange(datePath string, startDate, endD
616641 return hourDirs , nil
617642}
618643
644+ func (c * QueryClient ) getDBIndex (ctx context.Context ) (metadata.DBIndex , error ) {
645+ var layers []metadata.Layer
646+ for _ , l := range config .Config .Gigapi .Layers {
647+ layers = append (layers , metadata.Layer {
648+ URL : l .URL ,
649+ Name : l .Name ,
650+ Type : l .Type ,
651+ TTLSec : int32 (l .TTL .Seconds ()),
652+ })
653+ }
654+ switch config .Config .Gigapi .Metadata .Type {
655+ case "json" :
656+ return metadata .NewJSONDBIndex (layers ), nil
657+ case "redis" :
658+ return metadata .NewRedisDbIndex (config .Config .Gigapi .Metadata .URL )
659+ }
660+ return nil , fmt .Errorf ("unsupported metadata type: %s" , config .Config .Gigapi .Metadata .Type )
661+ }
662+
619663// Query executes a query against the database
620664func (c * QueryClient ) Query (ctx context.Context , query , dbName string ) ([]map [string ]interface {}, error ) {
621665 // Ensure we have a context
@@ -633,36 +677,39 @@ func (c *QueryClient) Query(ctx context.Context, query, dbName string) ([]map[st
633677 // Handle special commands
634678 switch upperQuery {
635679 case "SHOW DATABASES" :
636- entries , err := os .ReadDir (c .DataDir )
680+ idx , err := c .getDBIndex (ctx )
681+ if err != nil {
682+ return nil , fmt .Errorf ("failed to get DB index: %v" , err )
683+ }
684+ entries , err := idx .Databases ()
637685 if err != nil {
638686 return nil , fmt .Errorf ("failed to read data directory: %v" , err )
639687 }
640688
641689 results := make ([]map [string ]interface {}, 0 )
642690 for _ , entry := range entries {
643- if entry .IsDir () {
644- results = append (results , map [string ]interface {}{
645- "database_name" : entry .Name (),
646- })
647- }
691+ results = append (results , map [string ]interface {}{
692+ "database_name" : entry ,
693+ })
648694 }
649695 return results , nil
650696
651697 case "SHOW TABLES" :
652698 // List directories inside the database folder
653- dbPath := filepath .Join (c .DataDir , dbName )
654- entries , err := os .ReadDir (dbPath )
699+ idx , err := c .getDBIndex (ctx )
700+ if err != nil {
701+ return nil , fmt .Errorf ("failed to get DB index: %v" , err )
702+ }
703+ entries , err := idx .Tables (dbName )
655704 if err != nil {
656705 return nil , fmt .Errorf ("failed to read database directory: %v" , err )
657706 }
658707
659708 results := make ([]map [string ]interface {}, 0 )
660709 for _ , entry := range entries {
661- if entry .IsDir () {
662- results = append (results , map [string ]interface {}{
663- "table_name" : entry .Name (),
664- })
665- }
710+ results = append (results , map [string ]interface {}{
711+ "table_name" : entry ,
712+ })
666713 }
667714 return results , nil
668715 }
@@ -672,13 +719,13 @@ func (c *QueryClient) Query(ctx context.Context, query, dbName string) ([]map[st
672719 if err != nil {
673720 // Fallback: Directly execute the query in DuckDB if ParseQuery fails
674721 // Set DuckDB safety option to disable external access
675- if _ , errSet := c .DB .Exec ("SET enable_external_access = false;" ); errSet != nil {
676- return nil , fmt .Errorf ("failed to disable external access: %v" , errSet )
722+ /* if _, errSet := c.DB.Exec("SET enable_external_access = false;"); errSet != nil {
723+ return nil, fmt.Errorf("failed to disable external access: %v", errSet)
677724 }
678725 if _, errSet := c.DB.Exec("SET lock_configuration = true;"); errSet != nil {
679- return nil , fmt .Errorf ("failed to lock configuration: %v" , errSet )
680- }
681-
726+ return nil, fmt.Errorf("failed to lock configuration: %v", errSet)
727+ }*/
728+
682729 stmt , err2 := c .DB .Prepare (query )
683730 if err2 != nil {
684731 return nil , fmt .Errorf ("failed to prepare query: %v" , err2 )
0 commit comments