From 45341ca962a9d27ddbccde4de45a663ce64db606 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 6 Apr 2026 14:26:16 +0300 Subject: [PATCH 1/8] fix(gcp): replace deprecated gcp credentials method with typed variant Use CredentialsFromJSONWithType with explicit ServiceAccount type instead of deprecated CredentialsFromJSON to remove staticcheck linter suppression. --- connection/gcp.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/connection/gcp.go b/connection/gcp.go index c1b3db5e8..818f48da7 100644 --- a/connection/gcp.go +++ b/connection/gcp.go @@ -45,13 +45,12 @@ func (t *GCPConnection) FromModel(connection models.Connection) { } func (g *GCPConnection) TokenSource(ctx context.Context, scopes ...string) (oauth2.TokenSource, error) { - creds, err := google.CredentialsFromJSON(ctx, []byte(g.Credentials.ValueStatic), scopes...) //nolint:staticcheck + creds, err := google.CredentialsFromJSONWithParams(ctx, []byte(g.Credentials.ValueStatic), google.CredentialsParams{Scopes: scopes}) if err != nil { return nil, err } - tokenSource := creds.TokenSource - return tokenSource, nil + return creds.TokenSource, nil } func (g *GCPConnection) Validate() *GCPConnection { From 270c097fc8f5cddce37f3fb413a6a496a76872f3 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 6 Apr 2026 14:29:50 +0300 Subject: [PATCH 2/8] feat(db): add artifact_summary view with storage and connection metadata New view aggregates artifact counts and sizes grouped by content type, storage location, and connection with associated metadata for reporting and analytics. --- rbac/objects.go | 1 + views/046_artifact_summary.sql | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 views/046_artifact_summary.sql diff --git a/rbac/objects.go b/rbac/objects.go index a8469329f..dc45de7f8 100644 --- a/rbac/objects.go +++ b/rbac/objects.go @@ -24,6 +24,7 @@ var dbResourceObjMap = map[string]string{ "analysis_types": policy.ObjectDatabasePublic, "analyzer_types": policy.ObjectDatabasePublic, "artifacts": policy.ObjectArtifact, + "artifact_summary": policy.ObjectArtifact, "canaries_with_status": policy.ObjectCanary, "canaries": policy.ObjectCanary, "casbin_rule": policy.ObjectAuth, diff --git a/views/046_artifact_summary.sql b/views/046_artifact_summary.sql new file mode 100644 index 000000000..d04afd5bf --- /dev/null +++ b/views/046_artifact_summary.sql @@ -0,0 +1,18 @@ +CREATE OR REPLACE VIEW artifact_summary AS +SELECT + content_type, + CASE WHEN a.content IS NOT NULL THEN 'inline' ELSE 'external' END AS storage, + a.connection_id, + c.name AS connection_name, + c.type AS connection_type, + COUNT(*) AS total_count, + COALESCE(SUM(a.size), 0) AS total_size +FROM artifacts a +LEFT JOIN connections c ON a.connection_id = c.id +WHERE a.deleted_at IS NULL +GROUP BY + content_type, + (CASE WHEN a.content IS NOT NULL THEN 'inline' ELSE 'external' END), + a.connection_id, + c.name, + c.type; From a88d32e20fce60f0e69180a061163841bb2f45c6 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 6 Apr 2026 14:31:11 +0300 Subject: [PATCH 3/8] feat(query): add structured query logging with timing and result summaries Introduces QueryLogger/QueryTimer to capture query execution time, argument details, and result counts. Refactors multiple query functions to use new logging infrastructure. Extracts BaseCatalogSearch to consolidate common search parameters and validation logic across catalog search endpoints. --- query/agent.go | 12 +- query/catalog_search.go | 185 ++++++++++++++++++++++ query/config.go | 59 ++++--- query/config_access.go | 49 +++++- query/config_changes.go | 263 ++++++++------------------------ query/config_insights.go | 110 +++++++++++++ query/config_relations.go | 22 ++- query/query_logger.go | 119 +++++++++++++++ tests/config_changes_test.go | 160 ++++++++++++------- tests/e2e-blobs/test.properties | 2 + tests/setup/template.go | 241 +++++++++++++++++++++++++++++ 11 files changed, 924 insertions(+), 298 deletions(-) create mode 100644 query/catalog_search.go create mode 100644 query/config_insights.go create mode 100644 query/query_logger.go create mode 100644 tests/e2e-blobs/test.properties create mode 100644 tests/setup/template.go diff --git a/query/agent.go b/query/agent.go index d20b3594f..517823bd9 100644 --- a/query/agent.go +++ b/query/agent.go @@ -11,17 +11,21 @@ import ( "gorm.io/gorm" ) -func FindAgent(ctx context.Context, name string) (*models.Agent, error) { +func FindAgent(ctx context.Context, name string) (result *models.Agent, err error) { + timer := NewQueryLogger(ctx).Start("Agent").Arg("name", name) + defer timer.End(&err) + var agent models.Agent - err := ctx.DB().Where("name = ?", name).First(&agent).Error + err = ctx.DB().Where("name = ?", name).First(&agent).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { + err = nil + timer.Results([]models.Agent{}) return nil, nil } - return nil, err } - + timer.Results([]models.Agent{agent}) return &agent, nil } diff --git a/query/catalog_search.go b/query/catalog_search.go new file mode 100644 index 000000000..e8693a516 --- /dev/null +++ b/query/catalog_search.go @@ -0,0 +1,185 @@ +package query + +import ( + "fmt" + "strings" + "time" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/pkg/kube/labels" + "github.com/flanksource/duty/types" + "github.com/google/uuid" + "github.com/timberio/go-datemath" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type BaseCatalogSearch struct { + CatalogID string `query:"id" json:"id"` + ConfigType string `query:"config_type" json:"config_type"` + IncludeDeletedConfigs bool `query:"include_deleted_configs" json:"include_deleted_configs"` + Depth int `query:"depth" json:"depth"` + Tags string `query:"tags" json:"tags"` + AgentID string `query:"agent_id" json:"agent_id"` + From string `query:"from" json:"from"` + To string `query:"to" json:"to"` + PageSize int `query:"page_size" json:"page_size"` + Page int `query:"page" json:"page"` + SortBy string `query:"sort_by" json:"sort_by"` + Recursive ChangeRelationDirection `query:"recursive" json:"recursive"` + Soft bool `query:"soft" json:"soft"` + + sortOrder string + configIDs []uuid.UUID + FromTime *time.Time `query:"-" json:"-"` + ToTime *time.Time `query:"-" json:"-"` +} + +func (b *BaseCatalogSearch) SetDefaults() { + if b.PageSize <= 0 { + b.PageSize = 50 + } + if b.Page <= 0 { + b.Page = 1 + } + if b.Depth <= 0 { + b.Depth = 5 + } + if b.Recursive == "" { + b.Recursive = CatalogChangeRecursiveDownstream + } + if b.AgentID == "local" { + b.AgentID = uuid.Nil.String() + } +} + +func (b *BaseCatalogSearch) Validate() error { + if b.From != "" && b.FromTime == nil { + expr, err := datemath.Parse(b.From) + if err != nil { + return fmt.Errorf("invalid 'from' param: %w", err) + } + t := expr.Time() + b.FromTime = &t + } + if b.To != "" && b.ToTime == nil { + expr, err := datemath.Parse(b.To) + if err != nil { + return fmt.Errorf("invalid 'to' param: %w", err) + } + t := expr.Time() + b.ToTime = &t + } + if b.FromTime != nil && b.ToTime != nil && !b.FromTime.Before(*b.ToTime) { + return fmt.Errorf("'from' must be before 'to'") + } + if b.AgentID != "" { + if _, err := uuid.Parse(b.AgentID); err != nil { + return fmt.Errorf("agent_id(%s) must either be a valid uuid or `local`", b.AgentID) + } + } + return nil +} + +func (b *BaseCatalogSearch) ResolveConfigIDs(ctx context.Context) ([]uuid.UUID, error) { + if b.CatalogID == "" { + return nil, nil + } + parts := strings.Split(b.CatalogID, ",") + var ids []uuid.UUID + allValid := true + for _, p := range parts { + if id, err := uuid.Parse(strings.TrimSpace(p)); err == nil { + ids = append(ids, id) + } else { + allValid = false + break + } + } + if allValid && len(ids) > 0 { + b.configIDs = ids + return ids, nil + } + + response, err := SearchResources(ctx, SearchResourcesRequest{ + Configs: []types.ResourceSelector{{Search: b.CatalogID, Cache: "no-cache"}}, + Limit: 200, + }) + if err != nil { + return nil, fmt.Errorf("failed to resolve catalog query %q: %w", b.CatalogID, err) + } + for _, c := range response.Configs { + if id, err := uuid.Parse(c.ID); err == nil { + ids = append(ids, id) + } + } + b.configIDs = ids + return ids, nil +} + +func (b *BaseCatalogSearch) ConfigIDs() []uuid.UUID { + return b.configIDs +} + +func (b *BaseCatalogSearch) ApplyClauses() ([]clause.Expression, func(*gorm.DB) *gorm.DB) { + var clauses []clause.Expression + var tagsFn func(*gorm.DB) *gorm.DB + + if b.AgentID != "" { + if c, err := parseAndBuildFilteringQuery(b.AgentID, "agent_id", false); err == nil { + clauses = append(clauses, c...) + } + } + if b.ConfigType != "" { + if c, err := parseAndBuildFilteringQuery(b.ConfigType, "type", false); err == nil { + clauses = append(clauses, c...) + } + } + if b.FromTime != nil { + clauses = append(clauses, clause.Gte{Column: clause.Column{Name: "created_at"}, Value: *b.FromTime}) + } + if b.ToTime != nil { + clauses = append(clauses, clause.Lte{Column: clause.Column{Name: "created_at"}, Value: *b.ToTime}) + } + if !b.IncludeDeletedConfigs { + clauses = append(clauses, clause.Eq{Column: clause.Column{Name: "deleted_at"}, Value: nil}) + } + if b.Tags != "" { + if parsedLabelSelector, err := labels.Parse(b.Tags); err == nil { + requirements, _ := parsedLabelSelector.Requirements() + tagsFn = func(db *gorm.DB) *gorm.DB { + for _, r := range requirements { + db = jsonColumnRequirementsToSQLClause(db, "tags", r) + } + return db + } + } + } + return clauses, tagsFn +} + +func (b *BaseCatalogSearch) String() string { + s := "" + if b.AgentID != "" { + s += fmt.Sprintf("agent=%s ", b.AgentID) + } + if b.CatalogID != "" { + s += fmt.Sprintf("id=%s ", b.CatalogID) + } + if b.ConfigType != "" { + s += fmt.Sprintf("config_type=%s ", b.ConfigType) + } + if b.Tags != "" { + s += fmt.Sprintf("tags=%s ", b.Tags) + } + if b.From != "" { + s += fmt.Sprintf("from=%s ", b.From) + } + if b.To != "" { + s += fmt.Sprintf("to=%s ", b.To) + } + if b.Recursive != "" { + s += fmt.Sprintf("recursive=%s ", b.Recursive) + } + return strings.TrimSpace(s) +} diff --git a/query/config.go b/query/config.go index be91777b3..6dd6e4c78 100644 --- a/query/config.go +++ b/query/config.go @@ -460,7 +460,10 @@ func FindConfigIDsByResourceSelector(ctx context.Context, limit int, resourceSel return queryTableWithResourceSelectors(ctx, "config_items", limit, resourceSelectors...) } -func FindConfigForComponent(ctx context.Context, componentID, configType string) ([]models.ConfigItem, error) { +func FindConfigForComponent(ctx context.Context, componentID, configType string) (results []models.ConfigItem, err error) { + timer := NewQueryLogger(ctx).Start("ConfigForComponent").Arg("componentID", componentID).Arg("configType", configType) + defer timer.End(&err) + db := ctx.DB() relationshipQuery := db.Table("config_component_relationships"). Select("config_id"). @@ -469,27 +472,31 @@ func FindConfigForComponent(ctx context.Context, componentID, configType string) if configType != "" { query = query.Where("type = @config_type OR config_class = @config_type", sql.Named("config_type", configType)) } - var dbConfigObjects []models.ConfigItem - err := query.Find(&dbConfigObjects).Error - return dbConfigObjects, err + err = query.Find(&results).Error + timer.Results(results) + return results, err } -func FindConfigChildrenIDsByLocation(ctx context.Context, configID uuid.UUID, prefix string) ([]uuid.UUID, error) { - var children []uuid.UUID - if err := ctx.DB().Raw(`SELECT id FROM get_children_id_by_location(?, ?)`, configID, prefix).Scan(&children).Error; err != nil { +func FindConfigChildrenIDsByLocation(ctx context.Context, configID uuid.UUID, prefix string) (results []uuid.UUID, err error) { + timer := NewQueryLogger(ctx).Start("ConfigChildrenIDs").Arg("configID", configID).Arg("prefix", prefix) + defer timer.End(&err) + + if err = ctx.DB().Raw(`SELECT id FROM get_children_id_by_location(?, ?)`, configID, prefix).Scan(&results).Error; err != nil { return nil, err } - - return children, nil + timer.Results(results) + return results, nil } -func FindConfigParentIDsByLocation(ctx context.Context, configID uuid.UUID, prefix string) ([]uuid.UUID, error) { - var parents []uuid.UUID - if err := ctx.DB().Raw(`SELECT id FROM get_parent_ids_by_location(?, ?)`, configID, prefix).Scan(&parents).Error; err != nil { +func FindConfigParentIDsByLocation(ctx context.Context, configID uuid.UUID, prefix string) (results []uuid.UUID, err error) { + timer := NewQueryLogger(ctx).Start("ConfigParentIDs").Arg("configID", configID).Arg("prefix", prefix) + defer timer.End(&err) + + if err = ctx.DB().Raw(`SELECT id FROM get_parent_ids_by_location(?, ?)`, configID, prefix).Scan(&results).Error; err != nil { return nil, err } - - return parents, nil + timer.Results(results) + return results, nil } type ConfigMinimal struct { @@ -498,20 +505,24 @@ type ConfigMinimal struct { Type string `json:"type"` } -func FindConfigChildrenByLocation(ctx context.Context, configID uuid.UUID, prefix string, includeDeleted bool) ([]ConfigMinimal, error) { - var children []ConfigMinimal - if err := ctx.DB().Raw(`SELECT id, name, type FROM get_children_by_location(?, ?, ?)`, configID, prefix, includeDeleted).Scan(&children).Error; err != nil { +func FindConfigChildrenByLocation(ctx context.Context, configID uuid.UUID, prefix string, includeDeleted bool) (results []ConfigMinimal, err error) { + timer := NewQueryLogger(ctx).Start("ConfigChildren").Arg("configID", configID).Arg("prefix", prefix) + defer timer.End(&err) + + if err = ctx.DB().Raw(`SELECT id, name, type FROM get_children_by_location(?, ?, ?)`, configID, prefix, includeDeleted).Scan(&results).Error; err != nil { return nil, err } - - return children, nil + timer.Results(results) + return results, nil } -func FindConfigParentsByLocation(ctx context.Context, configID uuid.UUID, prefix string, includeDeleted bool) ([]ConfigMinimal, error) { - var parents []ConfigMinimal - if err := ctx.DB().Raw(`SELECT id, name, type FROM get_parents_by_location(?, ?, ?)`, configID, prefix, includeDeleted).Scan(&parents).Error; err != nil { +func FindConfigParentsByLocation(ctx context.Context, configID uuid.UUID, prefix string, includeDeleted bool) (results []ConfigMinimal, err error) { + timer := NewQueryLogger(ctx).Start("ConfigParents").Arg("configID", configID).Arg("prefix", prefix) + defer timer.End(&err) + + if err = ctx.DB().Raw(`SELECT id, name, type FROM get_parents_by_location(?, ?, ?)`, configID, prefix, includeDeleted).Scan(&results).Error; err != nil { return nil, err } - - return parents, nil + timer.Results(results) + return results, nil } diff --git a/query/config_access.go b/query/config_access.go index f49223b70..4b0f94f59 100644 --- a/query/config_access.go +++ b/query/config_access.go @@ -1,18 +1,55 @@ package query import ( + "github.com/flanksource/duty/api" "github.com/flanksource/duty/context" "github.com/flanksource/duty/models" "github.com/google/uuid" ) -func FindConfigAccessByConfigIDs(ctx context.Context, configIDs []uuid.UUID) ([]models.ConfigAccessSummary, error) { - var configAccess []models.ConfigAccessSummary - if err := ctx.DB(). - Where("config_id IN (?)", configIDs). - Find(&configAccess).Error; err != nil { +type CatalogAccessSearchRequest struct { + BaseCatalogSearch `json:",inline"` +} + +type CatalogAccessSearchResponse struct { + Total int64 `json:"total"` + Access []models.ConfigAccessSummary `json:"access"` +} + +func FindCatalogAccess(ctx context.Context, req CatalogAccessSearchRequest) (results *CatalogAccessSearchResponse, err error) { + req.SetDefaults() + if err := req.Validate(); err != nil { + return nil, api.Errorf(api.EINVALID, "bad request: %v", err) + } + + timer := NewQueryLogger(ctx).Start("CatalogAccess") + defer timer.End(&err) + + configIDs, err := req.ResolveConfigIDs(ctx) + if err != nil { return nil, err } - return configAccess, nil + var output CatalogAccessSearchResponse + q := ctx.DB().Table("config_access_summary") + if len(configIDs) > 0 { + q = q.Where("config_id IN ?", configIDs) + } + + if err := q.Find(&output.Access).Error; err != nil { + return nil, err + } + output.Total = int64(len(output.Access)) + timer.Results(output.Access) + return &output, nil +} + +func FindConfigAccessByConfigIDs(ctx context.Context, configIDs []uuid.UUID) ([]models.ConfigAccessSummary, error) { + resp, err := FindCatalogAccess(ctx, CatalogAccessSearchRequest{ + BaseCatalogSearch: BaseCatalogSearch{configIDs: configIDs}, + }) + if err != nil { + return nil, err + } + return resp.Access, nil } diff --git a/query/config_changes.go b/query/config_changes.go index cecd4f070..94e2eeb36 100644 --- a/query/config_changes.go +++ b/query/config_changes.go @@ -8,7 +8,6 @@ import ( "github.com/flanksource/duty/api" "github.com/flanksource/duty/context" "github.com/flanksource/duty/models" - "github.com/flanksource/duty/pkg/kube/labels" "github.com/flanksource/duty/types" "github.com/google/uuid" "github.com/samber/lo" @@ -30,165 +29,60 @@ var allRecursiveOptions = []ChangeRelationDirection{CatalogChangeRecursiveUpstre var allowedConfigChangesSortColumns = []string{"name", "change_type", "summary", "source", "created_at", "count"} type CatalogChangesSearchRequest struct { - CatalogID string `query:"id" json:"id"` - ConfigType string `query:"config_type" json:"config_type"` - ChangeType string `query:"type" json:"type"` - Severity string `query:"severity" json:"severity"` - IncludeDeletedConfigs bool `query:"include_deleted_configs" json:"include_deleted_configs"` - Depth int `query:"depth" json:"depth"` - CreatedByRaw string `query:"created_by" json:"created_by"` - Summary string `query:"summary" json:"summary"` - Source string `query:"source" json:"source"` - Tags string `query:"tags" json:"tags"` - - // To Fetch from a particular agent, provide the agent id. - // Use `local` keyword to filter by the local agent. - AgentID string `query:"agent_id" json:"agent_id"` + BaseCatalogSearch `json:",inline"` + + ChangeType string `query:"type" json:"type"` + Severity string `query:"severity" json:"severity"` + CreatedByRaw string `query:"created_by" json:"created_by"` + Summary string `query:"summary" json:"summary"` + Source string `query:"source" json:"source"` createdBy *uuid.UUID externalCreatedBy string - // From date in datemath format - From string `query:"from" json:"from"` - // To date in datemath format - To string `query:"to" json:"to"` - // FromInsertedAt in datemath format FromInsertedAt string `query:"from_inserted_at" json:"from_inserted_at"` // ToInsertedAt in datemath format ToInsertedAt string `query:"to_inserted_at" json:"to_inserted_at"` - PageSize int `query:"page_size" json:"page_size"` - Page int `query:"page" json:"page"` - SortBy string `query:"sort_by" json:"sort_by"` - sortOrder string - - // upstream | downstream | both - Recursive ChangeRelationDirection `query:"recursive" json:"recursive"` - - // FIXME: Soft toggle does not work with Recursive=both - // In that case, soft relations are always returned - // It also returns ALL soft relations throughout the tree - // not just soft related to the main config item - Soft bool `query:"soft" json:"soft"` - - fromParsed time.Time - toParsed time.Time fromInsertedAtParsed time.Time toInsertedAtParsed time.Time } func (t CatalogChangesSearchRequest) String() string { - s := "" - if t.AgentID != "" { - s += fmt.Sprintf("agent: %s", t.AgentID) - } - if t.CatalogID != "" { - s += fmt.Sprintf("id: %s ", t.CatalogID) - } - if t.ConfigType != "" { - s += fmt.Sprintf("config_type: %s ", t.ConfigType) - } + s := t.BaseCatalogSearch.String() if t.ChangeType != "" { - s += fmt.Sprintf("type: %s ", t.ChangeType) + s += fmt.Sprintf(" type=%s", t.ChangeType) } if t.Severity != "" { - s += fmt.Sprintf("severity: %s ", t.Severity) + s += fmt.Sprintf(" severity=%s", t.Severity) } if t.Source != "" { - s += fmt.Sprintf("source: %s ", t.Source) - } - if t.IncludeDeletedConfigs { - s += fmt.Sprintf("include_deleted_configs: %t ", t.IncludeDeletedConfigs) - } - if t.Depth != 0 { - s += fmt.Sprintf("depth: %d ", t.Depth) + s += fmt.Sprintf(" source=%s", t.Source) } if t.CreatedByRaw != "" { - s += fmt.Sprintf("created_by: %s ", t.CreatedByRaw) + s += fmt.Sprintf(" created_by=%s", t.CreatedByRaw) } if t.Summary != "" { - s += fmt.Sprintf("summary: %s ", t.Summary) - } - if t.Tags != "" { - s += fmt.Sprintf("tags: %s ", t.Tags) - } - if t.From != "" { - s += fmt.Sprintf("from: %s ", t.From) - } - if t.To != "" { - s += fmt.Sprintf("to: %s ", t.To) - } - if t.FromInsertedAt != "" { - s += fmt.Sprintf("from_inserted_at: %s ", t.FromInsertedAt) - } - if t.ToInsertedAt != "" { - s += fmt.Sprintf("to_inserted_at: %s ", t.ToInsertedAt) - } - if t.PageSize != 0 { - s += fmt.Sprintf("page_size: %d ", t.PageSize) - } - if t.Page != 0 { - s += fmt.Sprintf("page: %d ", t.Page) - } - if t.SortBy != "" { - s += fmt.Sprintf("sort_by: %s %s ", t.SortBy, t.sortOrder) - } - if t.Recursive != "" { - s += fmt.Sprintf("recursive: %s ", t.Recursive) + s += fmt.Sprintf(" summary=%s", t.Summary) } return s } func (t *CatalogChangesSearchRequest) SetDefaults() { - if t.PageSize <= 0 { - t.PageSize = 50 - } - - if t.Page <= 0 { - t.Page = 1 - } - if t.From == "" && t.To == "" { t.From = "now-2d" } - - if t.Recursive == "" { - t.Recursive = CatalogChangeRecursiveDownstream - } - - if t.Depth <= 0 { - t.Depth = 5 - } - - if t.AgentID == "local" { - t.AgentID = uuid.Nil.String() - } + t.BaseCatalogSearch.SetDefaults() } func (t *CatalogChangesSearchRequest) Validate() error { - if !lo.Contains(allRecursiveOptions, t.Recursive) { - return fmt.Errorf("'recursive' must be one of %v", allRecursiveOptions) + if err := t.BaseCatalogSearch.Validate(); err != nil { + return err } - if t.From != "" { - if expr, err := datemath.Parse(t.From); err != nil { - return fmt.Errorf("invalid 'from' param: %w", err) - } else { - t.fromParsed = expr.Time() - } - } - - if t.To != "" { - if expr, err := datemath.Parse(t.To); err != nil { - return fmt.Errorf("invalid 'to' param: %w", err) - } else { - t.toParsed = expr.Time() - } - } - - if !t.fromParsed.IsZero() && !t.toParsed.IsZero() && !t.fromParsed.Before(t.toParsed) { - return fmt.Errorf("'from' must be before 'to'") + if !lo.Contains(allRecursiveOptions, t.Recursive) { + return fmt.Errorf("'recursive' must be one of %v", allRecursiveOptions) } if t.FromInsertedAt != "" { @@ -230,12 +124,6 @@ func (t *CatalogChangesSearchRequest) Validate() error { } } - if t.AgentID != "" { - if _, err := uuid.Parse(t.AgentID); err != nil { - return fmt.Errorf("agent_id(%s) must either be a valid uuid or `local`", t.AgentID) - } - } - return nil } @@ -299,84 +187,61 @@ func formSeverityQuery(severity string) string { return strings.Join(applicable, ",") } -func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (*CatalogChangesSearchResponse, error) { +func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (result *CatalogChangesSearchResponse, err error) { req.SetDefaults() if err := req.Validate(); err != nil { return nil, api.Errorf(api.EINVALID, "bad request: %v", err) } - ctx.Tracef("query changes: %s", req) - var clauses []clause.Expression + timer := NewQueryLogger(ctx).Start("CatalogChanges").Arg("query", req.String()) + defer timer.End(&err) - query := ctx.DB() - - if req.AgentID != "" { - clause, err := parseAndBuildFilteringQuery(req.AgentID, "agent_id", false) - if err != nil { - return nil, err - } - clauses = append(clauses, clause...) + configIDs, err := req.ResolveConfigIDs(ctx) + if err != nil { + return nil, err } - if req.ConfigType != "" { - clause, err := parseAndBuildFilteringQuery(req.ConfigType, "type", false) - if err != nil { - return nil, err - } - clauses = append(clauses, clause...) + baseClauses, tagsFn := req.ApplyClauses() + var clauses []clause.Expression + clauses = append(clauses, baseClauses...) + + dbQuery := ctx.DB() + if tagsFn != nil { + dbQuery = tagsFn(dbQuery) } if req.ChangeType != "" { - clause, err := parseAndBuildFilteringQuery(req.ChangeType, "change_type", false) - if err != nil { - return nil, err + if c, parseErr := parseAndBuildFilteringQuery(req.ChangeType, "change_type", false); parseErr == nil { + clauses = append(clauses, c...) + } else { + return nil, parseErr } - clauses = append(clauses, clause...) } if req.Severity != "" { - clause, err := parseAndBuildFilteringQuery(formSeverityQuery(req.Severity), "severity", false) - if err != nil { - return nil, api.Errorf(api.EINVALID, "failed to parse severity: %v", err) + if c, parseErr := parseAndBuildFilteringQuery(formSeverityQuery(req.Severity), "severity", false); parseErr == nil { + clauses = append(clauses, c...) + } else { + return nil, api.Errorf(api.EINVALID, "failed to parse severity: %v", parseErr) } - clauses = append(clauses, clause...) } if req.Summary != "" { - clause, err := parseAndBuildFilteringQuery(req.Summary, "summary", true) - if err != nil { - return nil, api.Errorf(api.EINVALID, "failed to parse summary: %v", err) + if c, parseErr := parseAndBuildFilteringQuery(req.Summary, "summary", true); parseErr == nil { + clauses = append(clauses, c...) + } else { + return nil, api.Errorf(api.EINVALID, "failed to parse summary: %v", parseErr) } - clauses = append(clauses, clause...) } if req.Source != "" { - clause, err := parseAndBuildFilteringQuery(req.Source, "source", true) - if err != nil { - return nil, api.Errorf(api.EINVALID, "failed to parse source: %v", err) - } - clauses = append(clauses, clause...) - } - - if req.Tags != "" { - parsedLabelSelector, err := labels.Parse(req.Tags) - if err != nil { - return nil, api.Errorf(api.EINVALID, "failed to parse label selector: %v", err) - } - requirements, _ := parsedLabelSelector.Requirements() - for _, r := range requirements { - query = jsonColumnRequirementsToSQLClause(query, "tags", r) + if c, parseErr := parseAndBuildFilteringQuery(req.Source, "source", true); parseErr == nil { + clauses = append(clauses, c...) + } else { + return nil, api.Errorf(api.EINVALID, "failed to parse source: %v", parseErr) } } - if !req.fromParsed.IsZero() { - clauses = append(clauses, clause.Gte{Column: clause.Column{Name: "created_at"}, Value: req.fromParsed}) - } - - if !req.toParsed.IsZero() { - clauses = append(clauses, clause.Lte{Column: clause.Column{Name: "created_at"}, Value: req.toParsed}) - } - if !req.fromInsertedAtParsed.IsZero() { clauses = append(clauses, clause.Gte{Column: clause.Column{Name: "inserted_at"}, Value: req.fromInsertedAtParsed}) } @@ -390,26 +255,26 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (* } if req.externalCreatedBy != "" { - clause, err := parseAndBuildFilteringQuery(req.externalCreatedBy, "external_created_by", true) - if err != nil { - return nil, api.Errorf(api.EINVALID, "failed to parse external createdby: %v", err) + if c, parseErr := parseAndBuildFilteringQuery(req.externalCreatedBy, "external_created_by", true); parseErr == nil { + clauses = append(clauses, c...) + } else { + return nil, api.Errorf(api.EINVALID, "failed to parse external createdby: %v", parseErr) } - clauses = append(clauses, clause...) - } - - if !req.IncludeDeletedConfigs { - clauses = append(clauses, clause.Eq{Column: clause.Column{Name: "deleted_at"}, Value: nil}) } - table := query.Table("catalog_changes") - if err := uuid.Validate(req.CatalogID); err == nil { - table = query.Table("related_changes_recursive(?,?,?,?,?)", req.CatalogID, req.Recursive, req.IncludeDeletedConfigs, req.Depth, req.Soft) - } else { - clause, err := parseAndBuildFilteringQuery(req.CatalogID, "config_id", false) - if err != nil { - return nil, err + // Determine table: single UUID uses related_changes_recursive, multi-ID or query uses IN clause + table := dbQuery.Table("catalog_changes") + if len(configIDs) == 1 { + table = dbQuery.Table("related_changes_recursive(?,?,?,?,?)", configIDs[0], req.Recursive, req.IncludeDeletedConfigs, req.Depth, req.Soft) + } else if len(configIDs) > 1 { + table = table.Where("config_id IN ?", configIDs) + } else if req.CatalogID != "" { + // Fallback: treat as filtering expression on config_id + if c, parseErr := parseAndBuildFilteringQuery(req.CatalogID, "config_id", false); parseErr == nil { + clauses = append(clauses, c...) + } else { + return nil, parseErr } - clauses = append(clauses, clause...) } var output CatalogChangesSearchResponse @@ -418,6 +283,7 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (* } if output.Total == 0 { + timer.Results(output.Changes) return &output, nil } @@ -440,6 +306,7 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (* } output.Summarize() + timer.Results(output.Changes) return &output, nil } diff --git a/query/config_insights.go b/query/config_insights.go new file mode 100644 index 000000000..39772e36b --- /dev/null +++ b/query/config_insights.go @@ -0,0 +1,110 @@ +package query + +import ( + "github.com/flanksource/duty/api" + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "gorm.io/gorm/clause" +) + +type CatalogInsightsSearchRequest struct { + BaseCatalogSearch `json:",inline"` + Status string `query:"status" json:"status"` + Severity string `query:"severity" json:"severity"` + Analyzer string `query:"analyzer" json:"analyzer"` + AnalysisType string `query:"analysis_type" json:"analysis_type"` +} + +func (r *CatalogInsightsSearchRequest) SetDefaults() { + r.BaseCatalogSearch.SetDefaults() + if r.Status == "" { + r.Status = "open" + } +} + +type CatalogInsightsSearchResponse struct { + Total int64 `json:"total"` + Insights []models.ConfigAnalysis `json:"insights"` +} + +func FindCatalogInsights(ctx context.Context, req CatalogInsightsSearchRequest) (results *CatalogInsightsSearchResponse, err error) { + req.SetDefaults() + if err := req.Validate(); err != nil { + return nil, api.Errorf(api.EINVALID, "bad request: %v", err) + } + + timer := NewQueryLogger(ctx).Start("CatalogInsights") + defer timer.End(&err) + + configIDs, err := req.ResolveConfigIDs(ctx) + if err != nil { + return nil, err + } + + var clauses []clause.Expression + baseClauses, tagsFn := req.ApplyClauses() + clauses = append(clauses, baseClauses...) + + q := ctx.DB().Table("config_analysis") + + if len(configIDs) > 0 { + clauses = append(clauses, clause.Eq{Column: clause.Column{Name: "config_id"}, Value: nil}) + q = q.Where("config_id IN ?", configIDs) + } + + if req.Status != "" { + if c, parseErr := parseAndBuildFilteringQuery(req.Status, "status", false); parseErr == nil { + clauses = append(clauses, c...) + } + } + if req.Severity != "" { + if c, parseErr := parseAndBuildFilteringQuery(formSeverityQuery(req.Severity), "severity", false); parseErr == nil { + clauses = append(clauses, c...) + } + } + if req.Analyzer != "" { + if c, parseErr := parseAndBuildFilteringQuery(req.Analyzer, "analyzer", true); parseErr == nil { + clauses = append(clauses, c...) + } + } + if req.AnalysisType != "" { + if c, parseErr := parseAndBuildFilteringQuery(req.AnalysisType, "analysis_type", false); parseErr == nil { + clauses = append(clauses, c...) + } + } + + if tagsFn != nil { + q = tagsFn(q) + } + + var output CatalogInsightsSearchResponse + // Remove the dummy deleted_at clause for config_analysis (it doesn't have deleted_at) + filteredClauses := make([]clause.Expression, 0, len(clauses)) + for _, c := range clauses { + if eq, ok := c.(clause.Eq); ok && eq.Column.(clause.Column).Name == "deleted_at" { + continue + } + if eq, ok := c.(clause.Eq); ok && eq.Column.(clause.Column).Name == "config_id" && eq.Value == nil { + continue + } + filteredClauses = append(filteredClauses, c) + } + + if err := q.Clauses(filteredClauses...).Count(&output.Total).Error; err != nil { + return nil, err + } + if output.Total == 0 { + timer.Results(output.Insights) + return &output, nil + } + + filteredClauses = append(filteredClauses, + clause.Limit{Limit: &req.PageSize, Offset: (req.Page - 1) * req.PageSize}, + ) + + if err := q.Clauses(filteredClauses...).Find(&output.Insights).Error; err != nil { + return nil, err + } + timer.Results(output.Insights) + return &output, nil +} diff --git a/query/config_relations.go b/query/config_relations.go index 3aee87c65..654855d55 100644 --- a/query/config_relations.go +++ b/query/config_relations.go @@ -11,6 +11,10 @@ import ( "github.com/samber/lo" ) +func (r RelatedConfig) QueryLogSummary() string { + return r.Type + "/" + r.Name +} + type RelatedConfig struct { Relation string `json:"relation"` RelatedIDs pq.StringArray `json:"related_ids" gorm:"type:[]text"` @@ -72,8 +76,12 @@ const ( Soft RelationType = "soft" ) -func GetRelatedConfigs(ctx context.Context, query RelationQuery) ([]RelatedConfig, error) { - var relatedConfigs []RelatedConfig +func GetRelatedConfigs(ctx context.Context, query RelationQuery) (results []RelatedConfig, err error) { + timer := NewQueryLogger(ctx).Start("RelatedConfigs"). + Arg("id", query.ID).Arg("direction", query.Relation). + Arg("incoming", query.Incoming).Arg("outgoing", query.Outgoing) + defer timer.End(&err) + if query.MaxDepth == nil { query.MaxDepth = lo.ToPtr(5) } @@ -86,15 +94,15 @@ func GetRelatedConfigs(ctx context.Context, query RelationQuery) ([]RelatedConfi // FIXME: Self config is returned here for creating graph in UI. We need to update UI to // add the node itself. Issue: github.com/flanksource/duty/issues/1722 - err := ctx.DB().Raw("SELECT * FROM related_configs_recursive(?, ?, ?, ?, ?, ?)", + err = ctx.DB().Raw("SELECT * FROM related_configs_recursive(?, ?, ?, ?, ?, ?)", query.ID, query.Relation, query.IncludeDeleted, *query.MaxDepth, query.Incoming, - query.Outgoing).Find(&relatedConfigs).Error - - relatedConfigs = lo.Filter(relatedConfigs, func(c RelatedConfig, _ int) bool { return c.ID != query.ID }) + query.Outgoing).Find(&results).Error - return relatedConfigs, err + results = lo.Filter(results, func(c RelatedConfig, _ int) bool { return c.ID != query.ID }) + timer.Results(results) + return results, err } diff --git a/query/query_logger.go b/query/query_logger.go new file mode 100644 index 000000000..a576eae3d --- /dev/null +++ b/query/query_logger.go @@ -0,0 +1,119 @@ +package query + +import ( + "fmt" + "reflect" + "strings" + "time" + + clickyapi "github.com/flanksource/clicky/api" + "github.com/flanksource/commons/logger" + "github.com/flanksource/duty/context" +) + +type QueryLogger struct { + logger logger.Verbose +} + +type QueryTimer struct { + logger logger.Verbose + label clickyapi.Text + start time.Time + results any + ended bool +} + +func NewQueryLogger(ctx context.Context) QueryLogger { + l := ctx.Logger.V(3) + if ctx.Properties().On(false, "query.log") { + l = ctx.Logger.V(0) + } + return QueryLogger{logger: l} +} + +func (q QueryLogger) Start(entity string) *QueryTimer { + return &QueryTimer{ + logger: q.logger, + label: clickyapi.Text{Content: "[" + entity + "]", Style: "text-blue-600 font-bold"}, + start: time.Now(), + } +} + +func (t *QueryTimer) Arg(key string, value any) *QueryTimer { + t.label = t.label.AddText(fmt.Sprintf(" %s=", key), "text-gray-500"). + AddText(fmt.Sprintf("%v", value)) + return t +} + +func (t *QueryTimer) Results(results any) *QueryTimer { + t.results = results + return t +} + +func (t *QueryTimer) End(err *error) { + if t.ended { + return + } + t.ended = true + if !t.logger.Enabled() { + return + } + + elapsed := time.Since(t.start) + label := t.label.AddText(" => ", "text-gray-400") + + if err != nil && *err != nil { + label = label.AddText(fmt.Sprintf("error: %v", *err), "text-red-600") + } else if t.results != nil { + count := sliceLen(t.results) + countStyle := "text-green-600" + if count == 0 { + countStyle = "text-red-600" + } + label = label.AddText(fmt.Sprintf("%d", count), countStyle) + label = label.AddText(summaryText(t.results, count), "text-gray-400") + } else { + label = label.AddText("timed out", "text-yellow-600") + } + + label = label.AddText(fmt.Sprintf(" in %dms", elapsed.Milliseconds()), "text-gray-400") + t.logger.Infof("%s", label.ANSI()) +} + +func sliceLen(v any) int { + rv := reflect.ValueOf(v) + if rv.Kind() == reflect.Slice { + return rv.Len() + } + if rv.Kind() == reflect.Ptr && !rv.IsNil() { + return 1 + } + return 0 +} + +func summaryText(v any, count int) string { + if count == 0 { + return "" + } + + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Slice { + return "" + } + + const maxInline = 2 + shown := count + if shown > maxInline { + shown = maxInline + } + + var parts []string + for i := 0; i < shown; i++ { + parts = append(parts, itemLogSummary(rv.Index(i).Interface())) + } + summary := strings.Join(parts, ", ") + if count > maxInline { + summary += fmt.Sprintf(", ...%d more", count-maxInline) + } + return " [" + summary + "]" +} diff --git a/tests/config_changes_test.go b/tests/config_changes_test.go index c776165e1..4b98b2ca7 100644 --- a/tests/config_changes_test.go +++ b/tests/config_changes_test.go @@ -113,9 +113,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { var findChanges = func(id uuid.UUID, filter query.ChangeRelationDirection, deleted bool) (*query.CatalogChangesSearchResponse, error) { return query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: id.String(), - IncludeDeletedConfigs: deleted, - Recursive: filter, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: id.String(), + IncludeDeletedConfigs: deleted, + Recursive: filter, + }, }) } @@ -144,9 +146,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should return changes of a root node along with soft", func() { relatedChanges, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: "downstream", - Soft: true, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: "downstream", + Soft: true, + }, }) Expect(err).To(BeNil()) @@ -168,9 +172,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should return changes of a leaf node along with soft", func() { relatedChanges, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: X.ID.String(), - Recursive: "downstream", - Soft: true, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: X.ID.String(), + Recursive: "downstream", + Soft: true, + }, }) Expect(err).To(BeNil()) @@ -194,9 +200,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should return changes of a leaf node along with soft", func() { relatedChanges, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: "upstream", - Soft: true, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: "upstream", + Soft: true, + }, }) Expect(err).To(BeNil()) @@ -218,9 +226,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should return changes of a leaf node along with soft", func() { relatedChanges, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: X.ID.String(), - Recursive: "upstream", - Soft: true, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: X.ID.String(), + Recursive: "upstream", + Soft: true, + }, }) Expect(err).To(BeNil()) @@ -235,7 +245,9 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("FindCatalogChanges func", func() { ginkgo.It("Without catalog id", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - ConfigType: "Kubernetes::Pod,Kubernetes::ReplicaSet", + BaseCatalogSearch: query.BaseCatalogSearch{ + ConfigType: "Kubernetes::Pod,Kubernetes::ReplicaSet", + }, ChangeType: "!NotificationSent", }) Expect(err).To(BeNil()) @@ -250,9 +262,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("Config type filter", func() { ginkgo.It("IN", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - ConfigType: "Kubernetes::Pod,Kubernetes::ReplicaSet", + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + ConfigType: "Kubernetes::Pod,Kubernetes::ReplicaSet", + }, }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(3))) @@ -263,9 +277,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("NOT IN", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - ConfigType: "!Kubernetes::ReplicaSet", + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + ConfigType: "!Kubernetes::ReplicaSet", + }, }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(5))) @@ -279,8 +295,10 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("Change type filter", func() { ginkgo.It("IN", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: X.ID.String(), - Recursive: query.CatalogChangeRecursiveAll, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: X.ID.String(), + Recursive: query.CatalogChangeRecursiveAll, + }, ChangeType: "diff", }) Expect(err).To(BeNil()) @@ -291,8 +309,10 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("NOT IN", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + }, ChangeType: "!diff,!Pulled", }) Expect(err).To(BeNil()) @@ -305,9 +325,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("Severity filter", func() { ginkgo.It("NOT", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - Severity: "!info", + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + }, + Severity: "!info", }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(3))) @@ -318,9 +340,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should return the given severity and higher", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - Severity: string(models.SeverityMedium), + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + }, + Severity: string(models.SeverityMedium), }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(2))) @@ -333,10 +357,12 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("Pagination", func() { ginkgo.It("Page size", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - SortBy: "summary", - PageSize: 2, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + SortBy: "summary", + PageSize: 2, + }, }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(6))) @@ -347,11 +373,13 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("Page number", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - SortBy: "summary", - PageSize: 2, - Page: 2, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + SortBy: "summary", + PageSize: 2, + Page: 2, + }, }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(6))) @@ -364,8 +392,10 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("recursive mode", func() { ginkgo.It("upstream", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: W.ID.String(), - Recursive: query.CatalogChangeRecursiveUpstream, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: W.ID.String(), + Recursive: query.CatalogChangeRecursiveUpstream, + }, }) Expect(err).To(BeNil()) Expect(len(response.Changes)).To(Equal(2)) @@ -376,8 +406,10 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It(string(query.CatalogChangeRecursiveDownstream), func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: V.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: V.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + }, }) Expect(err).To(BeNil()) Expect(len(response.Changes)).To(Equal(4)) @@ -388,8 +420,10 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It(string(query.CatalogChangeRecursiveAll), func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: V.ID.String(), - Recursive: query.CatalogChangeRecursiveAll, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: V.ID.String(), + Recursive: query.CatalogChangeRecursiveAll, + }, }) Expect(err).To(BeNil()) Expect(len(response.Changes)).To(Equal(8)) @@ -402,10 +436,12 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should handle datemath", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - From: "now-65m", - To: "now-1s", + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + From: "now-65m", + To: "now-1s", + }, }) Expect(err).To(BeNil()) Expect(response.Total).To(BeNumerically(">=", 1)) @@ -413,8 +449,10 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should filter by inserted_at", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + }, FromInsertedAt: "now-1h", }) Expect(err).To(BeNil()) @@ -428,9 +466,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("Sorting", func() { ginkgo.It("Descending", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - SortBy: "-name", + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + SortBy: "-name", + }, }) Expect(err).To(BeNil()) Expect(len(response.Changes)).To(Equal(6)) @@ -441,9 +481,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("Ascending", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - SortBy: "name", + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + SortBy: "name", + }, }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(6))) diff --git a/tests/e2e-blobs/test.properties b/tests/e2e-blobs/test.properties new file mode 100644 index 000000000..4cff42f1d --- /dev/null +++ b/tests/e2e-blobs/test.properties @@ -0,0 +1,2 @@ +log.level.migrate=info +log.level.blobs=trace3 diff --git a/tests/setup/template.go b/tests/setup/template.go new file mode 100644 index 000000000..02a3770ff --- /dev/null +++ b/tests/setup/template.go @@ -0,0 +1,241 @@ +package setup + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + + embeddedPG "github.com/fergusstrange/embedded-postgres" + "github.com/flanksource/commons/logger" + "github.com/flanksource/commons/properties" + "github.com/flanksource/duty" + "github.com/flanksource/duty/context" + dutyKubernetes "github.com/flanksource/duty/kubernetes" + "github.com/flanksource/duty/shutdown" + "github.com/flanksource/duty/telemetry" + "github.com/flanksource/duty/tests/fixtures/dummy" + "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +type SetupOpts struct { + DummyData bool +} + +type templateInfo struct { + AdminURL string `json:"admin_url"` + TemplateDB string `json:"template_db"` + Port int `json:"port"` +} + +func (t templateInfo) Marshal() []byte { + data, err := json.Marshal(t) + if err != nil { + panic(fmt.Sprintf("failed to marshal templateInfo: %v", err)) + } + return data +} + +func unmarshalTemplateInfo(data []byte) templateInfo { + var info templateInfo + if err := json.Unmarshal(data, &info); err != nil { + panic(fmt.Sprintf("failed to unmarshal templateInfo: %v", err)) + } + return info +} + +var ( + adminURL string + nodeDBName string +) + +func SetupTemplate(opts SetupOpts) []byte { + if err := properties.LoadFile(findFileInPath("test.properties", 2)); err != nil { + logger.Errorf("Failed to load test properties: %v", err) + } + + defer telemetry.InitTracer() + + var port int + if val, ok := os.LookupEnv(TEST_DB_PORT); ok { + parsed, err := strconv.ParseInt(val, 10, 32) + if err != nil { + panic(fmt.Sprintf("failed to parse TEST_DB_PORT: %v", err)) + } + port = int(parsed) + } else { + port = duty.FreePort() + } + + templateDB := "duty_test_template" + + url := os.Getenv(DUTY_DB_URL) + if url != "" && !recreateDatabase { + // DUTY_DB_CREATE=false: use direct connection, no template + PgUrl = url + return templateInfo{AdminURL: url, TemplateDB: "", Port: port}.Marshal() + } + + adminConn, err := ensurePostgres(port) + if err != nil { + panic(fmt.Sprintf("failed to start postgres: %v", err)) + } + adminURL = adminConn + + // Always recreate — dummy data uses uuid.New() so a cached template has stale UUIDs + _ = execPostgres(adminConn, fmt.Sprintf("ALTER DATABASE %s WITH is_template = false", templateDB)) + _ = execPostgres(adminConn, fmt.Sprintf( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '%s' AND pid <> pg_backend_pid()", templateDB)) + _ = execPostgres(adminConn, fmt.Sprintf("DROP DATABASE IF EXISTS %s (FORCE)", templateDB)) + + if err := execPostgres(adminConn, fmt.Sprintf("CREATE DATABASE %s", templateDB)); err != nil { + panic(fmt.Sprintf("failed to create template db: %v", err)) + } + + templateURL := strings.Replace(adminConn, "/postgres", "/"+templateDB, 1) + if !strings.Contains(adminConn, "/postgres") { + templateURL = fmt.Sprintf("postgres://postgres:postgres@localhost:%d/%s?sslmode=disable", port, templateDB) + } + + dbOptions := []duty.StartOption{duty.DisablePostgrest, duty.RunMigrations, duty.WithUrl(templateURL)} + if !disableRLS { + dbOptions = append(dbOptions, duty.EnableRLS) + } + + ctx, stop, err := duty.Start(templateDB, dbOptions...) + if err != nil { + panic(fmt.Sprintf("failed to start duty for template: %v", err)) + } + + if err := ctx.DB().Exec("SET TIME ZONE 'UTC'").Error; err != nil { + panic(fmt.Sprintf("failed to set timezone: %v", err)) + } + + if opts.DummyData { + dummyData = dummy.GetStaticDummyData(ctx.DB()) + if err := dummyData.Delete(ctx.DB()); err != nil { + logger.Errorf(err.Error()) + } + if err := dummyData.Populate(ctx); err != nil { + panic(fmt.Sprintf("failed to populate dummy data: %v", err)) + } + logger.Infof("Created dummy data in template (%d checks)", len(dummyData.Checks)) + } + + // Close all connections so the DB can be used as a template + stop() + _ = execPostgres(adminConn, fmt.Sprintf( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '%s' AND pid <> pg_backend_pid()", templateDB)) + + if err := execPostgres(adminConn, fmt.Sprintf("ALTER DATABASE %s WITH is_template = true", templateDB)); err != nil { + panic(fmt.Sprintf("failed to mark template db: %v", err)) + } + + return templateInfo{AdminURL: adminConn, TemplateDB: templateDB, Port: port}.Marshal() +} + +func SetupNode(data []byte, opts SetupOpts) context.Context { + info := unmarshalTemplateInfo(data) + + if info.TemplateDB == "" { + // Direct connection mode (DUTY_DB_CREATE=false) + PgUrl = info.AdminURL + ctx, _, err := duty.Start("direct", duty.ClientOnly, duty.WithUrl(PgUrl)) + if err != nil { + panic(fmt.Sprintf("failed to connect to db: %v", err)) + } + return setupNodeContext(ctx, "direct") + } + + adminURL = info.AdminURL + nodeDBName = fmt.Sprintf("duty_test_node%d", ginkgo.GinkgoParallelProcess()) + + // Drop and clone from template + _ = execPostgres(adminURL, fmt.Sprintf("DROP DATABASE IF EXISTS %s (FORCE)", nodeDBName)) + + // Terminate any lingering connections to the template before cloning + _ = execPostgres(adminURL, fmt.Sprintf( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '%s' AND pid <> pg_backend_pid()", info.TemplateDB)) + + // Unmark template temporarily for cloning (some pg versions need this) + _ = execPostgres(adminURL, fmt.Sprintf("ALTER DATABASE %s WITH is_template = false", info.TemplateDB)) + if err := execPostgres(adminURL, fmt.Sprintf("CREATE DATABASE %s TEMPLATE %s", nodeDBName, info.TemplateDB)); err != nil { + panic(fmt.Sprintf("failed to clone template: %v", err)) + } + _ = execPostgres(adminURL, fmt.Sprintf("ALTER DATABASE %s WITH is_template = true", info.TemplateDB)) + + // Build node connection URL + if strings.Contains(adminURL, "/postgres") { + PgUrl = strings.Replace(adminURL, "/postgres", "/"+nodeDBName, 1) + } else { + PgUrl = fmt.Sprintf("postgres://postgres:postgres@localhost:%d/%s?sslmode=disable", info.Port, nodeDBName) + } + + // Skip migrations — the clone is byte-for-byte identical to the template + ctx, _, err := duty.Start(nodeDBName, duty.ClientOnly, duty.WithUrl(PgUrl)) + if err != nil { + panic(fmt.Sprintf("failed to connect to node db: %v", err)) + } + + return setupNodeContext(ctx, nodeDBName) +} + +func setupNodeContext(ctx context.Context, dbName string) context.Context { + if err := ctx.DB().Exec("SET TIME ZONE 'UTC'").Error; err != nil { + panic(fmt.Sprintf("failed to set timezone: %v", err)) + } + + ctx = ctx.WithValue("db_name", dbName).WithValue("db_url", PgUrl) + + clientset := fake.NewClientset(&v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cm", Namespace: "default"}, + Data: map[string]string{"foo": "bar"}, + }, &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-secret", Namespace: "default"}, + Data: map[string][]byte{"foo": []byte("secret")}, + }) + + return ctx.WithLocalKubernetes(dutyKubernetes.NewKubeClient(logger.GetLogger("k8s"), clientset, nil)) +} + +func SynchronizedAfterSuiteAllNodes() { + if nodeDBName != "" && adminURL != "" { + if err := execPostgres(adminURL, fmt.Sprintf("DROP DATABASE IF EXISTS %s (FORCE)", nodeDBName)); err != nil { + logger.Errorf("failed to drop node db: %v", err) + } + } +} + +func SynchronizedAfterSuiteNode1() { + shutdown.Shutdown() +} + + +func ensurePostgres(port int) (string, error) { + url := os.Getenv(DUTY_DB_URL) + if url != "" { + postgresDBUrl = url + return url, nil + } + + if postgresServer == nil { + config, _ := GetEmbeddedPGConfig("postgres", port) + + if v, ok := os.LookupEnv(DUTY_DB_DATA_DIR); ok { + config = config.DataPath(v) + } + + postgresServer = embeddedPG.NewDatabase(config) + logger.Infof("starting embedded postgres on port %d", port) + if err := postgresServer.Start(); err != nil { + return "", err + } + logger.Infof("Started postgres on port %d", port) + } + + return fmt.Sprintf("postgres://postgres:postgres@localhost:%d/postgres?sslmode=disable", port), nil +} From 8b34baa919bf8e5d35f71c7213d40c17e5b87418 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 6 Apr 2026 14:31:54 +0300 Subject: [PATCH 4/8] feat(api): add configurable postgrest architecture override Allow PGRST_ARCH environment variable to override the default runtime architecture when installing postgrest binary, enabling custom architecture targets. --- api/config.go | 7 +++++++ postgrest/postgrest.go | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/api/config.go b/api/config.go index 1f1e788fe..0ea8d5b28 100644 --- a/api/config.go +++ b/api/config.go @@ -14,6 +14,7 @@ var DefaultConfig = Config{ Postgrest: PostgrestConfig{ Version: "v14.6", DBRole: "postgrest_api", + Arch: runtime.GOARCH, AnonDBRole: "", Port: 3000, AdminPort: 3001, @@ -22,7 +23,9 @@ var DefaultConfig = Config{ } func init() { + DefaultConfig.Postgrest = DefaultConfig.Postgrest.ReadEnv() v := DefaultConfig.Postgrest.Version + if strings.HasPrefix(v, "v14") && v != "v14.1" && v != "v14.0" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" { logger.Warnf("PostgREST v14.2+ does not have a darwin/arm64 binary, defaulting to v14.1 for darwin/amd64") @@ -124,6 +127,7 @@ type PostgrestConfig struct { LogLevel string URL string Version string + Arch string JWTSecret string DBRole string AnonDBRole string @@ -146,6 +150,9 @@ func (p PostgrestConfig) ReadEnv() PostgrestConfig { if v := os.Getenv("PGRST_VERSION"); v != "" { clone.Version = v } + if v := os.Getenv("PGRST_ARCH"); v != "" { + clone.Arch = v + } return clone } diff --git a/postgrest/postgrest.go b/postgrest/postgrest.go index e32f39ae1..221de2025 100644 --- a/postgrest/postgrest.go +++ b/postgrest/postgrest.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "runtime" "strconv" "github.com/flanksource/commons/exec" @@ -17,7 +18,10 @@ func GoOffline() error { } func runBinary(config api.Config, msg string, args ...any) error { - result, err := deps.InstallWithContext(context.Background(), "postgrest", config.Postgrest.Version, deps.WithBinDir(".bin")) + result, err := deps.InstallWithContext(context.Background(), "postgrest", + config.Postgrest.Version, + deps.WithBinDir(".bin"), + deps.WithOS(runtime.GOOS, config.Postgrest.Arch)) if err != nil { return fmt.Errorf("failed to install postgREST: %w", err) } From ad405f0b7315294f6216b917faee1078f5b85846 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 6 Apr 2026 16:34:34 +0300 Subject: [PATCH 5/8] feat(api): add pagination to catalog access search and config tree traversal Implements proper pagination by counting total results before applying limit/offset. Adds new ConfigTree function to build hierarchical config relationships including parents, children, and related configs. --- query/config_access.go | 11 +- query/config_tree.go | 242 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 query/config_tree.go diff --git a/query/config_access.go b/query/config_access.go index 4b0f94f59..c45f58a46 100644 --- a/query/config_access.go +++ b/query/config_access.go @@ -36,10 +36,17 @@ func FindCatalogAccess(ctx context.Context, req CatalogAccessSearchRequest) (res q = q.Where("config_id IN ?", configIDs) } - if err := q.Find(&output.Access).Error; err != nil { + if err := q.Count(&output.Total).Error; err != nil { + return nil, err + } + if output.Total == 0 { + timer.Results(output.Access) + return &output, nil + } + + if err := q.Limit(req.PageSize).Offset((req.Page - 1) * req.PageSize).Find(&output.Access).Error; err != nil { return nil, err } - output.Total = int64(len(output.Access)) timer.Results(output.Access) return &output, nil } diff --git a/query/config_tree.go b/query/config_tree.go new file mode 100644 index 000000000..a67d4d5ee --- /dev/null +++ b/query/config_tree.go @@ -0,0 +1,242 @@ +package query + +import ( + "strings" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "github.com/google/uuid" + "github.com/samber/lo" +) + +type ConfigTreeNode struct { + models.ConfigItem `json:",inline"` + EdgeType string `json:"edgeType,omitempty"` + Relation string `json:"relation,omitempty"` + Children []*ConfigTreeNode `json:"children,omitempty"` +} + +type ConfigTreeOptions struct { + Direction RelationDirection + Incoming RelationType + Outgoing RelationType +} + +func ConfigTree(ctx context.Context, configID uuid.UUID, opts ConfigTreeOptions) (*ConfigTreeNode, error) { + config, err := GetCachedConfig(ctx, configID.String()) + if err != nil { + return nil, err + } + if config == nil { + return nil, nil + } + + parents := resolveParentsFromPath(ctx, config) + + childIDs, err := ExpandConfigChildren(ctx, []uuid.UUID{config.ID}) + if err != nil { + return nil, err + } + childIDs = lo.Filter(childIDs, func(id uuid.UUID, _ int) bool { return id != config.ID }) + + var children []models.ConfigItem + if len(childIDs) > 0 { + children, err = GetConfigsByIDs(ctx, childIDs) + if err != nil { + return nil, err + } + } + + if opts.Direction == "" { + opts.Direction = All + } + relType := Hard + if opts.Incoming != "" { + relType = opts.Incoming + } + outType := Hard + if opts.Outgoing != "" { + outType = opts.Outgoing + } + + related, err := GetRelatedConfigs(ctx, RelationQuery{ + ID: config.ID, + Relation: opts.Direction, + Incoming: relType, + Outgoing: outType, + }) + if err != nil { + return nil, err + } + + return buildConfigTree(config, parents, children, related), nil +} + +func resolveParentsFromPath(ctx context.Context, config *models.ConfigItem) []models.ConfigItem { + if config.Path == "" { + return nil + } + segments := strings.Split(config.Path, ".") + var parentIDs []uuid.UUID + for _, seg := range segments { + id, err := uuid.Parse(seg) + if err != nil || id == config.ID { + continue + } + parentIDs = append(parentIDs, id) + } + if len(parentIDs) == 0 { + return nil + } + items, err := GetConfigsByIDs(ctx, parentIDs) + if err != nil || len(items) == 0 { + return nil + } + byID := make(map[uuid.UUID]models.ConfigItem, len(items)) + for _, ci := range items { + byID[ci.ID] = ci + } + var parents []models.ConfigItem + for _, id := range parentIDs { + if ci, ok := byID[id]; ok { + parents = append(parents, ci) + } + } + return parents +} + +func ExpandConfigChildren(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { + allIDs := make(map[uuid.UUID]struct{}, len(ids)) + for _, id := range ids { + allIDs[id] = struct{}{} + } + for _, id := range ids { + var children []uuid.UUID + if err := ctx.DB().Raw("SELECT child_id FROM lookup_config_children(?, -1)", id.String()). + Scan(&children).Error; err != nil { + return nil, err + } + for _, child := range children { + allIDs[child] = struct{}{} + } + } + return lo.Keys(allIDs), nil +} + +type ptrNode struct { + models.ConfigItem + edgeType string + relation string + children []*ptrNode +} + +func buildConfigTree(config *models.ConfigItem, parents []models.ConfigItem, children []models.ConfigItem, related []RelatedConfig) *ConfigTreeNode { + nodes := make(map[uuid.UUID]*ptrNode) + + for _, p := range parents { + nodes[p.ID] = &ptrNode{ConfigItem: p, edgeType: "parent"} + } + + targetNode := &ptrNode{ConfigItem: *config, edgeType: "target"} + nodes[config.ID] = targetNode + + if len(parents) > 0 { + for i := 0; i < len(parents)-1; i++ { + nodes[parents[i].ID].children = append(nodes[parents[i].ID].children, nodes[parents[i+1].ID]) + } + nodes[parents[len(parents)-1].ID].children = append(nodes[parents[len(parents)-1].ID].children, targetNode) + } + + for _, c := range children { + nodes[c.ID] = &ptrNode{ConfigItem: c, edgeType: "child"} + } + for _, c := range children { + parentID := lo.FromPtr(c.ParentID) + if parent, ok := nodes[parentID]; ok { + parent.children = append(parent.children, nodes[c.ID]) + } else { + targetNode.children = append(targetNode.children, nodes[c.ID]) + } + } + + parentIDs := make(map[uuid.UUID]bool, len(parents)) + for _, p := range parents { + parentIDs[p.ID] = true + } + + for _, rc := range related { + if parentIDs[rc.ID] || rc.ID == config.ID { + continue + } + if _, exists := nodes[rc.ID]; !exists { + nodes[rc.ID] = &ptrNode{ + ConfigItem: relatedToConfigItem(rc), + edgeType: "related", + relation: rc.Relation, + } + } + } + + wired := make(map[uuid.UUID]bool) + for _, rc := range related { + if parentIDs[rc.ID] || rc.ID == config.ID || wired[rc.ID] { + continue + } + wired[rc.ID] = true + node := nodes[rc.ID] + if rc.Path != "" { + segments := strings.Split(rc.Path, ".") + if parentStr := segments[len(segments)-1]; parentStr != "" { + if pid, err := uuid.Parse(parentStr); err == nil { + if parent, ok := nodes[pid]; ok && parent != node && !parentIDs[pid] { + parent.children = append(parent.children, node) + continue + } + } + } + } + targetNode.children = append(targetNode.children, node) + } + + var root *ptrNode + if len(parents) > 0 { + root = nodes[parents[0].ID] + } else { + root = targetNode + } + + return toConfigTreeNode(root, make(map[*ptrNode]bool)) +} + +func toConfigTreeNode(n *ptrNode, visited map[*ptrNode]bool) *ConfigTreeNode { + result := &ConfigTreeNode{ + ConfigItem: n.ConfigItem, + EdgeType: n.edgeType, + Relation: n.relation, + } + if visited[n] { + return result + } + visited[n] = true + for _, c := range n.children { + result.Children = append(result.Children, toConfigTreeNode(c, visited)) + } + return result +} + +func relatedToConfigItem(rc RelatedConfig) models.ConfigItem { + ci := models.ConfigItem{ + ID: rc.ID, + } + ci.Name = &rc.Name + ci.Type = &rc.Type + ci.Tags = rc.Tags + ci.Status = rc.Status + ci.Health = rc.Health + ci.Path = rc.Path + ci.CreatedAt = rc.CreatedAt + ci.UpdatedAt = &rc.UpdatedAt + ci.DeletedAt = rc.DeletedAt + ci.AgentID = rc.AgentID + return ci +} From 27a19bbd6c4132ca18f49383a02b46eeef49fa10 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 6 Apr 2026 17:03:53 +0300 Subject: [PATCH 6/8] fix(gcp): detect credential type before requesting token source Switch from CredentialsFromJSONWithParams to CredentialsFromJSONWithType to properly handle different GCP credential formats. Add credential type detection helper to parse the JSON and extract the credential type field. --- connection/gcp.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/connection/gcp.go b/connection/gcp.go index 818f48da7..8b146a754 100644 --- a/connection/gcp.go +++ b/connection/gcp.go @@ -1,6 +1,7 @@ package connection import ( + "encoding/json" "fmt" "strings" "time" @@ -45,7 +46,12 @@ func (t *GCPConnection) FromModel(connection models.Connection) { } func (g *GCPConnection) TokenSource(ctx context.Context, scopes ...string) (oauth2.TokenSource, error) { - creds, err := google.CredentialsFromJSONWithParams(ctx, []byte(g.Credentials.ValueStatic), google.CredentialsParams{Scopes: scopes}) + credType, err := detectCredentialType([]byte(g.Credentials.ValueStatic)) + if err != nil { + return nil, fmt.Errorf("detecting credential type: %w", err) + } + + creds, err := google.CredentialsFromJSONWithType(ctx, []byte(g.Credentials.ValueStatic), credType, scopes...) if err != nil { return nil, err } @@ -53,6 +59,19 @@ func (g *GCPConnection) TokenSource(ctx context.Context, scopes ...string) (oaut return creds.TokenSource, nil } +func detectCredentialType(jsonData []byte) (google.CredentialsType, error) { + var f struct { + Type string `json:"type"` + } + if err := json.Unmarshal(jsonData, &f); err != nil { + return "", fmt.Errorf("parsing credentials JSON: %w", err) + } + if f.Type == "" { + return google.ServiceAccount, nil + } + return google.CredentialsType(f.Type), nil +} + func (g *GCPConnection) Validate() *GCPConnection { if g == nil { return &GCPConnection{} From b0f21686770ec3561dbd6581f7ce7e13e83bf538 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 6 Apr 2026 17:43:57 +0300 Subject: [PATCH 7/8] feat(query): add grouped summary logging for query results Implement QueryLogSummary interface to provide category-based summaries in query logs. Adds methods to multiple models (ConfigAnalysis, ConfigAccessSummary, ConfigChangeRow, RelatedConfig) and creates groupedSummary function to display counts by category (e.g. "CodeDeployment: 5, BackupCompleted: 3"). Also improves error handling in config tree resolution and adds embedded postgres cleanup hook. --- models/config.go | 4 ++++ models/config_access.go | 4 ++++ query/catalog_search.go | 4 ++++ query/config_changes.go | 11 +++++++++- query/config_relations.go | 2 +- query/config_tree.go | 42 +++++++++++++++++++++++++++++++-------- query/query_logger.go | 35 +++++++++++++++++++++++++++++++- tests/setup/template.go | 5 +++++ 8 files changed, 96 insertions(+), 11 deletions(-) diff --git a/models/config.go b/models/config.go index d3e82aa27..e4a68a56f 100644 --- a/models/config.go +++ b/models/config.go @@ -974,6 +974,10 @@ func (a ConfigAnalysis) PK() string { return a.ID.String() } +func (a ConfigAnalysis) QueryLogSummary() string { + return string(a.AnalysisType) + "/" + a.Analyzer +} + func (a ConfigAnalysis) TableName() string { return "config_analysis" } diff --git a/models/config_access.go b/models/config_access.go index ad889d698..c58e8b008 100644 --- a/models/config_access.go +++ b/models/config_access.go @@ -237,6 +237,10 @@ type ConfigAccessSummary struct { LastReviewedBy *uuid.UUID `json:"last_reviewed_by,omitempty"` } +func (e ConfigAccessSummary) QueryLogSummary() string { + return e.ConfigType +} + func (e ConfigAccessSummary) TableName() string { return "config_access_summary" } diff --git a/query/catalog_search.go b/query/catalog_search.go index e8693a516..9a6b766da 100644 --- a/query/catalog_search.go +++ b/query/catalog_search.go @@ -174,9 +174,13 @@ func (b *BaseCatalogSearch) String() string { } if b.From != "" { s += fmt.Sprintf("from=%s ", b.From) + } else if b.FromTime != nil { + s += fmt.Sprintf("from=%s ", b.FromTime.Format("2006-01-02")) } if b.To != "" { s += fmt.Sprintf("to=%s ", b.To) + } else if b.ToTime != nil { + s += fmt.Sprintf("to=%s ", b.ToTime.Format("2006-01-02")) } if b.Recursive != "" { s += fmt.Sprintf("recursive=%s ", b.Recursive) diff --git a/query/config_changes.go b/query/config_changes.go index 94e2eeb36..22d2990b0 100644 --- a/query/config_changes.go +++ b/query/config_changes.go @@ -149,6 +149,10 @@ type ConfigChangeRow struct { InsertedAt *time.Time `gorm:"column:inserted_at" json:"inserted_at,omitempty"` } +func (r ConfigChangeRow) QueryLogSummary() string { + return r.ChangeType +} + type CatalogChangesSearchResponse struct { Summary map[string]int `json:"summary,omitempty"` Total int64 `json:"total,omitempty"` @@ -164,7 +168,6 @@ func (t *CatalogChangesSearchResponse) Summarize() { func formSeverityQuery(severity string) string { if strings.HasPrefix(severity, "!") { - // For `Not` queries, we don't need to make any changes. return severity } @@ -177,13 +180,19 @@ func formSeverityQuery(severity string) string { } var applicable []string + found := false for _, s := range severities { applicable = append(applicable, string(s)) if string(s) == severity { + found = true break } } + if !found { + return "__invalid__" + } + return strings.Join(applicable, ",") } diff --git a/query/config_relations.go b/query/config_relations.go index 654855d55..868431b51 100644 --- a/query/config_relations.go +++ b/query/config_relations.go @@ -12,7 +12,7 @@ import ( ) func (r RelatedConfig) QueryLogSummary() string { - return r.Type + "/" + r.Name + return r.Type } type RelatedConfig struct { diff --git a/query/config_tree.go b/query/config_tree.go index a67d4d5ee..d62541bb9 100644 --- a/query/config_tree.go +++ b/query/config_tree.go @@ -1,6 +1,7 @@ package query import ( + "fmt" "strings" "github.com/flanksource/duty/context" @@ -22,6 +23,25 @@ type ConfigTreeOptions struct { Outgoing RelationType } +func (n *ConfigTreeNode) OutgoingIDs() []uuid.UUID { + var ids []uuid.UUID + n.collectOutgoing(&ids, make(map[uuid.UUID]bool)) + return ids +} + +func (n *ConfigTreeNode) collectOutgoing(ids *[]uuid.UUID, seen map[uuid.UUID]bool) { + if seen[n.ID] { + return + } + seen[n.ID] = true + if n.EdgeType != "parent" { + *ids = append(*ids, n.ID) + } + for _, c := range n.Children { + c.collectOutgoing(ids, seen) + } +} + func ConfigTree(ctx context.Context, configID uuid.UUID, opts ConfigTreeOptions) (*ConfigTreeNode, error) { config, err := GetCachedConfig(ctx, configID.String()) if err != nil { @@ -31,7 +51,10 @@ func ConfigTree(ctx context.Context, configID uuid.UUID, opts ConfigTreeOptions) return nil, nil } - parents := resolveParentsFromPath(ctx, config) + parents, err := resolveParentsFromPath(ctx, config) + if err != nil { + return nil, err + } childIDs, err := ExpandConfigChildren(ctx, []uuid.UUID{config.ID}) if err != nil { @@ -54,7 +77,7 @@ func ConfigTree(ctx context.Context, configID uuid.UUID, opts ConfigTreeOptions) if opts.Incoming != "" { relType = opts.Incoming } - outType := Hard + outType := Both if opts.Outgoing != "" { outType = opts.Outgoing } @@ -72,9 +95,9 @@ func ConfigTree(ctx context.Context, configID uuid.UUID, opts ConfigTreeOptions) return buildConfigTree(config, parents, children, related), nil } -func resolveParentsFromPath(ctx context.Context, config *models.ConfigItem) []models.ConfigItem { +func resolveParentsFromPath(ctx context.Context, config *models.ConfigItem) ([]models.ConfigItem, error) { if config.Path == "" { - return nil + return nil, nil } segments := strings.Split(config.Path, ".") var parentIDs []uuid.UUID @@ -86,11 +109,14 @@ func resolveParentsFromPath(ctx context.Context, config *models.ConfigItem) []mo parentIDs = append(parentIDs, id) } if len(parentIDs) == 0 { - return nil + return nil, nil } items, err := GetConfigsByIDs(ctx, parentIDs) - if err != nil || len(items) == 0 { - return nil + if err != nil { + return nil, fmt.Errorf("resolving parents from path: %w", err) + } + if len(items) == 0 { + return nil, nil } byID := make(map[uuid.UUID]models.ConfigItem, len(items)) for _, ci := range items { @@ -102,7 +128,7 @@ func resolveParentsFromPath(ctx context.Context, config *models.ConfigItem) []mo parents = append(parents, ci) } } - return parents + return parents, nil } func ExpandConfigChildren(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { diff --git a/query/query_logger.go b/query/query_logger.go index a576eae3d..a30ba3591 100644 --- a/query/query_logger.go +++ b/query/query_logger.go @@ -40,8 +40,12 @@ func (q QueryLogger) Start(entity string) *QueryTimer { } func (t *QueryTimer) Arg(key string, value any) *QueryTimer { + s := fmt.Sprintf("%v", value) + if len(s) > 80 { + s = s[:77] + "..." + } t.label = t.label.AddText(fmt.Sprintf(" %s=", key), "text-gray-500"). - AddText(fmt.Sprintf("%v", value)) + AddText(s) return t } @@ -101,6 +105,11 @@ func summaryText(v any, count int) string { return "" } + // Try grouped summary first (e.g. "CodeDeployment: 5, BackupCompleted: 3") + if grouped := groupedSummary(rv); grouped != "" { + return " [" + grouped + "]" + } + const maxInline = 2 shown := count if shown > maxInline { @@ -117,3 +126,27 @@ func summaryText(v any, count int) string { } return " [" + summary + "]" } + +func groupedSummary(rv reflect.Value) string { + if rv.Len() == 0 { + return "" + } + first := rv.Index(0).Interface() + if _, ok := first.(QueryLogSummary); !ok { + return "" + } + counts := make(map[string]int) + var order []string + for i := 0; i < rv.Len(); i++ { + s := rv.Index(i).Interface().(QueryLogSummary).QueryLogSummary() + if counts[s] == 0 { + order = append(order, s) + } + counts[s]++ + } + var parts []string + for _, key := range order { + parts = append(parts, fmt.Sprintf("%s: %d", key, counts[key])) + } + return strings.Join(parts, ", ") +} diff --git a/tests/setup/template.go b/tests/setup/template.go index 02a3770ff..b97d7127f 100644 --- a/tests/setup/template.go +++ b/tests/setup/template.go @@ -234,6 +234,11 @@ func ensurePostgres(port int) (string, error) { if err := postgresServer.Start(); err != nil { return "", err } + shutdown.AddHookWithPriority("stop embedded postgres", shutdown.PriorityCritical, func() { + if err := postgresServer.Stop(); err != nil { + logger.Errorf("failed to stop embedded postgres: %v", err) + } + }) logger.Infof("Started postgres on port %d", port) } From 1d3b03a069acf22b6cdaefc6a1b3e0d0e3a02194 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 6 Apr 2026 22:29:03 +0300 Subject: [PATCH 8/8] fix(query): address PR review issues and add lenient search mode - formSeverityQuery returns error instead of "__invalid__" for unknown severity - ApplyClauses returns error and accepts excludeColumns to skip inapplicable columns - FindCatalogInsights/FindCatalogChanges propagate filter parse errors - Remove unsafe type assertion in config_insights clause filtering - timer.Results only called on successful DB query in GetRelatedConfigs - Fix path segment off-by-one in buildConfigTree (use penultimate segment) - Empty config resolution returns empty results instead of global unscoped query - FindConfigAccessByConfigIDs uses large PageSize to avoid truncation - Add Lenient flag to BaseCatalogSearch for global search use cases --- query/catalog_search.go | 71 ++++++++++++++++++++++++--------- query/config_access.go | 5 ++- query/config_changes.go | 84 ++++++++++++++++++++++++--------------- query/config_insights.go | 51 ++++++++++++++---------- query/config_relations.go | 8 ++-- query/config_tree.go | 13 +++--- 6 files changed, 150 insertions(+), 82 deletions(-) diff --git a/query/catalog_search.go b/query/catalog_search.go index 9a6b766da..b9eb74db5 100644 --- a/query/catalog_search.go +++ b/query/catalog_search.go @@ -29,6 +29,11 @@ type BaseCatalogSearch struct { Recursive ChangeRelationDirection `query:"recursive" json:"recursive"` Soft bool `query:"soft" json:"soft"` + // Lenient silently ignores invalid filters, inapplicable columns, + // and validation errors instead of returning errors. + // Useful for global search where the same filters are applied across different table types. + Lenient bool `query:"lenient" json:"lenient"` + sortOrder string configIDs []uuid.UUID FromTime *time.Time `query:"-" json:"-"` @@ -57,25 +62,39 @@ func (b *BaseCatalogSearch) Validate() error { if b.From != "" && b.FromTime == nil { expr, err := datemath.Parse(b.From) if err != nil { - return fmt.Errorf("invalid 'from' param: %w", err) + if !b.Lenient { + return fmt.Errorf("invalid 'from' param: %w", err) + } + b.From = "" + } else { + t := expr.Time() + b.FromTime = &t } - t := expr.Time() - b.FromTime = &t } if b.To != "" && b.ToTime == nil { expr, err := datemath.Parse(b.To) if err != nil { - return fmt.Errorf("invalid 'to' param: %w", err) + if !b.Lenient { + return fmt.Errorf("invalid 'to' param: %w", err) + } + b.To = "" + } else { + t := expr.Time() + b.ToTime = &t } - t := expr.Time() - b.ToTime = &t } if b.FromTime != nil && b.ToTime != nil && !b.FromTime.Before(*b.ToTime) { - return fmt.Errorf("'from' must be before 'to'") + if !b.Lenient { + return fmt.Errorf("'from' must be before 'to'") + } + b.ToTime = nil } if b.AgentID != "" { if _, err := uuid.Parse(b.AgentID); err != nil { - return fmt.Errorf("agent_id(%s) must either be a valid uuid or `local`", b.AgentID) + if !b.Lenient { + return fmt.Errorf("agent_id(%s) must either be a valid uuid or `local`", b.AgentID) + } + b.AgentID = "" } } return nil @@ -121,31 +140,45 @@ func (b *BaseCatalogSearch) ConfigIDs() []uuid.UUID { return b.configIDs } -func (b *BaseCatalogSearch) ApplyClauses() ([]clause.Expression, func(*gorm.DB) *gorm.DB) { +func (b *BaseCatalogSearch) ApplyClauses(excludeColumns ...string) ([]clause.Expression, func(*gorm.DB) *gorm.DB, error) { + excluded := make(map[string]bool, len(excludeColumns)) + for _, c := range excludeColumns { + excluded[c] = true + } + var clauses []clause.Expression var tagsFn func(*gorm.DB) *gorm.DB - if b.AgentID != "" { - if c, err := parseAndBuildFilteringQuery(b.AgentID, "agent_id", false); err == nil { + if b.AgentID != "" && !excluded["agent_id"] { + c, err := parseAndBuildFilteringQuery(b.AgentID, "agent_id", false) + if err != nil && !b.Lenient { + return nil, nil, fmt.Errorf("invalid agent_id filter: %w", err) + } else if err == nil { clauses = append(clauses, c...) } } - if b.ConfigType != "" { - if c, err := parseAndBuildFilteringQuery(b.ConfigType, "type", false); err == nil { + if b.ConfigType != "" && !excluded["type"] { + c, err := parseAndBuildFilteringQuery(b.ConfigType, "type", false) + if err != nil && !b.Lenient { + return nil, nil, fmt.Errorf("invalid config_type filter: %w", err) + } else if err == nil { clauses = append(clauses, c...) } } - if b.FromTime != nil { + if b.FromTime != nil && !excluded["created_at"] { clauses = append(clauses, clause.Gte{Column: clause.Column{Name: "created_at"}, Value: *b.FromTime}) } - if b.ToTime != nil { + if b.ToTime != nil && !excluded["created_at"] { clauses = append(clauses, clause.Lte{Column: clause.Column{Name: "created_at"}, Value: *b.ToTime}) } - if !b.IncludeDeletedConfigs { + if !b.IncludeDeletedConfigs && !excluded["deleted_at"] { clauses = append(clauses, clause.Eq{Column: clause.Column{Name: "deleted_at"}, Value: nil}) } - if b.Tags != "" { - if parsedLabelSelector, err := labels.Parse(b.Tags); err == nil { + if b.Tags != "" && !excluded["tags"] { + parsedLabelSelector, err := labels.Parse(b.Tags) + if err != nil && !b.Lenient { + return nil, nil, fmt.Errorf("invalid tags filter: %w", err) + } else if err == nil { requirements, _ := parsedLabelSelector.Requirements() tagsFn = func(db *gorm.DB) *gorm.DB { for _, r := range requirements { @@ -155,7 +188,7 @@ func (b *BaseCatalogSearch) ApplyClauses() ([]clause.Expression, func(*gorm.DB) } } } - return clauses, tagsFn + return clauses, tagsFn, nil } func (b *BaseCatalogSearch) String() string { diff --git a/query/config_access.go b/query/config_access.go index c45f58a46..d21399b86 100644 --- a/query/config_access.go +++ b/query/config_access.go @@ -29,6 +29,9 @@ func FindCatalogAccess(ctx context.Context, req CatalogAccessSearchRequest) (res if err != nil { return nil, err } + if len(configIDs) == 0 && req.CatalogID != "" { + return &CatalogAccessSearchResponse{}, nil + } var output CatalogAccessSearchResponse q := ctx.DB().Table("config_access_summary") @@ -53,7 +56,7 @@ func FindCatalogAccess(ctx context.Context, req CatalogAccessSearchRequest) (res func FindConfigAccessByConfigIDs(ctx context.Context, configIDs []uuid.UUID) ([]models.ConfigAccessSummary, error) { resp, err := FindCatalogAccess(ctx, CatalogAccessSearchRequest{ - BaseCatalogSearch: BaseCatalogSearch{configIDs: configIDs}, + BaseCatalogSearch: BaseCatalogSearch{configIDs: configIDs, PageSize: 10000}, }) if err != nil { return nil, err diff --git a/query/config_changes.go b/query/config_changes.go index 22d2990b0..b9f58301b 100644 --- a/query/config_changes.go +++ b/query/config_changes.go @@ -82,12 +82,18 @@ func (t *CatalogChangesSearchRequest) Validate() error { } if !lo.Contains(allRecursiveOptions, t.Recursive) { - return fmt.Errorf("'recursive' must be one of %v", allRecursiveOptions) + if !t.Lenient { + return fmt.Errorf("'recursive' must be one of %v", allRecursiveOptions) + } + t.Recursive = CatalogChangeRecursiveDownstream } if t.FromInsertedAt != "" { if expr, err := datemath.Parse(t.FromInsertedAt); err != nil { - return fmt.Errorf("invalid 'from_inserted_at' param: %w", err) + if !t.Lenient { + return fmt.Errorf("invalid 'from_inserted_at' param: %w", err) + } + t.FromInsertedAt = "" } else { t.fromInsertedAtParsed = expr.Time() } @@ -95,14 +101,20 @@ func (t *CatalogChangesSearchRequest) Validate() error { if t.ToInsertedAt != "" { if expr, err := datemath.Parse(t.ToInsertedAt); err != nil { - return fmt.Errorf("invalid 'to_inserted_at' param: %w", err) + if !t.Lenient { + return fmt.Errorf("invalid 'to_inserted_at' param: %w", err) + } + t.ToInsertedAt = "" } else { t.toInsertedAtParsed = expr.Time() } } if !t.fromInsertedAtParsed.IsZero() && !t.toInsertedAtParsed.IsZero() && !t.fromInsertedAtParsed.Before(t.toInsertedAtParsed) { - return fmt.Errorf("'from_inserted_at' must be before 'to_inserted_at'") + if !t.Lenient { + return fmt.Errorf("'from_inserted_at' must be before 'to_inserted_at'") + } + t.toInsertedAtParsed = time.Time{} } if t.SortBy != "" { @@ -112,7 +124,10 @@ func (t *CatalogChangesSearchRequest) Validate() error { } if !lo.Contains(allowedConfigChangesSortColumns, t.SortBy) { - return fmt.Errorf("invalid 'sort_by' param: %s. allowed sort fields are: %s", t.SortBy, strings.Join(allowedConfigChangesSortColumns, ", ")) + if !t.Lenient { + return fmt.Errorf("invalid 'sort_by' param: %s. allowed sort fields are: %s", t.SortBy, strings.Join(allowedConfigChangesSortColumns, ", ")) + } + t.SortBy = "" } } @@ -166,9 +181,9 @@ func (t *CatalogChangesSearchResponse) Summarize() { } } -func formSeverityQuery(severity string) string { +func formSeverityQuery(severity string) (string, error) { if strings.HasPrefix(severity, "!") { - return severity + return severity, nil } severities := []models.Severity{ @@ -180,20 +195,14 @@ func formSeverityQuery(severity string) string { } var applicable []string - found := false for _, s := range severities { applicable = append(applicable, string(s)) if string(s) == severity { - found = true - break + return strings.Join(applicable, ","), nil } } - if !found { - return "__invalid__" - } - - return strings.Join(applicable, ",") + return "", fmt.Errorf("unknown severity %q", severity) } func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (result *CatalogChangesSearchResponse, err error) { @@ -209,8 +218,14 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (r if err != nil { return nil, err } + if len(configIDs) == 0 && req.CatalogID != "" { + return &CatalogChangesSearchResponse{}, nil + } - baseClauses, tagsFn := req.ApplyClauses() + baseClauses, tagsFn, err := req.ApplyClauses() + if err != nil { + return nil, api.Errorf(api.EINVALID, "bad request: %v", err) + } var clauses []clause.Expression clauses = append(clauses, baseClauses...) @@ -220,34 +235,39 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (r } if req.ChangeType != "" { - if c, parseErr := parseAndBuildFilteringQuery(req.ChangeType, "change_type", false); parseErr == nil { - clauses = append(clauses, c...) - } else { + if c, parseErr := parseAndBuildFilteringQuery(req.ChangeType, "change_type", false); parseErr != nil && !req.Lenient { return nil, parseErr + } else if parseErr == nil { + clauses = append(clauses, c...) } } if req.Severity != "" { - if c, parseErr := parseAndBuildFilteringQuery(formSeverityQuery(req.Severity), "severity", false); parseErr == nil { - clauses = append(clauses, c...) - } else { - return nil, api.Errorf(api.EINVALID, "failed to parse severity: %v", parseErr) + severityQuery, err := formSeverityQuery(req.Severity) + if err != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "invalid severity: %v", err) + } else if err == nil { + if c, parseErr := parseAndBuildFilteringQuery(severityQuery, "severity", false); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse severity: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) + } } } if req.Summary != "" { - if c, parseErr := parseAndBuildFilteringQuery(req.Summary, "summary", true); parseErr == nil { - clauses = append(clauses, c...) - } else { + if c, parseErr := parseAndBuildFilteringQuery(req.Summary, "summary", true); parseErr != nil && !req.Lenient { return nil, api.Errorf(api.EINVALID, "failed to parse summary: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) } } if req.Source != "" { - if c, parseErr := parseAndBuildFilteringQuery(req.Source, "source", true); parseErr == nil { - clauses = append(clauses, c...) - } else { + if c, parseErr := parseAndBuildFilteringQuery(req.Source, "source", true); parseErr != nil && !req.Lenient { return nil, api.Errorf(api.EINVALID, "failed to parse source: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) } } @@ -264,10 +284,10 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (r } if req.externalCreatedBy != "" { - if c, parseErr := parseAndBuildFilteringQuery(req.externalCreatedBy, "external_created_by", true); parseErr == nil { - clauses = append(clauses, c...) - } else { + if c, parseErr := parseAndBuildFilteringQuery(req.externalCreatedBy, "external_created_by", true); parseErr != nil && !req.Lenient { return nil, api.Errorf(api.EINVALID, "failed to parse external createdby: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) } } diff --git a/query/config_insights.go b/query/config_insights.go index 39772e36b..8254327bb 100644 --- a/query/config_insights.go +++ b/query/config_insights.go @@ -40,35 +40,54 @@ func FindCatalogInsights(ctx context.Context, req CatalogInsightsSearchRequest) if err != nil { return nil, err } + if len(configIDs) == 0 && req.CatalogID != "" { + return &CatalogInsightsSearchResponse{}, nil + } + // config_analysis table doesn't have deleted_at, agent_id, type, or tags columns + baseClauses, tagsFn, err := req.ApplyClauses("deleted_at", "agent_id", "type", "tags") + if err != nil { + return nil, api.Errorf(api.EINVALID, "bad request: %v", err) + } var clauses []clause.Expression - baseClauses, tagsFn := req.ApplyClauses() clauses = append(clauses, baseClauses...) q := ctx.DB().Table("config_analysis") if len(configIDs) > 0 { - clauses = append(clauses, clause.Eq{Column: clause.Column{Name: "config_id"}, Value: nil}) q = q.Where("config_id IN ?", configIDs) } if req.Status != "" { - if c, parseErr := parseAndBuildFilteringQuery(req.Status, "status", false); parseErr == nil { + if c, parseErr := parseAndBuildFilteringQuery(req.Status, "status", false); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse status: %v", parseErr) + } else if parseErr == nil { clauses = append(clauses, c...) } } if req.Severity != "" { - if c, parseErr := parseAndBuildFilteringQuery(formSeverityQuery(req.Severity), "severity", false); parseErr == nil { - clauses = append(clauses, c...) + severityQuery, err := formSeverityQuery(req.Severity) + if err != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "invalid severity: %v", err) + } else if err == nil { + if c, parseErr := parseAndBuildFilteringQuery(severityQuery, "severity", false); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse severity: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) + } } } if req.Analyzer != "" { - if c, parseErr := parseAndBuildFilteringQuery(req.Analyzer, "analyzer", true); parseErr == nil { + if c, parseErr := parseAndBuildFilteringQuery(req.Analyzer, "analyzer", true); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse analyzer: %v", parseErr) + } else if parseErr == nil { clauses = append(clauses, c...) } } if req.AnalysisType != "" { - if c, parseErr := parseAndBuildFilteringQuery(req.AnalysisType, "analysis_type", false); parseErr == nil { + if c, parseErr := parseAndBuildFilteringQuery(req.AnalysisType, "analysis_type", false); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse analysis_type: %v", parseErr) + } else if parseErr == nil { clauses = append(clauses, c...) } } @@ -78,19 +97,7 @@ func FindCatalogInsights(ctx context.Context, req CatalogInsightsSearchRequest) } var output CatalogInsightsSearchResponse - // Remove the dummy deleted_at clause for config_analysis (it doesn't have deleted_at) - filteredClauses := make([]clause.Expression, 0, len(clauses)) - for _, c := range clauses { - if eq, ok := c.(clause.Eq); ok && eq.Column.(clause.Column).Name == "deleted_at" { - continue - } - if eq, ok := c.(clause.Eq); ok && eq.Column.(clause.Column).Name == "config_id" && eq.Value == nil { - continue - } - filteredClauses = append(filteredClauses, c) - } - - if err := q.Clauses(filteredClauses...).Count(&output.Total).Error; err != nil { + if err := q.Clauses(clauses...).Count(&output.Total).Error; err != nil { return nil, err } if output.Total == 0 { @@ -98,11 +105,11 @@ func FindCatalogInsights(ctx context.Context, req CatalogInsightsSearchRequest) return &output, nil } - filteredClauses = append(filteredClauses, + clauses = append(clauses, clause.Limit{Limit: &req.PageSize, Offset: (req.Page - 1) * req.PageSize}, ) - if err := q.Clauses(filteredClauses...).Find(&output.Insights).Error; err != nil { + if err := q.Clauses(clauses...).Find(&output.Insights).Error; err != nil { return nil, err } timer.Results(output.Insights) diff --git a/query/config_relations.go b/query/config_relations.go index 868431b51..8d99f1467 100644 --- a/query/config_relations.go +++ b/query/config_relations.go @@ -94,15 +94,17 @@ func GetRelatedConfigs(ctx context.Context, query RelationQuery) (results []Rela // FIXME: Self config is returned here for creating graph in UI. We need to update UI to // add the node itself. Issue: github.com/flanksource/duty/issues/1722 - err = ctx.DB().Raw("SELECT * FROM related_configs_recursive(?, ?, ?, ?, ?, ?)", + if err = ctx.DB().Raw("SELECT * FROM related_configs_recursive(?, ?, ?, ?, ?, ?)", query.ID, query.Relation, query.IncludeDeleted, *query.MaxDepth, query.Incoming, - query.Outgoing).Find(&results).Error + query.Outgoing).Find(&results).Error; err != nil { + return nil, err + } results = lo.Filter(results, func(c RelatedConfig, _ int) bool { return c.ID != query.ID }) timer.Results(results) - return results, err + return results, nil } diff --git a/query/config_tree.go b/query/config_tree.go index d62541bb9..d371a19e0 100644 --- a/query/config_tree.go +++ b/query/config_tree.go @@ -212,11 +212,14 @@ func buildConfigTree(config *models.ConfigItem, parents []models.ConfigItem, chi node := nodes[rc.ID] if rc.Path != "" { segments := strings.Split(rc.Path, ".") - if parentStr := segments[len(segments)-1]; parentStr != "" { - if pid, err := uuid.Parse(parentStr); err == nil { - if parent, ok := nodes[pid]; ok && parent != node && !parentIDs[pid] { - parent.children = append(parent.children, node) - continue + // Last segment is the node's own ID (SetParent appends ci.ID), so use penultimate + if len(segments) >= 2 { + if parentStr := segments[len(segments)-2]; parentStr != "" { + if pid, err := uuid.Parse(parentStr); err == nil { + if parent, ok := nodes[pid]; ok && parent != node && !parentIDs[pid] { + parent.children = append(parent.children, node) + continue + } } } }