diff --git a/api/config.go b/api/config.go index 1f1e788fe..0ea8d5b28 100644 --- a/api/config.go +++ b/api/config.go @@ -14,6 +14,7 @@ var DefaultConfig = Config{ Postgrest: PostgrestConfig{ Version: "v14.6", DBRole: "postgrest_api", + Arch: runtime.GOARCH, AnonDBRole: "", Port: 3000, AdminPort: 3001, @@ -22,7 +23,9 @@ var DefaultConfig = Config{ } func init() { + DefaultConfig.Postgrest = DefaultConfig.Postgrest.ReadEnv() v := DefaultConfig.Postgrest.Version + if strings.HasPrefix(v, "v14") && v != "v14.1" && v != "v14.0" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" { logger.Warnf("PostgREST v14.2+ does not have a darwin/arm64 binary, defaulting to v14.1 for darwin/amd64") @@ -124,6 +127,7 @@ type PostgrestConfig struct { LogLevel string URL string Version string + Arch string JWTSecret string DBRole string AnonDBRole string @@ -146,6 +150,9 @@ func (p PostgrestConfig) ReadEnv() PostgrestConfig { if v := os.Getenv("PGRST_VERSION"); v != "" { clone.Version = v } + if v := os.Getenv("PGRST_ARCH"); v != "" { + clone.Arch = v + } return clone } diff --git a/connection/gcp.go b/connection/gcp.go index c1b3db5e8..8b146a754 100644 --- a/connection/gcp.go +++ b/connection/gcp.go @@ -1,6 +1,7 @@ package connection import ( + "encoding/json" "fmt" "strings" "time" @@ -45,13 +46,30 @@ func (t *GCPConnection) FromModel(connection models.Connection) { } func (g *GCPConnection) TokenSource(ctx context.Context, scopes ...string) (oauth2.TokenSource, error) { - creds, err := google.CredentialsFromJSON(ctx, []byte(g.Credentials.ValueStatic), scopes...) //nolint:staticcheck + credType, err := detectCredentialType([]byte(g.Credentials.ValueStatic)) + if err != nil { + return nil, fmt.Errorf("detecting credential type: %w", err) + } + + creds, err := google.CredentialsFromJSONWithType(ctx, []byte(g.Credentials.ValueStatic), credType, scopes...) if err != nil { return nil, err } - tokenSource := creds.TokenSource - return tokenSource, nil + return creds.TokenSource, nil +} + +func detectCredentialType(jsonData []byte) (google.CredentialsType, error) { + var f struct { + Type string `json:"type"` + } + if err := json.Unmarshal(jsonData, &f); err != nil { + return "", fmt.Errorf("parsing credentials JSON: %w", err) + } + if f.Type == "" { + return google.ServiceAccount, nil + } + return google.CredentialsType(f.Type), nil } func (g *GCPConnection) Validate() *GCPConnection { diff --git a/models/config.go b/models/config.go index d3e82aa27..e4a68a56f 100644 --- a/models/config.go +++ b/models/config.go @@ -974,6 +974,10 @@ func (a ConfigAnalysis) PK() string { return a.ID.String() } +func (a ConfigAnalysis) QueryLogSummary() string { + return string(a.AnalysisType) + "/" + a.Analyzer +} + func (a ConfigAnalysis) TableName() string { return "config_analysis" } diff --git a/models/config_access.go b/models/config_access.go index ad889d698..c58e8b008 100644 --- a/models/config_access.go +++ b/models/config_access.go @@ -237,6 +237,10 @@ type ConfigAccessSummary struct { LastReviewedBy *uuid.UUID `json:"last_reviewed_by,omitempty"` } +func (e ConfigAccessSummary) QueryLogSummary() string { + return e.ConfigType +} + func (e ConfigAccessSummary) TableName() string { return "config_access_summary" } diff --git a/postgrest/postgrest.go b/postgrest/postgrest.go index e32f39ae1..221de2025 100644 --- a/postgrest/postgrest.go +++ b/postgrest/postgrest.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "runtime" "strconv" "github.com/flanksource/commons/exec" @@ -17,7 +18,10 @@ func GoOffline() error { } func runBinary(config api.Config, msg string, args ...any) error { - result, err := deps.InstallWithContext(context.Background(), "postgrest", config.Postgrest.Version, deps.WithBinDir(".bin")) + result, err := deps.InstallWithContext(context.Background(), "postgrest", + config.Postgrest.Version, + deps.WithBinDir(".bin"), + deps.WithOS(runtime.GOOS, config.Postgrest.Arch)) if err != nil { return fmt.Errorf("failed to install postgREST: %w", err) } diff --git a/query/agent.go b/query/agent.go index d20b3594f..517823bd9 100644 --- a/query/agent.go +++ b/query/agent.go @@ -11,17 +11,21 @@ import ( "gorm.io/gorm" ) -func FindAgent(ctx context.Context, name string) (*models.Agent, error) { +func FindAgent(ctx context.Context, name string) (result *models.Agent, err error) { + timer := NewQueryLogger(ctx).Start("Agent").Arg("name", name) + defer timer.End(&err) + var agent models.Agent - err := ctx.DB().Where("name = ?", name).First(&agent).Error + err = ctx.DB().Where("name = ?", name).First(&agent).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { + err = nil + timer.Results([]models.Agent{}) return nil, nil } - return nil, err } - + timer.Results([]models.Agent{agent}) return &agent, nil } diff --git a/query/catalog_search.go b/query/catalog_search.go new file mode 100644 index 000000000..b9eb74db5 --- /dev/null +++ b/query/catalog_search.go @@ -0,0 +1,222 @@ +package query + +import ( + "fmt" + "strings" + "time" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/pkg/kube/labels" + "github.com/flanksource/duty/types" + "github.com/google/uuid" + "github.com/timberio/go-datemath" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type BaseCatalogSearch struct { + CatalogID string `query:"id" json:"id"` + ConfigType string `query:"config_type" json:"config_type"` + IncludeDeletedConfigs bool `query:"include_deleted_configs" json:"include_deleted_configs"` + Depth int `query:"depth" json:"depth"` + Tags string `query:"tags" json:"tags"` + AgentID string `query:"agent_id" json:"agent_id"` + From string `query:"from" json:"from"` + To string `query:"to" json:"to"` + PageSize int `query:"page_size" json:"page_size"` + Page int `query:"page" json:"page"` + SortBy string `query:"sort_by" json:"sort_by"` + Recursive ChangeRelationDirection `query:"recursive" json:"recursive"` + Soft bool `query:"soft" json:"soft"` + + // Lenient silently ignores invalid filters, inapplicable columns, + // and validation errors instead of returning errors. + // Useful for global search where the same filters are applied across different table types. + Lenient bool `query:"lenient" json:"lenient"` + + sortOrder string + configIDs []uuid.UUID + FromTime *time.Time `query:"-" json:"-"` + ToTime *time.Time `query:"-" json:"-"` +} + +func (b *BaseCatalogSearch) SetDefaults() { + if b.PageSize <= 0 { + b.PageSize = 50 + } + if b.Page <= 0 { + b.Page = 1 + } + if b.Depth <= 0 { + b.Depth = 5 + } + if b.Recursive == "" { + b.Recursive = CatalogChangeRecursiveDownstream + } + if b.AgentID == "local" { + b.AgentID = uuid.Nil.String() + } +} + +func (b *BaseCatalogSearch) Validate() error { + if b.From != "" && b.FromTime == nil { + expr, err := datemath.Parse(b.From) + if err != nil { + if !b.Lenient { + return fmt.Errorf("invalid 'from' param: %w", err) + } + b.From = "" + } else { + t := expr.Time() + b.FromTime = &t + } + } + if b.To != "" && b.ToTime == nil { + expr, err := datemath.Parse(b.To) + if err != nil { + if !b.Lenient { + return fmt.Errorf("invalid 'to' param: %w", err) + } + b.To = "" + } else { + t := expr.Time() + b.ToTime = &t + } + } + if b.FromTime != nil && b.ToTime != nil && !b.FromTime.Before(*b.ToTime) { + if !b.Lenient { + return fmt.Errorf("'from' must be before 'to'") + } + b.ToTime = nil + } + if b.AgentID != "" { + if _, err := uuid.Parse(b.AgentID); err != nil { + if !b.Lenient { + return fmt.Errorf("agent_id(%s) must either be a valid uuid or `local`", b.AgentID) + } + b.AgentID = "" + } + } + return nil +} + +func (b *BaseCatalogSearch) ResolveConfigIDs(ctx context.Context) ([]uuid.UUID, error) { + if b.CatalogID == "" { + return nil, nil + } + parts := strings.Split(b.CatalogID, ",") + var ids []uuid.UUID + allValid := true + for _, p := range parts { + if id, err := uuid.Parse(strings.TrimSpace(p)); err == nil { + ids = append(ids, id) + } else { + allValid = false + break + } + } + if allValid && len(ids) > 0 { + b.configIDs = ids + return ids, nil + } + + response, err := SearchResources(ctx, SearchResourcesRequest{ + Configs: []types.ResourceSelector{{Search: b.CatalogID, Cache: "no-cache"}}, + Limit: 200, + }) + if err != nil { + return nil, fmt.Errorf("failed to resolve catalog query %q: %w", b.CatalogID, err) + } + for _, c := range response.Configs { + if id, err := uuid.Parse(c.ID); err == nil { + ids = append(ids, id) + } + } + b.configIDs = ids + return ids, nil +} + +func (b *BaseCatalogSearch) ConfigIDs() []uuid.UUID { + return b.configIDs +} + +func (b *BaseCatalogSearch) ApplyClauses(excludeColumns ...string) ([]clause.Expression, func(*gorm.DB) *gorm.DB, error) { + excluded := make(map[string]bool, len(excludeColumns)) + for _, c := range excludeColumns { + excluded[c] = true + } + + var clauses []clause.Expression + var tagsFn func(*gorm.DB) *gorm.DB + + if b.AgentID != "" && !excluded["agent_id"] { + c, err := parseAndBuildFilteringQuery(b.AgentID, "agent_id", false) + if err != nil && !b.Lenient { + return nil, nil, fmt.Errorf("invalid agent_id filter: %w", err) + } else if err == nil { + clauses = append(clauses, c...) + } + } + if b.ConfigType != "" && !excluded["type"] { + c, err := parseAndBuildFilteringQuery(b.ConfigType, "type", false) + if err != nil && !b.Lenient { + return nil, nil, fmt.Errorf("invalid config_type filter: %w", err) + } else if err == nil { + clauses = append(clauses, c...) + } + } + if b.FromTime != nil && !excluded["created_at"] { + clauses = append(clauses, clause.Gte{Column: clause.Column{Name: "created_at"}, Value: *b.FromTime}) + } + if b.ToTime != nil && !excluded["created_at"] { + clauses = append(clauses, clause.Lte{Column: clause.Column{Name: "created_at"}, Value: *b.ToTime}) + } + if !b.IncludeDeletedConfigs && !excluded["deleted_at"] { + clauses = append(clauses, clause.Eq{Column: clause.Column{Name: "deleted_at"}, Value: nil}) + } + if b.Tags != "" && !excluded["tags"] { + parsedLabelSelector, err := labels.Parse(b.Tags) + if err != nil && !b.Lenient { + return nil, nil, fmt.Errorf("invalid tags filter: %w", err) + } else if err == nil { + requirements, _ := parsedLabelSelector.Requirements() + tagsFn = func(db *gorm.DB) *gorm.DB { + for _, r := range requirements { + db = jsonColumnRequirementsToSQLClause(db, "tags", r) + } + return db + } + } + } + return clauses, tagsFn, nil +} + +func (b *BaseCatalogSearch) String() string { + s := "" + if b.AgentID != "" { + s += fmt.Sprintf("agent=%s ", b.AgentID) + } + if b.CatalogID != "" { + s += fmt.Sprintf("id=%s ", b.CatalogID) + } + if b.ConfigType != "" { + s += fmt.Sprintf("config_type=%s ", b.ConfigType) + } + if b.Tags != "" { + s += fmt.Sprintf("tags=%s ", b.Tags) + } + if b.From != "" { + s += fmt.Sprintf("from=%s ", b.From) + } else if b.FromTime != nil { + s += fmt.Sprintf("from=%s ", b.FromTime.Format("2006-01-02")) + } + if b.To != "" { + s += fmt.Sprintf("to=%s ", b.To) + } else if b.ToTime != nil { + s += fmt.Sprintf("to=%s ", b.ToTime.Format("2006-01-02")) + } + if b.Recursive != "" { + s += fmt.Sprintf("recursive=%s ", b.Recursive) + } + return strings.TrimSpace(s) +} diff --git a/query/config.go b/query/config.go index be91777b3..6dd6e4c78 100644 --- a/query/config.go +++ b/query/config.go @@ -460,7 +460,10 @@ func FindConfigIDsByResourceSelector(ctx context.Context, limit int, resourceSel return queryTableWithResourceSelectors(ctx, "config_items", limit, resourceSelectors...) } -func FindConfigForComponent(ctx context.Context, componentID, configType string) ([]models.ConfigItem, error) { +func FindConfigForComponent(ctx context.Context, componentID, configType string) (results []models.ConfigItem, err error) { + timer := NewQueryLogger(ctx).Start("ConfigForComponent").Arg("componentID", componentID).Arg("configType", configType) + defer timer.End(&err) + db := ctx.DB() relationshipQuery := db.Table("config_component_relationships"). Select("config_id"). @@ -469,27 +472,31 @@ func FindConfigForComponent(ctx context.Context, componentID, configType string) if configType != "" { query = query.Where("type = @config_type OR config_class = @config_type", sql.Named("config_type", configType)) } - var dbConfigObjects []models.ConfigItem - err := query.Find(&dbConfigObjects).Error - return dbConfigObjects, err + err = query.Find(&results).Error + timer.Results(results) + return results, err } -func FindConfigChildrenIDsByLocation(ctx context.Context, configID uuid.UUID, prefix string) ([]uuid.UUID, error) { - var children []uuid.UUID - if err := ctx.DB().Raw(`SELECT id FROM get_children_id_by_location(?, ?)`, configID, prefix).Scan(&children).Error; err != nil { +func FindConfigChildrenIDsByLocation(ctx context.Context, configID uuid.UUID, prefix string) (results []uuid.UUID, err error) { + timer := NewQueryLogger(ctx).Start("ConfigChildrenIDs").Arg("configID", configID).Arg("prefix", prefix) + defer timer.End(&err) + + if err = ctx.DB().Raw(`SELECT id FROM get_children_id_by_location(?, ?)`, configID, prefix).Scan(&results).Error; err != nil { return nil, err } - - return children, nil + timer.Results(results) + return results, nil } -func FindConfigParentIDsByLocation(ctx context.Context, configID uuid.UUID, prefix string) ([]uuid.UUID, error) { - var parents []uuid.UUID - if err := ctx.DB().Raw(`SELECT id FROM get_parent_ids_by_location(?, ?)`, configID, prefix).Scan(&parents).Error; err != nil { +func FindConfigParentIDsByLocation(ctx context.Context, configID uuid.UUID, prefix string) (results []uuid.UUID, err error) { + timer := NewQueryLogger(ctx).Start("ConfigParentIDs").Arg("configID", configID).Arg("prefix", prefix) + defer timer.End(&err) + + if err = ctx.DB().Raw(`SELECT id FROM get_parent_ids_by_location(?, ?)`, configID, prefix).Scan(&results).Error; err != nil { return nil, err } - - return parents, nil + timer.Results(results) + return results, nil } type ConfigMinimal struct { @@ -498,20 +505,24 @@ type ConfigMinimal struct { Type string `json:"type"` } -func FindConfigChildrenByLocation(ctx context.Context, configID uuid.UUID, prefix string, includeDeleted bool) ([]ConfigMinimal, error) { - var children []ConfigMinimal - if err := ctx.DB().Raw(`SELECT id, name, type FROM get_children_by_location(?, ?, ?)`, configID, prefix, includeDeleted).Scan(&children).Error; err != nil { +func FindConfigChildrenByLocation(ctx context.Context, configID uuid.UUID, prefix string, includeDeleted bool) (results []ConfigMinimal, err error) { + timer := NewQueryLogger(ctx).Start("ConfigChildren").Arg("configID", configID).Arg("prefix", prefix) + defer timer.End(&err) + + if err = ctx.DB().Raw(`SELECT id, name, type FROM get_children_by_location(?, ?, ?)`, configID, prefix, includeDeleted).Scan(&results).Error; err != nil { return nil, err } - - return children, nil + timer.Results(results) + return results, nil } -func FindConfigParentsByLocation(ctx context.Context, configID uuid.UUID, prefix string, includeDeleted bool) ([]ConfigMinimal, error) { - var parents []ConfigMinimal - if err := ctx.DB().Raw(`SELECT id, name, type FROM get_parents_by_location(?, ?, ?)`, configID, prefix, includeDeleted).Scan(&parents).Error; err != nil { +func FindConfigParentsByLocation(ctx context.Context, configID uuid.UUID, prefix string, includeDeleted bool) (results []ConfigMinimal, err error) { + timer := NewQueryLogger(ctx).Start("ConfigParents").Arg("configID", configID).Arg("prefix", prefix) + defer timer.End(&err) + + if err = ctx.DB().Raw(`SELECT id, name, type FROM get_parents_by_location(?, ?, ?)`, configID, prefix, includeDeleted).Scan(&results).Error; err != nil { return nil, err } - - return parents, nil + timer.Results(results) + return results, nil } diff --git a/query/config_access.go b/query/config_access.go index f49223b70..d21399b86 100644 --- a/query/config_access.go +++ b/query/config_access.go @@ -1,18 +1,65 @@ package query import ( + "github.com/flanksource/duty/api" "github.com/flanksource/duty/context" "github.com/flanksource/duty/models" "github.com/google/uuid" ) -func FindConfigAccessByConfigIDs(ctx context.Context, configIDs []uuid.UUID) ([]models.ConfigAccessSummary, error) { - var configAccess []models.ConfigAccessSummary - if err := ctx.DB(). - Where("config_id IN (?)", configIDs). - Find(&configAccess).Error; err != nil { +type CatalogAccessSearchRequest struct { + BaseCatalogSearch `json:",inline"` +} + +type CatalogAccessSearchResponse struct { + Total int64 `json:"total"` + Access []models.ConfigAccessSummary `json:"access"` +} + +func FindCatalogAccess(ctx context.Context, req CatalogAccessSearchRequest) (results *CatalogAccessSearchResponse, err error) { + req.SetDefaults() + if err := req.Validate(); err != nil { + return nil, api.Errorf(api.EINVALID, "bad request: %v", err) + } + + timer := NewQueryLogger(ctx).Start("CatalogAccess") + defer timer.End(&err) + + configIDs, err := req.ResolveConfigIDs(ctx) + if err != nil { + return nil, err + } + if len(configIDs) == 0 && req.CatalogID != "" { + return &CatalogAccessSearchResponse{}, nil + } + + var output CatalogAccessSearchResponse + q := ctx.DB().Table("config_access_summary") + if len(configIDs) > 0 { + q = q.Where("config_id IN ?", configIDs) + } + + if err := q.Count(&output.Total).Error; err != nil { return nil, err } + if output.Total == 0 { + timer.Results(output.Access) + return &output, nil + } - return configAccess, nil + if err := q.Limit(req.PageSize).Offset((req.Page - 1) * req.PageSize).Find(&output.Access).Error; err != nil { + return nil, err + } + timer.Results(output.Access) + return &output, nil +} + +func FindConfigAccessByConfigIDs(ctx context.Context, configIDs []uuid.UUID) ([]models.ConfigAccessSummary, error) { + resp, err := FindCatalogAccess(ctx, CatalogAccessSearchRequest{ + BaseCatalogSearch: BaseCatalogSearch{configIDs: configIDs, PageSize: 10000}, + }) + if err != nil { + return nil, err + } + return resp.Access, nil } diff --git a/query/config_changes.go b/query/config_changes.go index cecd4f070..b9f58301b 100644 --- a/query/config_changes.go +++ b/query/config_changes.go @@ -8,7 +8,6 @@ import ( "github.com/flanksource/duty/api" "github.com/flanksource/duty/context" "github.com/flanksource/duty/models" - "github.com/flanksource/duty/pkg/kube/labels" "github.com/flanksource/duty/types" "github.com/google/uuid" "github.com/samber/lo" @@ -30,170 +29,71 @@ var allRecursiveOptions = []ChangeRelationDirection{CatalogChangeRecursiveUpstre var allowedConfigChangesSortColumns = []string{"name", "change_type", "summary", "source", "created_at", "count"} type CatalogChangesSearchRequest struct { - CatalogID string `query:"id" json:"id"` - ConfigType string `query:"config_type" json:"config_type"` - ChangeType string `query:"type" json:"type"` - Severity string `query:"severity" json:"severity"` - IncludeDeletedConfigs bool `query:"include_deleted_configs" json:"include_deleted_configs"` - Depth int `query:"depth" json:"depth"` - CreatedByRaw string `query:"created_by" json:"created_by"` - Summary string `query:"summary" json:"summary"` - Source string `query:"source" json:"source"` - Tags string `query:"tags" json:"tags"` - - // To Fetch from a particular agent, provide the agent id. - // Use `local` keyword to filter by the local agent. - AgentID string `query:"agent_id" json:"agent_id"` + BaseCatalogSearch `json:",inline"` + + ChangeType string `query:"type" json:"type"` + Severity string `query:"severity" json:"severity"` + CreatedByRaw string `query:"created_by" json:"created_by"` + Summary string `query:"summary" json:"summary"` + Source string `query:"source" json:"source"` createdBy *uuid.UUID externalCreatedBy string - // From date in datemath format - From string `query:"from" json:"from"` - // To date in datemath format - To string `query:"to" json:"to"` - // FromInsertedAt in datemath format FromInsertedAt string `query:"from_inserted_at" json:"from_inserted_at"` // ToInsertedAt in datemath format ToInsertedAt string `query:"to_inserted_at" json:"to_inserted_at"` - PageSize int `query:"page_size" json:"page_size"` - Page int `query:"page" json:"page"` - SortBy string `query:"sort_by" json:"sort_by"` - sortOrder string - - // upstream | downstream | both - Recursive ChangeRelationDirection `query:"recursive" json:"recursive"` - - // FIXME: Soft toggle does not work with Recursive=both - // In that case, soft relations are always returned - // It also returns ALL soft relations throughout the tree - // not just soft related to the main config item - Soft bool `query:"soft" json:"soft"` - - fromParsed time.Time - toParsed time.Time fromInsertedAtParsed time.Time toInsertedAtParsed time.Time } func (t CatalogChangesSearchRequest) String() string { - s := "" - if t.AgentID != "" { - s += fmt.Sprintf("agent: %s", t.AgentID) - } - if t.CatalogID != "" { - s += fmt.Sprintf("id: %s ", t.CatalogID) - } - if t.ConfigType != "" { - s += fmt.Sprintf("config_type: %s ", t.ConfigType) - } + s := t.BaseCatalogSearch.String() if t.ChangeType != "" { - s += fmt.Sprintf("type: %s ", t.ChangeType) + s += fmt.Sprintf(" type=%s", t.ChangeType) } if t.Severity != "" { - s += fmt.Sprintf("severity: %s ", t.Severity) + s += fmt.Sprintf(" severity=%s", t.Severity) } if t.Source != "" { - s += fmt.Sprintf("source: %s ", t.Source) - } - if t.IncludeDeletedConfigs { - s += fmt.Sprintf("include_deleted_configs: %t ", t.IncludeDeletedConfigs) - } - if t.Depth != 0 { - s += fmt.Sprintf("depth: %d ", t.Depth) + s += fmt.Sprintf(" source=%s", t.Source) } if t.CreatedByRaw != "" { - s += fmt.Sprintf("created_by: %s ", t.CreatedByRaw) + s += fmt.Sprintf(" created_by=%s", t.CreatedByRaw) } if t.Summary != "" { - s += fmt.Sprintf("summary: %s ", t.Summary) - } - if t.Tags != "" { - s += fmt.Sprintf("tags: %s ", t.Tags) - } - if t.From != "" { - s += fmt.Sprintf("from: %s ", t.From) - } - if t.To != "" { - s += fmt.Sprintf("to: %s ", t.To) - } - if t.FromInsertedAt != "" { - s += fmt.Sprintf("from_inserted_at: %s ", t.FromInsertedAt) - } - if t.ToInsertedAt != "" { - s += fmt.Sprintf("to_inserted_at: %s ", t.ToInsertedAt) - } - if t.PageSize != 0 { - s += fmt.Sprintf("page_size: %d ", t.PageSize) - } - if t.Page != 0 { - s += fmt.Sprintf("page: %d ", t.Page) - } - if t.SortBy != "" { - s += fmt.Sprintf("sort_by: %s %s ", t.SortBy, t.sortOrder) - } - if t.Recursive != "" { - s += fmt.Sprintf("recursive: %s ", t.Recursive) + s += fmt.Sprintf(" summary=%s", t.Summary) } return s } func (t *CatalogChangesSearchRequest) SetDefaults() { - if t.PageSize <= 0 { - t.PageSize = 50 - } - - if t.Page <= 0 { - t.Page = 1 - } - if t.From == "" && t.To == "" { t.From = "now-2d" } - - if t.Recursive == "" { - t.Recursive = CatalogChangeRecursiveDownstream - } - - if t.Depth <= 0 { - t.Depth = 5 - } - - if t.AgentID == "local" { - t.AgentID = uuid.Nil.String() - } + t.BaseCatalogSearch.SetDefaults() } func (t *CatalogChangesSearchRequest) Validate() error { - if !lo.Contains(allRecursiveOptions, t.Recursive) { - return fmt.Errorf("'recursive' must be one of %v", allRecursiveOptions) - } - - if t.From != "" { - if expr, err := datemath.Parse(t.From); err != nil { - return fmt.Errorf("invalid 'from' param: %w", err) - } else { - t.fromParsed = expr.Time() - } + if err := t.BaseCatalogSearch.Validate(); err != nil { + return err } - if t.To != "" { - if expr, err := datemath.Parse(t.To); err != nil { - return fmt.Errorf("invalid 'to' param: %w", err) - } else { - t.toParsed = expr.Time() + if !lo.Contains(allRecursiveOptions, t.Recursive) { + if !t.Lenient { + return fmt.Errorf("'recursive' must be one of %v", allRecursiveOptions) } - } - - if !t.fromParsed.IsZero() && !t.toParsed.IsZero() && !t.fromParsed.Before(t.toParsed) { - return fmt.Errorf("'from' must be before 'to'") + t.Recursive = CatalogChangeRecursiveDownstream } if t.FromInsertedAt != "" { if expr, err := datemath.Parse(t.FromInsertedAt); err != nil { - return fmt.Errorf("invalid 'from_inserted_at' param: %w", err) + if !t.Lenient { + return fmt.Errorf("invalid 'from_inserted_at' param: %w", err) + } + t.FromInsertedAt = "" } else { t.fromInsertedAtParsed = expr.Time() } @@ -201,14 +101,20 @@ func (t *CatalogChangesSearchRequest) Validate() error { if t.ToInsertedAt != "" { if expr, err := datemath.Parse(t.ToInsertedAt); err != nil { - return fmt.Errorf("invalid 'to_inserted_at' param: %w", err) + if !t.Lenient { + return fmt.Errorf("invalid 'to_inserted_at' param: %w", err) + } + t.ToInsertedAt = "" } else { t.toInsertedAtParsed = expr.Time() } } if !t.fromInsertedAtParsed.IsZero() && !t.toInsertedAtParsed.IsZero() && !t.fromInsertedAtParsed.Before(t.toInsertedAtParsed) { - return fmt.Errorf("'from_inserted_at' must be before 'to_inserted_at'") + if !t.Lenient { + return fmt.Errorf("'from_inserted_at' must be before 'to_inserted_at'") + } + t.toInsertedAtParsed = time.Time{} } if t.SortBy != "" { @@ -218,7 +124,10 @@ func (t *CatalogChangesSearchRequest) Validate() error { } if !lo.Contains(allowedConfigChangesSortColumns, t.SortBy) { - return fmt.Errorf("invalid 'sort_by' param: %s. allowed sort fields are: %s", t.SortBy, strings.Join(allowedConfigChangesSortColumns, ", ")) + if !t.Lenient { + return fmt.Errorf("invalid 'sort_by' param: %s. allowed sort fields are: %s", t.SortBy, strings.Join(allowedConfigChangesSortColumns, ", ")) + } + t.SortBy = "" } } @@ -230,12 +139,6 @@ func (t *CatalogChangesSearchRequest) Validate() error { } } - if t.AgentID != "" { - if _, err := uuid.Parse(t.AgentID); err != nil { - return fmt.Errorf("agent_id(%s) must either be a valid uuid or `local`", t.AgentID) - } - } - return nil } @@ -261,6 +164,10 @@ type ConfigChangeRow struct { InsertedAt *time.Time `gorm:"column:inserted_at" json:"inserted_at,omitempty"` } +func (r ConfigChangeRow) QueryLogSummary() string { + return r.ChangeType +} + type CatalogChangesSearchResponse struct { Summary map[string]int `json:"summary,omitempty"` Total int64 `json:"total,omitempty"` @@ -274,10 +181,9 @@ func (t *CatalogChangesSearchResponse) Summarize() { } } -func formSeverityQuery(severity string) string { +func formSeverityQuery(severity string) (string, error) { if strings.HasPrefix(severity, "!") { - // For `Not` queries, we don't need to make any changes. - return severity + return severity, nil } severities := []models.Severity{ @@ -292,89 +198,77 @@ func formSeverityQuery(severity string) string { for _, s := range severities { applicable = append(applicable, string(s)) if string(s) == severity { - break + return strings.Join(applicable, ","), nil } } - return strings.Join(applicable, ",") + return "", fmt.Errorf("unknown severity %q", severity) } -func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (*CatalogChangesSearchResponse, error) { +func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (result *CatalogChangesSearchResponse, err error) { req.SetDefaults() if err := req.Validate(); err != nil { return nil, api.Errorf(api.EINVALID, "bad request: %v", err) } - ctx.Tracef("query changes: %s", req) - var clauses []clause.Expression + timer := NewQueryLogger(ctx).Start("CatalogChanges").Arg("query", req.String()) + defer timer.End(&err) - query := ctx.DB() + configIDs, err := req.ResolveConfigIDs(ctx) + if err != nil { + return nil, err + } + if len(configIDs) == 0 && req.CatalogID != "" { + return &CatalogChangesSearchResponse{}, nil + } - if req.AgentID != "" { - clause, err := parseAndBuildFilteringQuery(req.AgentID, "agent_id", false) - if err != nil { - return nil, err - } - clauses = append(clauses, clause...) + baseClauses, tagsFn, err := req.ApplyClauses() + if err != nil { + return nil, api.Errorf(api.EINVALID, "bad request: %v", err) } + var clauses []clause.Expression + clauses = append(clauses, baseClauses...) - if req.ConfigType != "" { - clause, err := parseAndBuildFilteringQuery(req.ConfigType, "type", false) - if err != nil { - return nil, err - } - clauses = append(clauses, clause...) + dbQuery := ctx.DB() + if tagsFn != nil { + dbQuery = tagsFn(dbQuery) } if req.ChangeType != "" { - clause, err := parseAndBuildFilteringQuery(req.ChangeType, "change_type", false) - if err != nil { - return nil, err + if c, parseErr := parseAndBuildFilteringQuery(req.ChangeType, "change_type", false); parseErr != nil && !req.Lenient { + return nil, parseErr + } else if parseErr == nil { + clauses = append(clauses, c...) } - clauses = append(clauses, clause...) } if req.Severity != "" { - clause, err := parseAndBuildFilteringQuery(formSeverityQuery(req.Severity), "severity", false) - if err != nil { - return nil, api.Errorf(api.EINVALID, "failed to parse severity: %v", err) + severityQuery, err := formSeverityQuery(req.Severity) + if err != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "invalid severity: %v", err) + } else if err == nil { + if c, parseErr := parseAndBuildFilteringQuery(severityQuery, "severity", false); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse severity: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) + } } - clauses = append(clauses, clause...) } if req.Summary != "" { - clause, err := parseAndBuildFilteringQuery(req.Summary, "summary", true) - if err != nil { - return nil, api.Errorf(api.EINVALID, "failed to parse summary: %v", err) + if c, parseErr := parseAndBuildFilteringQuery(req.Summary, "summary", true); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse summary: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) } - clauses = append(clauses, clause...) } if req.Source != "" { - clause, err := parseAndBuildFilteringQuery(req.Source, "source", true) - if err != nil { - return nil, api.Errorf(api.EINVALID, "failed to parse source: %v", err) + if c, parseErr := parseAndBuildFilteringQuery(req.Source, "source", true); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse source: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) } - clauses = append(clauses, clause...) - } - - if req.Tags != "" { - parsedLabelSelector, err := labels.Parse(req.Tags) - if err != nil { - return nil, api.Errorf(api.EINVALID, "failed to parse label selector: %v", err) - } - requirements, _ := parsedLabelSelector.Requirements() - for _, r := range requirements { - query = jsonColumnRequirementsToSQLClause(query, "tags", r) - } - } - - if !req.fromParsed.IsZero() { - clauses = append(clauses, clause.Gte{Column: clause.Column{Name: "created_at"}, Value: req.fromParsed}) - } - - if !req.toParsed.IsZero() { - clauses = append(clauses, clause.Lte{Column: clause.Column{Name: "created_at"}, Value: req.toParsed}) } if !req.fromInsertedAtParsed.IsZero() { @@ -390,26 +284,26 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (* } if req.externalCreatedBy != "" { - clause, err := parseAndBuildFilteringQuery(req.externalCreatedBy, "external_created_by", true) - if err != nil { - return nil, api.Errorf(api.EINVALID, "failed to parse external createdby: %v", err) + if c, parseErr := parseAndBuildFilteringQuery(req.externalCreatedBy, "external_created_by", true); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse external createdby: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) } - clauses = append(clauses, clause...) - } - - if !req.IncludeDeletedConfigs { - clauses = append(clauses, clause.Eq{Column: clause.Column{Name: "deleted_at"}, Value: nil}) } - table := query.Table("catalog_changes") - if err := uuid.Validate(req.CatalogID); err == nil { - table = query.Table("related_changes_recursive(?,?,?,?,?)", req.CatalogID, req.Recursive, req.IncludeDeletedConfigs, req.Depth, req.Soft) - } else { - clause, err := parseAndBuildFilteringQuery(req.CatalogID, "config_id", false) - if err != nil { - return nil, err + // Determine table: single UUID uses related_changes_recursive, multi-ID or query uses IN clause + table := dbQuery.Table("catalog_changes") + if len(configIDs) == 1 { + table = dbQuery.Table("related_changes_recursive(?,?,?,?,?)", configIDs[0], req.Recursive, req.IncludeDeletedConfigs, req.Depth, req.Soft) + } else if len(configIDs) > 1 { + table = table.Where("config_id IN ?", configIDs) + } else if req.CatalogID != "" { + // Fallback: treat as filtering expression on config_id + if c, parseErr := parseAndBuildFilteringQuery(req.CatalogID, "config_id", false); parseErr == nil { + clauses = append(clauses, c...) + } else { + return nil, parseErr } - clauses = append(clauses, clause...) } var output CatalogChangesSearchResponse @@ -418,6 +312,7 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (* } if output.Total == 0 { + timer.Results(output.Changes) return &output, nil } @@ -440,6 +335,7 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (* } output.Summarize() + timer.Results(output.Changes) return &output, nil } diff --git a/query/config_insights.go b/query/config_insights.go new file mode 100644 index 000000000..8254327bb --- /dev/null +++ b/query/config_insights.go @@ -0,0 +1,117 @@ +package query + +import ( + "github.com/flanksource/duty/api" + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "gorm.io/gorm/clause" +) + +type CatalogInsightsSearchRequest struct { + BaseCatalogSearch `json:",inline"` + Status string `query:"status" json:"status"` + Severity string `query:"severity" json:"severity"` + Analyzer string `query:"analyzer" json:"analyzer"` + AnalysisType string `query:"analysis_type" json:"analysis_type"` +} + +func (r *CatalogInsightsSearchRequest) SetDefaults() { + r.BaseCatalogSearch.SetDefaults() + if r.Status == "" { + r.Status = "open" + } +} + +type CatalogInsightsSearchResponse struct { + Total int64 `json:"total"` + Insights []models.ConfigAnalysis `json:"insights"` +} + +func FindCatalogInsights(ctx context.Context, req CatalogInsightsSearchRequest) (results *CatalogInsightsSearchResponse, err error) { + req.SetDefaults() + if err := req.Validate(); err != nil { + return nil, api.Errorf(api.EINVALID, "bad request: %v", err) + } + + timer := NewQueryLogger(ctx).Start("CatalogInsights") + defer timer.End(&err) + + configIDs, err := req.ResolveConfigIDs(ctx) + if err != nil { + return nil, err + } + if len(configIDs) == 0 && req.CatalogID != "" { + return &CatalogInsightsSearchResponse{}, nil + } + + // config_analysis table doesn't have deleted_at, agent_id, type, or tags columns + baseClauses, tagsFn, err := req.ApplyClauses("deleted_at", "agent_id", "type", "tags") + if err != nil { + return nil, api.Errorf(api.EINVALID, "bad request: %v", err) + } + var clauses []clause.Expression + clauses = append(clauses, baseClauses...) + + q := ctx.DB().Table("config_analysis") + + if len(configIDs) > 0 { + q = q.Where("config_id IN ?", configIDs) + } + + if req.Status != "" { + if c, parseErr := parseAndBuildFilteringQuery(req.Status, "status", false); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse status: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) + } + } + if req.Severity != "" { + severityQuery, err := formSeverityQuery(req.Severity) + if err != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "invalid severity: %v", err) + } else if err == nil { + if c, parseErr := parseAndBuildFilteringQuery(severityQuery, "severity", false); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse severity: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) + } + } + } + if req.Analyzer != "" { + if c, parseErr := parseAndBuildFilteringQuery(req.Analyzer, "analyzer", true); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse analyzer: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) + } + } + if req.AnalysisType != "" { + if c, parseErr := parseAndBuildFilteringQuery(req.AnalysisType, "analysis_type", false); parseErr != nil && !req.Lenient { + return nil, api.Errorf(api.EINVALID, "failed to parse analysis_type: %v", parseErr) + } else if parseErr == nil { + clauses = append(clauses, c...) + } + } + + if tagsFn != nil { + q = tagsFn(q) + } + + var output CatalogInsightsSearchResponse + if err := q.Clauses(clauses...).Count(&output.Total).Error; err != nil { + return nil, err + } + if output.Total == 0 { + timer.Results(output.Insights) + return &output, nil + } + + clauses = append(clauses, + clause.Limit{Limit: &req.PageSize, Offset: (req.Page - 1) * req.PageSize}, + ) + + if err := q.Clauses(clauses...).Find(&output.Insights).Error; err != nil { + return nil, err + } + timer.Results(output.Insights) + return &output, nil +} diff --git a/query/config_relations.go b/query/config_relations.go index 3aee87c65..8d99f1467 100644 --- a/query/config_relations.go +++ b/query/config_relations.go @@ -11,6 +11,10 @@ import ( "github.com/samber/lo" ) +func (r RelatedConfig) QueryLogSummary() string { + return r.Type +} + type RelatedConfig struct { Relation string `json:"relation"` RelatedIDs pq.StringArray `json:"related_ids" gorm:"type:[]text"` @@ -72,8 +76,12 @@ const ( Soft RelationType = "soft" ) -func GetRelatedConfigs(ctx context.Context, query RelationQuery) ([]RelatedConfig, error) { - var relatedConfigs []RelatedConfig +func GetRelatedConfigs(ctx context.Context, query RelationQuery) (results []RelatedConfig, err error) { + timer := NewQueryLogger(ctx).Start("RelatedConfigs"). + Arg("id", query.ID).Arg("direction", query.Relation). + Arg("incoming", query.Incoming).Arg("outgoing", query.Outgoing) + defer timer.End(&err) + if query.MaxDepth == nil { query.MaxDepth = lo.ToPtr(5) } @@ -86,15 +94,17 @@ func GetRelatedConfigs(ctx context.Context, query RelationQuery) ([]RelatedConfi // FIXME: Self config is returned here for creating graph in UI. We need to update UI to // add the node itself. Issue: github.com/flanksource/duty/issues/1722 - err := ctx.DB().Raw("SELECT * FROM related_configs_recursive(?, ?, ?, ?, ?, ?)", + if err = ctx.DB().Raw("SELECT * FROM related_configs_recursive(?, ?, ?, ?, ?, ?)", query.ID, query.Relation, query.IncludeDeleted, *query.MaxDepth, query.Incoming, - query.Outgoing).Find(&relatedConfigs).Error - - relatedConfigs = lo.Filter(relatedConfigs, func(c RelatedConfig, _ int) bool { return c.ID != query.ID }) + query.Outgoing).Find(&results).Error; err != nil { + return nil, err + } - return relatedConfigs, err + results = lo.Filter(results, func(c RelatedConfig, _ int) bool { return c.ID != query.ID }) + timer.Results(results) + return results, nil } diff --git a/query/config_tree.go b/query/config_tree.go new file mode 100644 index 000000000..d371a19e0 --- /dev/null +++ b/query/config_tree.go @@ -0,0 +1,271 @@ +package query + +import ( + "fmt" + "strings" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "github.com/google/uuid" + "github.com/samber/lo" +) + +type ConfigTreeNode struct { + models.ConfigItem `json:",inline"` + EdgeType string `json:"edgeType,omitempty"` + Relation string `json:"relation,omitempty"` + Children []*ConfigTreeNode `json:"children,omitempty"` +} + +type ConfigTreeOptions struct { + Direction RelationDirection + Incoming RelationType + Outgoing RelationType +} + +func (n *ConfigTreeNode) OutgoingIDs() []uuid.UUID { + var ids []uuid.UUID + n.collectOutgoing(&ids, make(map[uuid.UUID]bool)) + return ids +} + +func (n *ConfigTreeNode) collectOutgoing(ids *[]uuid.UUID, seen map[uuid.UUID]bool) { + if seen[n.ID] { + return + } + seen[n.ID] = true + if n.EdgeType != "parent" { + *ids = append(*ids, n.ID) + } + for _, c := range n.Children { + c.collectOutgoing(ids, seen) + } +} + +func ConfigTree(ctx context.Context, configID uuid.UUID, opts ConfigTreeOptions) (*ConfigTreeNode, error) { + config, err := GetCachedConfig(ctx, configID.String()) + if err != nil { + return nil, err + } + if config == nil { + return nil, nil + } + + parents, err := resolveParentsFromPath(ctx, config) + if err != nil { + return nil, err + } + + childIDs, err := ExpandConfigChildren(ctx, []uuid.UUID{config.ID}) + if err != nil { + return nil, err + } + childIDs = lo.Filter(childIDs, func(id uuid.UUID, _ int) bool { return id != config.ID }) + + var children []models.ConfigItem + if len(childIDs) > 0 { + children, err = GetConfigsByIDs(ctx, childIDs) + if err != nil { + return nil, err + } + } + + if opts.Direction == "" { + opts.Direction = All + } + relType := Hard + if opts.Incoming != "" { + relType = opts.Incoming + } + outType := Both + if opts.Outgoing != "" { + outType = opts.Outgoing + } + + related, err := GetRelatedConfigs(ctx, RelationQuery{ + ID: config.ID, + Relation: opts.Direction, + Incoming: relType, + Outgoing: outType, + }) + if err != nil { + return nil, err + } + + return buildConfigTree(config, parents, children, related), nil +} + +func resolveParentsFromPath(ctx context.Context, config *models.ConfigItem) ([]models.ConfigItem, error) { + if config.Path == "" { + return nil, nil + } + segments := strings.Split(config.Path, ".") + var parentIDs []uuid.UUID + for _, seg := range segments { + id, err := uuid.Parse(seg) + if err != nil || id == config.ID { + continue + } + parentIDs = append(parentIDs, id) + } + if len(parentIDs) == 0 { + return nil, nil + } + items, err := GetConfigsByIDs(ctx, parentIDs) + if err != nil { + return nil, fmt.Errorf("resolving parents from path: %w", err) + } + if len(items) == 0 { + return nil, nil + } + byID := make(map[uuid.UUID]models.ConfigItem, len(items)) + for _, ci := range items { + byID[ci.ID] = ci + } + var parents []models.ConfigItem + for _, id := range parentIDs { + if ci, ok := byID[id]; ok { + parents = append(parents, ci) + } + } + return parents, nil +} + +func ExpandConfigChildren(ctx context.Context, ids []uuid.UUID) ([]uuid.UUID, error) { + allIDs := make(map[uuid.UUID]struct{}, len(ids)) + for _, id := range ids { + allIDs[id] = struct{}{} + } + for _, id := range ids { + var children []uuid.UUID + if err := ctx.DB().Raw("SELECT child_id FROM lookup_config_children(?, -1)", id.String()). + Scan(&children).Error; err != nil { + return nil, err + } + for _, child := range children { + allIDs[child] = struct{}{} + } + } + return lo.Keys(allIDs), nil +} + +type ptrNode struct { + models.ConfigItem + edgeType string + relation string + children []*ptrNode +} + +func buildConfigTree(config *models.ConfigItem, parents []models.ConfigItem, children []models.ConfigItem, related []RelatedConfig) *ConfigTreeNode { + nodes := make(map[uuid.UUID]*ptrNode) + + for _, p := range parents { + nodes[p.ID] = &ptrNode{ConfigItem: p, edgeType: "parent"} + } + + targetNode := &ptrNode{ConfigItem: *config, edgeType: "target"} + nodes[config.ID] = targetNode + + if len(parents) > 0 { + for i := 0; i < len(parents)-1; i++ { + nodes[parents[i].ID].children = append(nodes[parents[i].ID].children, nodes[parents[i+1].ID]) + } + nodes[parents[len(parents)-1].ID].children = append(nodes[parents[len(parents)-1].ID].children, targetNode) + } + + for _, c := range children { + nodes[c.ID] = &ptrNode{ConfigItem: c, edgeType: "child"} + } + for _, c := range children { + parentID := lo.FromPtr(c.ParentID) + if parent, ok := nodes[parentID]; ok { + parent.children = append(parent.children, nodes[c.ID]) + } else { + targetNode.children = append(targetNode.children, nodes[c.ID]) + } + } + + parentIDs := make(map[uuid.UUID]bool, len(parents)) + for _, p := range parents { + parentIDs[p.ID] = true + } + + for _, rc := range related { + if parentIDs[rc.ID] || rc.ID == config.ID { + continue + } + if _, exists := nodes[rc.ID]; !exists { + nodes[rc.ID] = &ptrNode{ + ConfigItem: relatedToConfigItem(rc), + edgeType: "related", + relation: rc.Relation, + } + } + } + + wired := make(map[uuid.UUID]bool) + for _, rc := range related { + if parentIDs[rc.ID] || rc.ID == config.ID || wired[rc.ID] { + continue + } + wired[rc.ID] = true + node := nodes[rc.ID] + if rc.Path != "" { + segments := strings.Split(rc.Path, ".") + // Last segment is the node's own ID (SetParent appends ci.ID), so use penultimate + if len(segments) >= 2 { + if parentStr := segments[len(segments)-2]; parentStr != "" { + if pid, err := uuid.Parse(parentStr); err == nil { + if parent, ok := nodes[pid]; ok && parent != node && !parentIDs[pid] { + parent.children = append(parent.children, node) + continue + } + } + } + } + } + targetNode.children = append(targetNode.children, node) + } + + var root *ptrNode + if len(parents) > 0 { + root = nodes[parents[0].ID] + } else { + root = targetNode + } + + return toConfigTreeNode(root, make(map[*ptrNode]bool)) +} + +func toConfigTreeNode(n *ptrNode, visited map[*ptrNode]bool) *ConfigTreeNode { + result := &ConfigTreeNode{ + ConfigItem: n.ConfigItem, + EdgeType: n.edgeType, + Relation: n.relation, + } + if visited[n] { + return result + } + visited[n] = true + for _, c := range n.children { + result.Children = append(result.Children, toConfigTreeNode(c, visited)) + } + return result +} + +func relatedToConfigItem(rc RelatedConfig) models.ConfigItem { + ci := models.ConfigItem{ + ID: rc.ID, + } + ci.Name = &rc.Name + ci.Type = &rc.Type + ci.Tags = rc.Tags + ci.Status = rc.Status + ci.Health = rc.Health + ci.Path = rc.Path + ci.CreatedAt = rc.CreatedAt + ci.UpdatedAt = &rc.UpdatedAt + ci.DeletedAt = rc.DeletedAt + ci.AgentID = rc.AgentID + return ci +} diff --git a/query/query_logger.go b/query/query_logger.go new file mode 100644 index 000000000..a30ba3591 --- /dev/null +++ b/query/query_logger.go @@ -0,0 +1,152 @@ +package query + +import ( + "fmt" + "reflect" + "strings" + "time" + + clickyapi "github.com/flanksource/clicky/api" + "github.com/flanksource/commons/logger" + "github.com/flanksource/duty/context" +) + +type QueryLogger struct { + logger logger.Verbose +} + +type QueryTimer struct { + logger logger.Verbose + label clickyapi.Text + start time.Time + results any + ended bool +} + +func NewQueryLogger(ctx context.Context) QueryLogger { + l := ctx.Logger.V(3) + if ctx.Properties().On(false, "query.log") { + l = ctx.Logger.V(0) + } + return QueryLogger{logger: l} +} + +func (q QueryLogger) Start(entity string) *QueryTimer { + return &QueryTimer{ + logger: q.logger, + label: clickyapi.Text{Content: "[" + entity + "]", Style: "text-blue-600 font-bold"}, + start: time.Now(), + } +} + +func (t *QueryTimer) Arg(key string, value any) *QueryTimer { + s := fmt.Sprintf("%v", value) + if len(s) > 80 { + s = s[:77] + "..." + } + t.label = t.label.AddText(fmt.Sprintf(" %s=", key), "text-gray-500"). + AddText(s) + return t +} + +func (t *QueryTimer) Results(results any) *QueryTimer { + t.results = results + return t +} + +func (t *QueryTimer) End(err *error) { + if t.ended { + return + } + t.ended = true + if !t.logger.Enabled() { + return + } + + elapsed := time.Since(t.start) + label := t.label.AddText(" => ", "text-gray-400") + + if err != nil && *err != nil { + label = label.AddText(fmt.Sprintf("error: %v", *err), "text-red-600") + } else if t.results != nil { + count := sliceLen(t.results) + countStyle := "text-green-600" + if count == 0 { + countStyle = "text-red-600" + } + label = label.AddText(fmt.Sprintf("%d", count), countStyle) + label = label.AddText(summaryText(t.results, count), "text-gray-400") + } else { + label = label.AddText("timed out", "text-yellow-600") + } + + label = label.AddText(fmt.Sprintf(" in %dms", elapsed.Milliseconds()), "text-gray-400") + t.logger.Infof("%s", label.ANSI()) +} + +func sliceLen(v any) int { + rv := reflect.ValueOf(v) + if rv.Kind() == reflect.Slice { + return rv.Len() + } + if rv.Kind() == reflect.Ptr && !rv.IsNil() { + return 1 + } + return 0 +} + +func summaryText(v any, count int) string { + if count == 0 { + return "" + } + + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Slice { + return "" + } + + // Try grouped summary first (e.g. "CodeDeployment: 5, BackupCompleted: 3") + if grouped := groupedSummary(rv); grouped != "" { + return " [" + grouped + "]" + } + + const maxInline = 2 + shown := count + if shown > maxInline { + shown = maxInline + } + + var parts []string + for i := 0; i < shown; i++ { + parts = append(parts, itemLogSummary(rv.Index(i).Interface())) + } + summary := strings.Join(parts, ", ") + if count > maxInline { + summary += fmt.Sprintf(", ...%d more", count-maxInline) + } + return " [" + summary + "]" +} + +func groupedSummary(rv reflect.Value) string { + if rv.Len() == 0 { + return "" + } + first := rv.Index(0).Interface() + if _, ok := first.(QueryLogSummary); !ok { + return "" + } + counts := make(map[string]int) + var order []string + for i := 0; i < rv.Len(); i++ { + s := rv.Index(i).Interface().(QueryLogSummary).QueryLogSummary() + if counts[s] == 0 { + order = append(order, s) + } + counts[s]++ + } + var parts []string + for _, key := range order { + parts = append(parts, fmt.Sprintf("%s: %d", key, counts[key])) + } + return strings.Join(parts, ", ") +} diff --git a/rbac/objects.go b/rbac/objects.go index a8469329f..dc45de7f8 100644 --- a/rbac/objects.go +++ b/rbac/objects.go @@ -24,6 +24,7 @@ var dbResourceObjMap = map[string]string{ "analysis_types": policy.ObjectDatabasePublic, "analyzer_types": policy.ObjectDatabasePublic, "artifacts": policy.ObjectArtifact, + "artifact_summary": policy.ObjectArtifact, "canaries_with_status": policy.ObjectCanary, "canaries": policy.ObjectCanary, "casbin_rule": policy.ObjectAuth, diff --git a/tests/config_changes_test.go b/tests/config_changes_test.go index c776165e1..4b98b2ca7 100644 --- a/tests/config_changes_test.go +++ b/tests/config_changes_test.go @@ -113,9 +113,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { var findChanges = func(id uuid.UUID, filter query.ChangeRelationDirection, deleted bool) (*query.CatalogChangesSearchResponse, error) { return query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: id.String(), - IncludeDeletedConfigs: deleted, - Recursive: filter, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: id.String(), + IncludeDeletedConfigs: deleted, + Recursive: filter, + }, }) } @@ -144,9 +146,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should return changes of a root node along with soft", func() { relatedChanges, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: "downstream", - Soft: true, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: "downstream", + Soft: true, + }, }) Expect(err).To(BeNil()) @@ -168,9 +172,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should return changes of a leaf node along with soft", func() { relatedChanges, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: X.ID.String(), - Recursive: "downstream", - Soft: true, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: X.ID.String(), + Recursive: "downstream", + Soft: true, + }, }) Expect(err).To(BeNil()) @@ -194,9 +200,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should return changes of a leaf node along with soft", func() { relatedChanges, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: "upstream", - Soft: true, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: "upstream", + Soft: true, + }, }) Expect(err).To(BeNil()) @@ -218,9 +226,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should return changes of a leaf node along with soft", func() { relatedChanges, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: X.ID.String(), - Recursive: "upstream", - Soft: true, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: X.ID.String(), + Recursive: "upstream", + Soft: true, + }, }) Expect(err).To(BeNil()) @@ -235,7 +245,9 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("FindCatalogChanges func", func() { ginkgo.It("Without catalog id", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - ConfigType: "Kubernetes::Pod,Kubernetes::ReplicaSet", + BaseCatalogSearch: query.BaseCatalogSearch{ + ConfigType: "Kubernetes::Pod,Kubernetes::ReplicaSet", + }, ChangeType: "!NotificationSent", }) Expect(err).To(BeNil()) @@ -250,9 +262,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("Config type filter", func() { ginkgo.It("IN", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - ConfigType: "Kubernetes::Pod,Kubernetes::ReplicaSet", + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + ConfigType: "Kubernetes::Pod,Kubernetes::ReplicaSet", + }, }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(3))) @@ -263,9 +277,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("NOT IN", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - ConfigType: "!Kubernetes::ReplicaSet", + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + ConfigType: "!Kubernetes::ReplicaSet", + }, }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(5))) @@ -279,8 +295,10 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("Change type filter", func() { ginkgo.It("IN", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: X.ID.String(), - Recursive: query.CatalogChangeRecursiveAll, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: X.ID.String(), + Recursive: query.CatalogChangeRecursiveAll, + }, ChangeType: "diff", }) Expect(err).To(BeNil()) @@ -291,8 +309,10 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("NOT IN", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + }, ChangeType: "!diff,!Pulled", }) Expect(err).To(BeNil()) @@ -305,9 +325,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("Severity filter", func() { ginkgo.It("NOT", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - Severity: "!info", + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + }, + Severity: "!info", }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(3))) @@ -318,9 +340,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should return the given severity and higher", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - Severity: string(models.SeverityMedium), + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + }, + Severity: string(models.SeverityMedium), }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(2))) @@ -333,10 +357,12 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("Pagination", func() { ginkgo.It("Page size", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - SortBy: "summary", - PageSize: 2, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + SortBy: "summary", + PageSize: 2, + }, }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(6))) @@ -347,11 +373,13 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("Page number", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - SortBy: "summary", - PageSize: 2, - Page: 2, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + SortBy: "summary", + PageSize: 2, + Page: 2, + }, }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(6))) @@ -364,8 +392,10 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("recursive mode", func() { ginkgo.It("upstream", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: W.ID.String(), - Recursive: query.CatalogChangeRecursiveUpstream, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: W.ID.String(), + Recursive: query.CatalogChangeRecursiveUpstream, + }, }) Expect(err).To(BeNil()) Expect(len(response.Changes)).To(Equal(2)) @@ -376,8 +406,10 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It(string(query.CatalogChangeRecursiveDownstream), func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: V.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: V.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + }, }) Expect(err).To(BeNil()) Expect(len(response.Changes)).To(Equal(4)) @@ -388,8 +420,10 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It(string(query.CatalogChangeRecursiveAll), func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: V.ID.String(), - Recursive: query.CatalogChangeRecursiveAll, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: V.ID.String(), + Recursive: query.CatalogChangeRecursiveAll, + }, }) Expect(err).To(BeNil()) Expect(len(response.Changes)).To(Equal(8)) @@ -402,10 +436,12 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should handle datemath", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - From: "now-65m", - To: "now-1s", + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + From: "now-65m", + To: "now-1s", + }, }) Expect(err).To(BeNil()) Expect(response.Total).To(BeNumerically(">=", 1)) @@ -413,8 +449,10 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("should filter by inserted_at", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + }, FromInsertedAt: "now-1h", }) Expect(err).To(BeNil()) @@ -428,9 +466,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.Context("Sorting", func() { ginkgo.It("Descending", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - SortBy: "-name", + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + SortBy: "-name", + }, }) Expect(err).To(BeNil()) Expect(len(response.Changes)).To(Equal(6)) @@ -441,9 +481,11 @@ var _ = ginkgo.Describe("Config changes recursive", ginkgo.Ordered, func() { ginkgo.It("Ascending", func() { response, err := query.FindCatalogChanges(DefaultContext, query.CatalogChangesSearchRequest{ - CatalogID: U.ID.String(), - Recursive: query.CatalogChangeRecursiveDownstream, - SortBy: "name", + BaseCatalogSearch: query.BaseCatalogSearch{ + CatalogID: U.ID.String(), + Recursive: query.CatalogChangeRecursiveDownstream, + SortBy: "name", + }, }) Expect(err).To(BeNil()) Expect(response.Total).To(Equal(int64(6))) diff --git a/tests/e2e-blobs/test.properties b/tests/e2e-blobs/test.properties new file mode 100644 index 000000000..4cff42f1d --- /dev/null +++ b/tests/e2e-blobs/test.properties @@ -0,0 +1,2 @@ +log.level.migrate=info +log.level.blobs=trace3 diff --git a/tests/setup/template.go b/tests/setup/template.go new file mode 100644 index 000000000..b97d7127f --- /dev/null +++ b/tests/setup/template.go @@ -0,0 +1,246 @@ +package setup + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + + embeddedPG "github.com/fergusstrange/embedded-postgres" + "github.com/flanksource/commons/logger" + "github.com/flanksource/commons/properties" + "github.com/flanksource/duty" + "github.com/flanksource/duty/context" + dutyKubernetes "github.com/flanksource/duty/kubernetes" + "github.com/flanksource/duty/shutdown" + "github.com/flanksource/duty/telemetry" + "github.com/flanksource/duty/tests/fixtures/dummy" + "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +type SetupOpts struct { + DummyData bool +} + +type templateInfo struct { + AdminURL string `json:"admin_url"` + TemplateDB string `json:"template_db"` + Port int `json:"port"` +} + +func (t templateInfo) Marshal() []byte { + data, err := json.Marshal(t) + if err != nil { + panic(fmt.Sprintf("failed to marshal templateInfo: %v", err)) + } + return data +} + +func unmarshalTemplateInfo(data []byte) templateInfo { + var info templateInfo + if err := json.Unmarshal(data, &info); err != nil { + panic(fmt.Sprintf("failed to unmarshal templateInfo: %v", err)) + } + return info +} + +var ( + adminURL string + nodeDBName string +) + +func SetupTemplate(opts SetupOpts) []byte { + if err := properties.LoadFile(findFileInPath("test.properties", 2)); err != nil { + logger.Errorf("Failed to load test properties: %v", err) + } + + defer telemetry.InitTracer() + + var port int + if val, ok := os.LookupEnv(TEST_DB_PORT); ok { + parsed, err := strconv.ParseInt(val, 10, 32) + if err != nil { + panic(fmt.Sprintf("failed to parse TEST_DB_PORT: %v", err)) + } + port = int(parsed) + } else { + port = duty.FreePort() + } + + templateDB := "duty_test_template" + + url := os.Getenv(DUTY_DB_URL) + if url != "" && !recreateDatabase { + // DUTY_DB_CREATE=false: use direct connection, no template + PgUrl = url + return templateInfo{AdminURL: url, TemplateDB: "", Port: port}.Marshal() + } + + adminConn, err := ensurePostgres(port) + if err != nil { + panic(fmt.Sprintf("failed to start postgres: %v", err)) + } + adminURL = adminConn + + // Always recreate — dummy data uses uuid.New() so a cached template has stale UUIDs + _ = execPostgres(adminConn, fmt.Sprintf("ALTER DATABASE %s WITH is_template = false", templateDB)) + _ = execPostgres(adminConn, fmt.Sprintf( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '%s' AND pid <> pg_backend_pid()", templateDB)) + _ = execPostgres(adminConn, fmt.Sprintf("DROP DATABASE IF EXISTS %s (FORCE)", templateDB)) + + if err := execPostgres(adminConn, fmt.Sprintf("CREATE DATABASE %s", templateDB)); err != nil { + panic(fmt.Sprintf("failed to create template db: %v", err)) + } + + templateURL := strings.Replace(adminConn, "/postgres", "/"+templateDB, 1) + if !strings.Contains(adminConn, "/postgres") { + templateURL = fmt.Sprintf("postgres://postgres:postgres@localhost:%d/%s?sslmode=disable", port, templateDB) + } + + dbOptions := []duty.StartOption{duty.DisablePostgrest, duty.RunMigrations, duty.WithUrl(templateURL)} + if !disableRLS { + dbOptions = append(dbOptions, duty.EnableRLS) + } + + ctx, stop, err := duty.Start(templateDB, dbOptions...) + if err != nil { + panic(fmt.Sprintf("failed to start duty for template: %v", err)) + } + + if err := ctx.DB().Exec("SET TIME ZONE 'UTC'").Error; err != nil { + panic(fmt.Sprintf("failed to set timezone: %v", err)) + } + + if opts.DummyData { + dummyData = dummy.GetStaticDummyData(ctx.DB()) + if err := dummyData.Delete(ctx.DB()); err != nil { + logger.Errorf(err.Error()) + } + if err := dummyData.Populate(ctx); err != nil { + panic(fmt.Sprintf("failed to populate dummy data: %v", err)) + } + logger.Infof("Created dummy data in template (%d checks)", len(dummyData.Checks)) + } + + // Close all connections so the DB can be used as a template + stop() + _ = execPostgres(adminConn, fmt.Sprintf( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '%s' AND pid <> pg_backend_pid()", templateDB)) + + if err := execPostgres(adminConn, fmt.Sprintf("ALTER DATABASE %s WITH is_template = true", templateDB)); err != nil { + panic(fmt.Sprintf("failed to mark template db: %v", err)) + } + + return templateInfo{AdminURL: adminConn, TemplateDB: templateDB, Port: port}.Marshal() +} + +func SetupNode(data []byte, opts SetupOpts) context.Context { + info := unmarshalTemplateInfo(data) + + if info.TemplateDB == "" { + // Direct connection mode (DUTY_DB_CREATE=false) + PgUrl = info.AdminURL + ctx, _, err := duty.Start("direct", duty.ClientOnly, duty.WithUrl(PgUrl)) + if err != nil { + panic(fmt.Sprintf("failed to connect to db: %v", err)) + } + return setupNodeContext(ctx, "direct") + } + + adminURL = info.AdminURL + nodeDBName = fmt.Sprintf("duty_test_node%d", ginkgo.GinkgoParallelProcess()) + + // Drop and clone from template + _ = execPostgres(adminURL, fmt.Sprintf("DROP DATABASE IF EXISTS %s (FORCE)", nodeDBName)) + + // Terminate any lingering connections to the template before cloning + _ = execPostgres(adminURL, fmt.Sprintf( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '%s' AND pid <> pg_backend_pid()", info.TemplateDB)) + + // Unmark template temporarily for cloning (some pg versions need this) + _ = execPostgres(adminURL, fmt.Sprintf("ALTER DATABASE %s WITH is_template = false", info.TemplateDB)) + if err := execPostgres(adminURL, fmt.Sprintf("CREATE DATABASE %s TEMPLATE %s", nodeDBName, info.TemplateDB)); err != nil { + panic(fmt.Sprintf("failed to clone template: %v", err)) + } + _ = execPostgres(adminURL, fmt.Sprintf("ALTER DATABASE %s WITH is_template = true", info.TemplateDB)) + + // Build node connection URL + if strings.Contains(adminURL, "/postgres") { + PgUrl = strings.Replace(adminURL, "/postgres", "/"+nodeDBName, 1) + } else { + PgUrl = fmt.Sprintf("postgres://postgres:postgres@localhost:%d/%s?sslmode=disable", info.Port, nodeDBName) + } + + // Skip migrations — the clone is byte-for-byte identical to the template + ctx, _, err := duty.Start(nodeDBName, duty.ClientOnly, duty.WithUrl(PgUrl)) + if err != nil { + panic(fmt.Sprintf("failed to connect to node db: %v", err)) + } + + return setupNodeContext(ctx, nodeDBName) +} + +func setupNodeContext(ctx context.Context, dbName string) context.Context { + if err := ctx.DB().Exec("SET TIME ZONE 'UTC'").Error; err != nil { + panic(fmt.Sprintf("failed to set timezone: %v", err)) + } + + ctx = ctx.WithValue("db_name", dbName).WithValue("db_url", PgUrl) + + clientset := fake.NewClientset(&v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cm", Namespace: "default"}, + Data: map[string]string{"foo": "bar"}, + }, &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-secret", Namespace: "default"}, + Data: map[string][]byte{"foo": []byte("secret")}, + }) + + return ctx.WithLocalKubernetes(dutyKubernetes.NewKubeClient(logger.GetLogger("k8s"), clientset, nil)) +} + +func SynchronizedAfterSuiteAllNodes() { + if nodeDBName != "" && adminURL != "" { + if err := execPostgres(adminURL, fmt.Sprintf("DROP DATABASE IF EXISTS %s (FORCE)", nodeDBName)); err != nil { + logger.Errorf("failed to drop node db: %v", err) + } + } +} + +func SynchronizedAfterSuiteNode1() { + shutdown.Shutdown() +} + + +func ensurePostgres(port int) (string, error) { + url := os.Getenv(DUTY_DB_URL) + if url != "" { + postgresDBUrl = url + return url, nil + } + + if postgresServer == nil { + config, _ := GetEmbeddedPGConfig("postgres", port) + + if v, ok := os.LookupEnv(DUTY_DB_DATA_DIR); ok { + config = config.DataPath(v) + } + + postgresServer = embeddedPG.NewDatabase(config) + logger.Infof("starting embedded postgres on port %d", port) + if err := postgresServer.Start(); err != nil { + return "", err + } + shutdown.AddHookWithPriority("stop embedded postgres", shutdown.PriorityCritical, func() { + if err := postgresServer.Stop(); err != nil { + logger.Errorf("failed to stop embedded postgres: %v", err) + } + }) + logger.Infof("Started postgres on port %d", port) + } + + return fmt.Sprintf("postgres://postgres:postgres@localhost:%d/postgres?sslmode=disable", port), nil +} diff --git a/views/046_artifact_summary.sql b/views/046_artifact_summary.sql new file mode 100644 index 000000000..d04afd5bf --- /dev/null +++ b/views/046_artifact_summary.sql @@ -0,0 +1,18 @@ +CREATE OR REPLACE VIEW artifact_summary AS +SELECT + content_type, + CASE WHEN a.content IS NOT NULL THEN 'inline' ELSE 'external' END AS storage, + a.connection_id, + c.name AS connection_name, + c.type AS connection_type, + COUNT(*) AS total_count, + COALESCE(SUM(a.size), 0) AS total_size +FROM artifacts a +LEFT JOIN connections c ON a.connection_id = c.id +WHERE a.deleted_at IS NULL +GROUP BY + content_type, + (CASE WHEN a.content IS NOT NULL THEN 'inline' ELSE 'external' END), + a.connection_id, + c.name, + c.type;