Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 126 additions & 1 deletion internal/pkg/project/ignore/ignore.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,18 @@ package ignore
import (
"strings"

"github.com/keboola/keboola-as-code/internal/pkg/model"
"github.com/keboola/keboola-as-code/internal/pkg/utils/errors"
"github.com/keboola/keboola-sdk-go/v2/pkg/keboola"
)

// IgnoreConfigsOrRows applies ignore patterns and validates that no orchestrators reference ignored configs.
// Returns an error listing orchestrators that need to be explicitly added to .kbcignore.
func (f *File) IgnoreConfigsOrRows() error {
return f.applyIgnoredPatterns()
if err := f.applyIgnoredPatterns(); err != nil {
return err
}
return f.validateOrchestratorDependencies()
}

// applyIgnoredPatterns parses the content for ignore patterns and applies them to configurations or rows.
Expand Down Expand Up @@ -53,3 +60,121 @@ func (f *File) applyIgnorePattern(ignoreConfig string) error {

return nil
}

// validateOrchestratorDependencies checks if any non-ignored configs with reverse dependencies
// (orchestrators, schedulers, input mappings, etc.) reference ignored configs.
// Returns an error instructing the user to explicitly add those dependent configs to .kbcignore.
func (f *File) validateOrchestratorDependencies() error {
errs := errors.NewMultiError()

// Build map of ignored config IDs for quick lookup
ignoredConfigs := make(map[string]bool)
for _, cfg := range f.state.IgnoredConfigs() {
ignoredConfigs[cfg.ID.String()] = true
}

// Relation types where the "owner" config (the one with the relation) depends on the "target" config
// If target is ignored, owner should also be ignored
reverseDependencyRelations := []model.RelationType{
model.UsedInOrchestratorRelType, // Orchestrator tasks reference other configs
model.UsedInConfigInputMappingRelType, // Config input mappings reference source configs
model.UsedInRowInputMappingRelType, // Row input mappings reference source configs
model.SchedulerForRelType, // Schedulers reference target configs to run
}

// Check each config for reverse dependency relations
for _, targetCfg := range f.state.Configs() {
// Skip if this config is not ignored (only care about ignored configs being referenced)
if !ignoredConfigs[targetCfg.ID.String()] {
continue
}

// Check each reverse dependency relation type
for _, relType := range reverseDependencyRelations {
rels := targetCfg.Relations.GetByType(relType)
for _, r := range rels {
// Extract the dependent config ID from the relation
dependentConfigID := f.extractDependentConfigID(r)
if dependentConfigID == "" {
continue
}

// Find the dependent config in the registry
dependentCfg := f.findConfigByID(targetCfg.BranchID, dependentConfigID)
if dependentCfg == nil {
continue
}

// If dependent config is NOT ignored but references an ignored config, report error
if !dependentCfg.Ignore {
dependentPath := dependentCfg.Path()
if dependentPath == "" {
dependentPath = dependentCfg.ConfigKey.Desc()
}
targetPath := targetCfg.Path()
if targetPath == "" {
targetPath = targetCfg.ConfigKey.Desc()
}

relTypeDesc := f.getRelationTypeDescription(relType)
errs.Append(errors.Errorf(
"%s %q references ignored config %q, please add it to .kbcignore:\n %s/%s",
relTypeDesc,
dependentPath,
targetPath,
dependentCfg.ComponentID,
dependentCfg.ID,
))
}
}
}
}

if errs.Len() > 0 {
return errors.PrefixError(errs, "configurations with dependencies reference ignored configurations")
}
return nil
}

// extractDependentConfigID extracts the config ID of the dependent config from a relation.
// Returns empty string if the relation doesn't contain a config ID reference.
func (f *File) extractDependentConfigID(rel model.Relation) string {
switch r := rel.(type) {
case *model.UsedInOrchestratorRelation:
return r.ConfigID.String()
case *model.SchedulerForRelation:
return r.ConfigID.String()
case *model.UsedInConfigInputMappingRelation:
return r.UsedIn.ID.String()
case *model.UsedInRowInputMappingRelation:
return r.UsedIn.ConfigID.String()
default:
return ""
}
}

// findConfigByID finds a config by ID within the same branch.
func (f *File) findConfigByID(branchID keboola.BranchID, configID string) *model.ConfigState {
for _, cfg := range f.state.Configs() {
if cfg.BranchID == branchID && cfg.ID.String() == configID {
return cfg
}
}
return nil
}

// getRelationTypeDescription returns a human-readable description of the relation type.
func (f *File) getRelationTypeDescription(relType model.RelationType) string {
switch relType {
case model.UsedInOrchestratorRelType:
return "orchestrator"
case model.SchedulerForRelType:
return "scheduler"
case model.UsedInConfigInputMappingRelType:
return "config with input mapping"
case model.UsedInRowInputMappingRelType:
return "config row with input mapping"
default:
return "configuration"
}
}
6 changes: 4 additions & 2 deletions pkg/lib/operation/project/sync/pull/operation.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ func Run(ctx context.Context, projectState *project.State, o Options, d dependen
return err
}

ignoreConfigsAndRows(projectState)
IgnoreConfigsAndRows(projectState)
}

// Diff
Expand Down Expand Up @@ -152,7 +152,9 @@ func Run(ctx context.Context, projectState *project.State, o Options, d dependen
return nil
}

func ignoreConfigsAndRows(projectState *project.State) {
// IgnoreConfigsAndRows clears remote state of ignored configs/rows so they behave as absent.
// Exported to reuse in push operation as well.
func IgnoreConfigsAndRows(projectState *project.State) {
for _, v := range projectState.IgnoredConfigRows() {
v.SetRemoteState(nil)
}
Expand Down
4 changes: 4 additions & 0 deletions pkg/lib/operation/project/sync/push/operation.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/keboola/keboola-as-code/pkg/lib/operation/project/local/encrypt"
"github.com/keboola/keboola-as-code/pkg/lib/operation/project/local/validate"
createDiff "github.com/keboola/keboola-as-code/pkg/lib/operation/project/sync/diff/create"
pullop "github.com/keboola/keboola-as-code/pkg/lib/operation/project/sync/pull"
)

type Options struct {
Expand Down Expand Up @@ -76,6 +77,9 @@ func Run(ctx context.Context, projectState *project.State, o Options, d dependen
if err = file.IgnoreConfigsOrRows(); err != nil {
return err
}

// Also propagate ignore to orchestrators referencing ignored configs
pullop.IgnoreConfigsAndRows(projectState)
}

// Diff
Expand Down
34 changes: 34 additions & 0 deletions test/cli/pull/ignore-configurations-orchestrator/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Ignore Configurations Orchestrator - Pull Test

## Purpose
This E2E test verifies that when a configuration is ignored via `.kbcignore`,
the system **validates** that no orchestrators reference that ignored configuration.
If an orchestrator references an ignored config, the operation fails with an error
instructing the user to explicitly add the orchestrator to `.kbcignore`.

## Test Setup
- **Remote state**: Contains 3 configs:
1. `ex-generic-v2/empty` - ignored via .kbcignore
2. `ex-generic-v2/without-rows` - not ignored
3. `keboola.orchestrator/orchestrator` - references the "empty" config in Task 1

- **Local state**: Empty (initial pull)
- **.kbcignore**: Contains pattern to ignore the "empty" config (but NOT the orchestrator)

## Expected Behavior
- Pull operation **fails** with error code 1
- Error message instructs user to add the orchestrator to `.kbcignore`:
```
Error: orchestrators reference ignored configurations:
- orchestrator "..." references ignored config "...", please add the orchestrator to .kbcignore:
keboola.orchestrator/<orchestrator-id>
```

## Fix
User must explicitly add the orchestrator to `.kbcignore`:
```
ex-generic-v2/%%TEST_BRANCH_MAIN_CONFIG_EMPTY_ID%%
keboola.orchestrator/%%TEST_BRANCH_MAIN_CONFIG_ORCHESTRATOR_ID%%
```

This ensures users are aware of orchestrator dependencies and make explicit ignore decisions.
2 changes: 2 additions & 0 deletions test/cli/pull/ignore-configurations-orchestrator/args
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
pull --verbose --storage-api-token %%TEST_KBC_STORAGE_API_TOKEN%%

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Error: configurations with dependencies reference ignored configurations:
- orchestrator %q references ignored config %q, please add it to .kbcignore:
keboola.orchestrator/%%_%%
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
%A
DEBUG Project state has been successfully loaded.
%A
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Ignore the "empty" extractor config
# This should also cause the orchestrator that references it to be ignored
ex-generic-v2/%%TEST_BRANCH_MAIN_CONFIG_EMPTY_ID%%

Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"allBranchesConfigs": [],
"branches": [
{
"branch": {
"name": "Main",
"isDefault": true
},
"configs": [
"empty",
"without-rows",
"orchestrator"
]
}
],
"envs": {
"TEST_ORCHESTRATOR_TASK_1_COMPONENT_ID": "ex-generic-v2",
"TEST_ORCHESTRATOR_TASK_1_CONFIG_ID": "%%TEST_BRANCH_MAIN_CONFIG_EMPTY_ID%%",
"TEST_ORCHESTRATOR_TASK_2_COMPONENT_ID": "ex-generic-v2",
"TEST_ORCHESTRATOR_TASK_2_CONFIG_ID": "%%TEST_BRANCH_MAIN_CONFIG_WITHOUT_ROWS_ID%%"
}
}

Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@

Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"parameters": {
"api": {
"baseUrl": "https://jsonplaceholder.typicode.com"
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
test fixture
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"name": "without-rows",
"isDisabled": false
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"name": "Main",
"isDefault": true
}
34 changes: 34 additions & 0 deletions test/cli/push/ignore-configurations-orchestrator/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Ignore Configurations Orchestrator - Push Test

## Purpose
This E2E test verifies that when a configuration is ignored via `.kbcignore`,
the system **validates** that no orchestrators reference that ignored configuration.
If an orchestrator references an ignored config, the operation fails with an error
instructing the user to explicitly add the orchestrator to `.kbcignore`.

## Test Setup
- **Remote state**: Empty project
- **Local state**: Contains 3 configs:
1. `ex-generic-v2/empty` - ignored via .kbcignore
2. `ex-generic-v2/without-rows` - not ignored
3. `keboola.orchestrator/orchestrator` - references the "empty" config in Task 1

- **.kbcignore**: Contains wildcard pattern to ignore the "empty" config (but NOT the orchestrator)

## Expected Behavior
- Push operation **fails** with error code 1
- Error message instructs user to add the orchestrator to `.kbcignore`:
```
Error: orchestrators reference ignored configurations:
- orchestrator "..." references ignored config "...", please add the orchestrator to .kbcignore:
keboola.orchestrator/<orchestrator-id>
```

## Fix
User must explicitly add the orchestrator to `.kbcignore`:
```
ex-generic-v2/%%_%%
keboola.orchestrator/%%_%%
```

This ensures users are aware of orchestrator dependencies and make explicit ignore decisions.
2 changes: 2 additions & 0 deletions test/cli/push/ignore-configurations-orchestrator/args
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
push --storage-api-token %%TEST_KBC_STORAGE_API_TOKEN%%

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"branches": [
{
"id": "%%TEST_BRANCH_MAIN_ID%%",
"name": "Main",
"description": "",
"isDefault": true,
"metadata": {}
}
],
"configurations": [
{
"branchId": "%%TEST_BRANCH_MAIN_ID%%",
"componentId": "ex-generic-v2",
"id": "%%_%%",
"name": "without-rows",
"description": "",
"changeDescription": "",
"isDisabled": false,
"isDeleted": false,
"created": "%%_%%",
"version": 1,
"state": {},
"currentVersion": {
"changeDescription": "",
"created": "%%_%%",
"creatorToken": {
"id": "%%_%%",
"description": "test"
},
"version": 1,
"state": {}
},
"configuration": {}
}
]
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Error: configurations with dependencies reference ignored configurations:
- orchestrator %q references ignored config %q, please add it to .kbcignore:
keboola.orchestrator/%%_%%
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Ignore the "empty" extractor config
# This should also cause the orchestrator that references it to be ignored during push
ex-generic-v2/%%_%%
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
test fixture
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"name": "empty",
"isDisabled": false
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"parameters": {
"api": {
"baseUrl": "https://jsonplaceholder.typicode.com"
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
test fixture
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"name": "without-rows",
"isDisabled": false
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"name": "Main",
"isDefault": true
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
Loading
Loading