Skip to content

Commit 8d713d9

Browse files
henrichterhenrichter
committed
feat[extractor]: use DI for extraction
1 parent dc65389 commit 8d713d9

30 files changed

Lines changed: 128 additions & 110 deletions

‎internal/knowledge/extractor/controller.go‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010
"time"
1111

1212
"github.com/cobaltcore-dev/cortex/api/v1alpha1"
13-
1413
"github.com/cobaltcore-dev/cortex/internal/knowledge/db"
1514
"github.com/cobaltcore-dev/cortex/pkg/multicluster"
1615
corev1 "k8s.io/api/core/v1"
@@ -78,6 +77,7 @@ func (r *KnowledgeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
7877

7978
// Check if all datasources configured share the same database secret ref.
8079
var databaseSecretRef *corev1.SecretReference
80+
var dataSources []*v1alpha1.Datasource
8181
for _, dsRef := range knowledge.Spec.Dependencies.Datasources {
8282
ds := &v1alpha1.Datasource{}
8383
if err := r.Get(ctx, client.ObjectKey{
@@ -118,7 +118,34 @@ func (r *KnowledgeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
118118
}
119119
return ctrl.Result{}, nil
120120
}
121+
dataSources = append(dataSources, ds)
122+
}
123+
124+
var knowledgeSources []*v1alpha1.Knowledge
125+
for _, knRef := range knowledge.Spec.Dependencies.Knowledges {
126+
kn := &v1alpha1.Knowledge{}
127+
if err := r.Get(ctx, client.ObjectKey{
128+
Namespace: req.Namespace,
129+
Name: knRef.Name,
130+
}, kn); err != nil {
131+
log.Error(err, "failed to get knowledge", "name", knRef.Name)
132+
old := knowledge.DeepCopy()
133+
meta.SetStatusCondition(&knowledge.Status.Conditions, metav1.Condition{
134+
Type: v1alpha1.KnowledgeConditionReady,
135+
Status: metav1.ConditionFalse,
136+
Reason: "KnowledgeFetchFailed",
137+
Message: "failed to get knowledge: " + err.Error(),
138+
})
139+
patch := client.MergeFrom(old)
140+
if err := r.Status().Patch(ctx, knowledge, patch); err != nil {
141+
log.Error(err, "failed to patch knowledge status")
142+
return ctrl.Result{}, err
143+
}
144+
return ctrl.Result{}, err
145+
}
146+
knowledgeSources = append(knowledgeSources, kn)
121147
}
148+
122149
// When we have datasources reading from a database, connect to it.
123150
var authenticatedDatasourceDB *db.DB
124151
if databaseSecretRef != nil {
@@ -162,7 +189,7 @@ func (r *KnowledgeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
162189
return ctrl.Result{}, err
163190
}
164191

165-
features, err := extractor.Extract()
192+
features, err := extractor.Extract(dataSources, knowledgeSources)
166193
if err != nil {
167194
log.Error(err, "failed to extract features", "name", knowledge.Spec.Extractor.Name)
168195
old := knowledge.DeepCopy()

‎internal/knowledge/extractor/monitor.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,10 +94,10 @@ func monitorFeatureExtractor[F plugins.FeatureExtractor](label string, f F, m Mo
9494
}
9595

9696
// Run the wrapped feature extractor and measure the time it takes.
97-
func (m FeatureExtractorMonitor[F]) Extract() ([]plugins.Feature, error) {
97+
func (m FeatureExtractorMonitor[F]) Extract(d []*v1alpha1.Datasource, k []*v1alpha1.Knowledge) ([]plugins.Feature, error) {
9898
slog.Info("features: extracting", "extractor", m.label)
9999

100-
features, err := m.FeatureExtractor.Extract()
100+
features, err := m.FeatureExtractor.Extract(d, k)
101101
if err != nil {
102102
return nil, err
103103
}

‎internal/knowledge/extractor/monitor_test.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ func (m *mockFeatureExtractor) Init(datasourceDB *db.DB, client client.Client, s
2626
return m.initError
2727
}
2828

29-
func (m *mockFeatureExtractor) Extract() ([]plugins.Feature, error) {
29+
func (m *mockFeatureExtractor) Extract(_ []*v1alpha1.Datasource, _ []*v1alpha1.Knowledge) ([]plugins.Feature, error) {
3030
if m.extractError != nil {
3131
return nil, m.extractError
3232
}
@@ -118,7 +118,7 @@ func TestMonitorFeatureExtractor_Extract_Success(t *testing.T) {
118118

119119
wrappedExtractor := monitorFeatureExtractor("test-extractor", mockExtractor, monitor)
120120

121-
features, err := wrappedExtractor.Extract()
121+
features, err := wrappedExtractor.Extract([]*v1alpha1.Datasource{}, []*v1alpha1.Knowledge{})
122122
if err != nil {
123123
t.Errorf("Expected no error, got %v", err)
124124
}
@@ -154,7 +154,7 @@ func TestMonitorFeatureExtractor_Extract_Error(t *testing.T) {
154154

155155
wrappedExtractor := monitorFeatureExtractor("test-extractor", mockExtractor, monitor)
156156

157-
features, err := wrappedExtractor.Extract()
157+
features, err := wrappedExtractor.Extract([]*v1alpha1.Datasource{}, []*v1alpha1.Knowledge{})
158158
if !errors.Is(err, expectedError) {
159159
t.Errorf("Expected error %v, got %v", expectedError, err)
160160
}
@@ -171,7 +171,7 @@ func TestMonitorFeatureExtractor_Extract_EmptyFeatures(t *testing.T) {
171171

172172
wrappedExtractor := monitorFeatureExtractor("test-extractor", mockExtractor, monitor)
173173

174-
features, err := wrappedExtractor.Extract()
174+
features, err := wrappedExtractor.Extract([]*v1alpha1.Datasource{}, []*v1alpha1.Knowledge{})
175175
if err != nil {
176176
t.Errorf("Expected no error, got %v", err)
177177
}
@@ -202,7 +202,7 @@ func TestMonitorFeatureExtractor_NilMonitor(t *testing.T) {
202202
wrappedExtractor := monitorFeatureExtractor("test-extractor", mockExtractor, monitor)
203203

204204
// Should not panic even with nil metrics
205-
features, err := wrappedExtractor.Extract()
205+
features, err := wrappedExtractor.Extract([]*v1alpha1.Datasource{}, []*v1alpha1.Knowledge{})
206206
if err != nil {
207207
t.Errorf("Expected no error, got %v", err)
208208
}
@@ -235,7 +235,7 @@ func TestMonitorFeatureExtractor_MultipleExtractions(t *testing.T) {
235235
wrappedExtractor := monitorFeatureExtractor("test-extractor", mockExtractor, monitor)
236236

237237
// First extraction
238-
features1, err := wrappedExtractor.Extract()
238+
features1, err := wrappedExtractor.Extract([]*v1alpha1.Datasource{}, []*v1alpha1.Knowledge{})
239239
if err != nil {
240240
t.Errorf("First extraction failed: %v", err)
241241
}
@@ -249,7 +249,7 @@ func TestMonitorFeatureExtractor_MultipleExtractions(t *testing.T) {
249249
}
250250

251251
// Second extraction
252-
features2, err := wrappedExtractor.Extract()
252+
features2, err := wrappedExtractor.Extract([]*v1alpha1.Datasource{}, []*v1alpha1.Knowledge{})
253253
if err != nil {
254254
t.Errorf("Second extraction failed: %v", err)
255255
}
@@ -286,12 +286,12 @@ func TestMonitorFeatureExtractor_DifferentExtractorNames(t *testing.T) {
286286
wrappedExtractor2 := monitorFeatureExtractor("extractor-2", mockExtractor2, monitor)
287287

288288
// Extract from both
289-
_, err1 := wrappedExtractor1.Extract()
289+
_, err1 := wrappedExtractor1.Extract([]*v1alpha1.Datasource{}, []*v1alpha1.Knowledge{})
290290
if err1 != nil {
291291
t.Errorf("Extractor 1 failed: %v", err1)
292292
}
293293

294-
_, err2 := wrappedExtractor2.Extract()
294+
_, err2 := wrappedExtractor2.Extract([]*v1alpha1.Datasource{}, []*v1alpha1.Knowledge{})
295295
if err2 != nil {
296296
t.Errorf("Extractor 2 failed: %v", err2)
297297
}

‎internal/knowledge/extractor/plugins/compute/host_az.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package compute
66
import (
77
_ "embed"
88

9+
"github.com/cobaltcore-dev/cortex/api/v1alpha1"
910
"github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins"
1011
)
1112

@@ -28,6 +29,6 @@ type HostAZExtractor struct {
2829
var hostAZQuery string
2930

3031
// Extract the traits of a compute host from the database.
31-
func (e *HostAZExtractor) Extract() ([]plugins.Feature, error) {
32+
func (e *HostAZExtractor) Extract(_ []*v1alpha1.Datasource, _ []*v1alpha1.Knowledge) ([]plugins.Feature, error) {
3233
return e.ExtractSQL(hostAZQuery)
3334
}

‎internal/knowledge/extractor/plugins/compute/host_az_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func TestHostAZExtractor_Extract(t *testing.T) {
7070
if err := extractor.Init(&testDB, nil, config); err != nil {
7171
t.Fatalf("expected no error, got %v", err)
7272
}
73-
features, err := extractor.Extract()
73+
features, err := extractor.Extract([]*v1alpha1.Datasource{}, []*v1alpha1.Knowledge{})
7474
if err != nil {
7575
t.Fatalf("expected no error, got %v", err)
7676
}

‎internal/knowledge/extractor/plugins/compute/host_capabilities.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package compute
66
import (
77
_ "embed"
88

9+
"github.com/cobaltcore-dev/cortex/api/v1alpha1"
910
"github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins"
1011
)
1112

@@ -32,6 +33,6 @@ type HostCapabilitiesExtractor struct {
3233
var hostCapabilitiesQuery string
3334

3435
// Extract the traits of a compute host from the database.
35-
func (e *HostCapabilitiesExtractor) Extract() ([]plugins.Feature, error) {
36+
func (e *HostCapabilitiesExtractor) Extract(_ []*v1alpha1.Datasource, _ []*v1alpha1.Knowledge) ([]plugins.Feature, error) {
3637
return e.ExtractSQL(hostCapabilitiesQuery)
3738
}

‎internal/knowledge/extractor/plugins/compute/host_capabilities_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ func TestHostCapabilitiesExtractor_Extract(t *testing.T) {
5555
if err := extractor.Init(&testDB, nil, config); err != nil {
5656
t.Fatalf("expected no error, got %v", err)
5757
}
58-
features, err := extractor.Extract()
58+
features, err := extractor.Extract([]*v1alpha1.Datasource{}, []*v1alpha1.Knowledge{})
5959
if err != nil {
6060
t.Fatalf("expected no error, got %v", err)
6161
}

‎internal/knowledge/extractor/plugins/compute/host_details.go‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@
44
package compute
55

66
import (
7-
"context"
87
_ "embed"
98
"errors"
9+
"fmt"
10+
"slices"
1011
"strings"
1112

1213
"github.com/cobaltcore-dev/cortex/api/v1alpha1"
1314
"github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins"
14-
"sigs.k8s.io/controller-runtime/pkg/client"
1515
)
1616

1717
type HostDetails struct {
@@ -56,7 +56,7 @@ type HostDetailsExtractor struct {
5656
var hostDetailsQuery string
5757

5858
// Extract the traits of a compute host from the database.
59-
func (e *HostDetailsExtractor) Extract() ([]plugins.Feature, error) {
59+
func (e *HostDetailsExtractor) Extract(_ []*v1alpha1.Datasource, k []*v1alpha1.Knowledge) ([]plugins.Feature, error) {
6060
if e.DB == nil {
6161
return nil, errors.New("database connection is not initialized")
6262
}
@@ -66,16 +66,15 @@ func (e *HostDetailsExtractor) Extract() ([]plugins.Feature, error) {
6666
}
6767

6868
// Add the pinned projects to the host details.
69-
pinnedProjectsKnowledge := &v1alpha1.Knowledge{}
70-
if err := e.Client.Get(
71-
context.Background(),
72-
client.ObjectKey{Name: "host-pinned-projects"},
73-
pinnedProjectsKnowledge,
74-
); err != nil {
75-
return nil, err
69+
name := "host-pinned-projects"
70+
idx := slices.IndexFunc(k, func(k *v1alpha1.Knowledge) bool {
71+
return k.Name == name
72+
})
73+
if idx < 0 {
74+
return nil, fmt.Errorf("knowledge '%s' not found", name)
7675
}
7776
pinnedProjects, err := v1alpha1.
78-
UnboxFeatureList[HostPinnedProjects](pinnedProjectsKnowledge.Status.Raw)
77+
UnboxFeatureList[HostPinnedProjects](k[idx].Status.Raw)
7978
if err != nil {
8079
return nil, err
8180
}
@@ -99,16 +98,15 @@ func (e *HostDetailsExtractor) Extract() ([]plugins.Feature, error) {
9998
}
10099

101100
// Add the availability zones to the host details.
102-
azKnowledge := &v1alpha1.Knowledge{}
103-
if err := e.Client.Get(
104-
context.Background(),
105-
client.ObjectKey{Name: "host-az"},
106-
azKnowledge,
107-
); err != nil {
108-
return nil, err
101+
name = "host-az"
102+
idx = slices.IndexFunc(k, func(k *v1alpha1.Knowledge) bool {
103+
return k.Name == name
104+
})
105+
if idx < 0 {
106+
return nil, fmt.Errorf("knowledge '%s' not found", name)
109107
}
110108
hostAZs, err := v1alpha1.
111-
UnboxFeatureList[HostAZ](azKnowledge.Status.Raw)
109+
UnboxFeatureList[HostAZ](k[idx].Status.Raw)
112110
if err != nil {
113111
return nil, err
114112
}

‎internal/knowledge/extractor/plugins/compute/host_details_test.go‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,6 @@ func TestHostDetailsExtractor_Extract(t *testing.T) {
3030
testDB := db.DB{DbMap: dbEnv.DbMap}
3131
defer dbEnv.Close()
3232

33-
scheme, err := v1alpha1.SchemeBuilder.Build()
34-
if err != nil {
35-
t.Fatalf("expected no error, got %v", err)
36-
}
37-
3833
// Create dependency tables
3934
if err := testDB.CreateTable(
4035
testDB.AddTable(nova.Hypervisor{}),
@@ -109,21 +104,21 @@ func TestHostDetailsExtractor_Extract(t *testing.T) {
109104

110105
extractor := &HostDetailsExtractor{}
111106
config := v1alpha1.KnowledgeSpec{}
112-
client := fake.NewClientBuilder().
113-
WithScheme(scheme).
114-
WithObjects(&v1alpha1.Knowledge{
107+
client := fake.NewClientBuilder().Build()
108+
knowledges := []*v1alpha1.Knowledge{
109+
{
115110
ObjectMeta: v1.ObjectMeta{Name: "host-pinned-projects"},
116111
Status: v1alpha1.KnowledgeStatus{Raw: hostPinnedProjects},
117-
}).
118-
WithObjects(&v1alpha1.Knowledge{
112+
},
113+
{
119114
ObjectMeta: v1.ObjectMeta{Name: "host-az"},
120115
Status: v1alpha1.KnowledgeStatus{Raw: hostAvailabilityZones},
121-
}).
122-
Build()
116+
},
117+
}
123118
if err := extractor.Init(&testDB, client, config); err != nil {
124119
t.Fatalf("expected no error, got %v", err)
125120
}
126-
features, err := extractor.Extract()
121+
features, err := extractor.Extract([]*v1alpha1.Datasource{}, knowledges)
127122
if err != nil {
128123
t.Fatalf("expected no error, got %v", err)
129124
}

‎internal/knowledge/extractor/plugins/compute/host_pinned_project_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -533,7 +533,7 @@ func TestHostPinnedProjectsExtractor_Extract(t *testing.T) {
533533
t.Fatalf("expected no error, got %v", err)
534534
}
535535

536-
features, err := extractor.Extract()
536+
features, err := extractor.Extract([]*v1alpha1.Datasource{}, []*v1alpha1.Knowledge{})
537537
if err != nil {
538538
t.Fatalf("expected no error, got %v", err)
539539
}

0 commit comments

Comments
 (0)