diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go index 85a9f80be2e89..47b2944b20e1e 100644 --- a/pkg/templates/templates.go +++ b/pkg/templates/templates.go @@ -17,31 +17,25 @@ limitations under the License. package templates import ( - "bytes" "context" "fmt" - "io" "os" "strings" - "text/template" "k8s.io/klog/v2" - "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/util/pkg/vfs" ) type Templates struct { - cluster *kops.Cluster - resources map[string]fi.Resource - TemplateFunctions template.FuncMap + resources map[string]fi.Resource + isTemplate map[string]bool } -func LoadTemplates(ctx context.Context, cluster *kops.Cluster, base vfs.Path) (*Templates, error) { +func LoadTemplates(ctx context.Context, base vfs.Path) (*Templates, error) { t := &Templates{ - cluster: cluster, - resources: make(map[string]fi.Resource), - TemplateFunctions: make(template.FuncMap), + resources: make(map[string]fi.Resource), + isTemplate: make(map[string]bool), } err := t.loadFrom(ctx, base) if err != nil { @@ -54,6 +48,11 @@ func (t *Templates) Find(key string) fi.Resource { return t.resources[key] } +// IsTemplate reports whether key was loaded from a file ending in .template. +func (t *Templates) IsTemplate(key string) bool { + return t.isTemplate[key] +} + func (t *Templates) loadFrom(ctx context.Context, base vfs.Path) error { files, err := base.ReadTree(ctx) if err != nil { @@ -75,74 +74,11 @@ func (t *Templates) loadFrom(ctx context.Context, base vfs.Path) error { return fmt.Errorf("error getting relative path for %s", f) } - var resource fi.Resource - if strings.HasSuffix(key, ".template") { - key = strings.TrimSuffix(key, ".template") - klog.V(6).Infof("loading (templated) resource %q", key) - - resource = &templateResource{ - template: string(contents), - loader: t, - key: key, - } - } else { - klog.V(6).Infof("loading resource %q", key) - resource = fi.NewBytesResource(contents) - - } - - t.resources[key] = resource + isTemplate := strings.HasSuffix(key, ".template") + key = strings.TrimSuffix(key, ".template") + klog.V(6).Infof("loading resource %q", key) + t.resources[key] = fi.NewBytesResource(contents) + t.isTemplate[key] = isTemplate } return nil } - -func (l *Templates) executeTemplate(key string, d string) (string, error) { - t := template.New(key) - - funcMap := make(template.FuncMap) - // funcMap["Args"] = func() []string { - // return args - //} - //funcMap["RenderResource"] = func(resourceName string, args []string) (string, error) { - // return l.renderResource(resourceName, args) - //} - for k, fn := range l.TemplateFunctions { - funcMap[k] = fn - } - t.Funcs(funcMap) - - t.Option("missingkey=zero") - - spec := l.cluster.Spec - - _, err := t.Parse(d) - if err != nil { - return "", fmt.Errorf("error parsing template %q: %v", key, err) - } - - var buffer bytes.Buffer - err = t.ExecuteTemplate(&buffer, key, spec) - if err != nil { - return "", fmt.Errorf("error executing template %q: %v", key, err) - } - - return buffer.String(), nil -} - -type templateResource struct { - key string - loader *Templates - template string -} - -var _ fi.Resource = &templateResource{} - -func (a *templateResource) Open() (io.Reader, error) { - var err error - result, err := a.loader.executeTemplate(a.key, a.template) - if err != nil { - return nil, fmt.Errorf("error executing resource template %q: %v", a.key, err) - } - reader := bytes.NewReader([]byte(result)) - return reader, nil -} diff --git a/pkg/templates/templates_test.go b/pkg/templates/templates_test.go new file mode 100644 index 0000000000000..6f68c0cecd3bd --- /dev/null +++ b/pkg/templates/templates_test.go @@ -0,0 +1,58 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package templates + +import ( + "context" + "os" + "path/filepath" + "testing" + + "k8s.io/kops/util/pkg/vfs" +) + +func TestLoadTemplatesTracksTemplateSources(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "addons", "example"), 0755); err != nil { + t.Fatalf("creating addon dir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "addons", "example", "plain.yaml"), []byte("plain"), 0644); err != nil { + t.Fatalf("writing plain resource: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "addons", "example", "rendered.yaml.template"), []byte("template"), 0644); err != nil { + t.Fatalf("writing template resource: %v", err) + } + + templates, err := LoadTemplates(context.Background(), vfs.NewFSPath(dir)) + if err != nil { + t.Fatalf("loading templates: %v", err) + } + + if templates.Find("addons/example/plain.yaml") == nil { + t.Fatalf("plain resource was not loaded") + } + if templates.IsTemplate("addons/example/plain.yaml") { + t.Fatalf("plain resource reported as template") + } + + if templates.Find("addons/example/rendered.yaml") == nil { + t.Fatalf("template resource was not loaded under trimmed key") + } + if !templates.IsTemplate("addons/example/rendered.yaml") { + t.Fatalf("template resource did not report template source") + } +} diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index ea96879a3b0f9..51d8c7b565a40 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -513,9 +513,10 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { } } - tf := &TemplateFunctions{ - KopsModelContext: *modelContext, - cloud: cloud, + addonRenderer := &addonTemplateRenderer{ + modelContext: modelContext, + cloud: cloud, + secretStore: secretStore, } nodeUpAssets, err := nodemodel.BuildNodeUpAssets(ctx, assetBuilder) @@ -534,22 +535,18 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { } { - templates, err := templates.LoadTemplates(ctx, cluster, models.NewAssetPath("cloudup/resources")) + templates, err := templates.LoadTemplates(ctx, models.NewAssetPath("cloudup/resources")) if err != nil { return nil, fmt.Errorf("error loading templates: %v", err) } - err = tf.AddTo(templates.TemplateFunctions, secretStore) - if err != nil { - return nil, err - } - bcb := bootstrapchannelbuilder.NewBootstrapChannelBuilder( modelContext, clusterLifecycle, assetBuilder, templates, addons, + addonRenderer, ) l.Builders = append(l.Builders, diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/addonmanifest.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/addonmanifest.go new file mode 100644 index 0000000000000..aea1bce782c0d --- /dev/null +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/addonmanifest.go @@ -0,0 +1,192 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrapchannelbuilder + +import ( + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + channelsapi "k8s.io/kops/channels/pkg/api" + "k8s.io/kops/pkg/assets" + "k8s.io/kops/pkg/model" + "k8s.io/kops/pkg/model/components/addonmanifests" + "k8s.io/kops/pkg/model/iam" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/terraform" + "k8s.io/kops/upup/pkg/fi/fitasks" + "k8s.io/kops/upup/pkg/fi/utils" +) + +// +kops:fitask +type AddonManifest struct { + Name *string + Lifecycle fi.Lifecycle + Location *string + Contents fi.Resource + PublicACL *bool + + addonRenderer AddonTemplateRenderer + source fi.Resource + // addonSpec is mutated during Normalize and subsequently read by BootstrapChannel, + // which depends on this task so the mutations are fully visible by the time it runs. + addonSpec *channelsapi.AddonSpec + buildPrune bool + skipRemap bool + skipRender bool + modelContext *model.KopsModelContext + assetBuilder *assets.AssetBuilder + serviceAccounts map[types.NamespacedName]iam.Subject +} + +var ( + _ fi.CloudupTaskNormalize = (*AddonManifest)(nil) + _ fi.CloudupHasDependencies = (*AddonManifest)(nil) +) + +// GetDependencies makes the addon wait for every non-addon task, so templates that reach into +// the task graph see fully realized state. Intentionally pessimistic: addon rendering is fast +// enough that over-depending is not worth the footgun of under-declaring. +func (a *AddonManifest) GetDependencies(tasks map[string]fi.CloudupTask) []fi.CloudupTask { + dependencies := make([]fi.CloudupTask, 0, len(tasks)) + for _, task := range tasks { + if isAddonTask(task) { + continue + } + dependencies = append(dependencies, task) + } + return dependencies +} + +func (a *AddonManifest) Normalize(c *fi.CloudupContext) error { + if a.addonSpec == nil { + return fmt.Errorf("addon spec is not configured for %q", fi.ValueOf(a.Name)) + } + if a.source == nil { + return fmt.Errorf("addon source is not configured for %q", fi.ValueOf(a.Name)) + } + + manifestBytes, err := fi.ResourceAsBytes(a.source) + if err != nil { + return fmt.Errorf("error reading addon %q manifest: %v", fi.ValueOf(a.Name), err) + } + + if !a.skipRender && a.addonRenderer != nil { + manifestBytes, err = a.addonRenderer.RenderTemplate(fi.ValueOf(a.Location), manifestBytes, tasksVisibleToAddons(c.AllTasks())) + if err != nil { + return fmt.Errorf("error rendering addon %q template: %w", fi.ValueOf(a.Name), err) + } + } + + if !a.skipRemap { + manifestBytes, err = addonmanifests.RemapAddonManifest(a.addonSpec, a.modelContext, a.assetBuilder, manifestBytes, a.serviceAccounts) + if err != nil { + klog.Infof("invalid manifest: %s", string(manifestBytes)) + return fmt.Errorf("error remapping manifest %s: %v", fi.ValueOf(a.Location), err) + } + } + + manifestBytes = []byte(strings.TrimSpace(string(manifestBytes))) + + if a.buildPrune { + if err := buildPruneDirectives(a.addonSpec, manifestBytes); err != nil { + return fmt.Errorf("failed to configure pruning for %s: %w", fi.ValueOf(a.addonSpec.Name), err) + } + } + + rawManifest := string(manifestBytes) + manifestHash, err := utils.HashString(rawManifest) + if err != nil { + return fmt.Errorf("error hashing manifest: %v", err) + } + a.addonSpec.ManifestHash = manifestHash + a.Contents = fi.NewBytesResource(manifestBytes) + + return nil +} + +// Find returns a sparsely-populated AddonManifest reflecting the stored ManagedFile: only the +// fields needed by CheckChanges/Render/RenderTerraform (which delegate to toManagedFile) are set. +// Render-only fields such as addonSpec and source are intentionally left nil since they have no +// meaning for an already-materialized remote file. +func (a *AddonManifest) Find(c *fi.CloudupContext) (*AddonManifest, error) { + managedFile := a.toManagedFile() + actual, err := managedFile.Find(c) + if err != nil || actual == nil { + return nil, err + } + a.PublicACL = managedFile.PublicACL + + return &AddonManifest{ + Name: a.Name, + Lifecycle: a.Lifecycle, + Location: a.Location, + Contents: actual.Contents, + PublicACL: actual.PublicACL, + }, nil +} + +func (a *AddonManifest) Run(c *fi.CloudupContext) error { + return fi.CloudupDefaultDeltaRunMethod(a, c) +} + +func (a *AddonManifest) CheckChanges(actual, expected, changes *AddonManifest) error { + return expected.toManagedFile().CheckChanges(actual.toManagedFile(), expected.toManagedFile(), changes.toManagedFile()) +} + +func (a *AddonManifest) Render(c *fi.CloudupContext, actual, expected, changes *AddonManifest) error { + return expected.toManagedFile().Render(c, actual.toManagedFile(), expected.toManagedFile(), changes.toManagedFile()) +} + +func (a *AddonManifest) RenderTerraform(c *fi.CloudupContext, t *terraform.TerraformTarget, actual, expected, changes *AddonManifest) error { + return expected.toManagedFile().RenderTerraform(c, t, actual.toManagedFile(), expected.toManagedFile(), changes.toManagedFile()) +} + +func (a *AddonManifest) toManagedFile() *fitasks.ManagedFile { + if a == nil { + return nil + } + return &fitasks.ManagedFile{ + Name: a.Name, + Lifecycle: a.Lifecycle, + Location: a.Location, + Contents: a.Contents, + PublicACL: a.PublicACL, + } +} + +// tasksVisibleToAddons returns all tasks that are valid references from an addon template. +func tasksVisibleToAddons(tasks map[string]fi.CloudupTask) map[string]fi.CloudupTask { + filtered := make(map[string]fi.CloudupTask, len(tasks)) + for key, task := range tasks { + if isAddonTask(task) { + continue + } + filtered[key] = task + } + return filtered +} + +// isAddonTask reports whether a task belongs to the addon-rendering pipeline. +func isAddonTask(task fi.CloudupTask) bool { + switch task.(type) { + case *AddonManifest, *BootstrapChannel: + return true + } + return false +} diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/addonmanifest_fitask.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/addonmanifest_fitask.go new file mode 100644 index 0000000000000..473c005d5a523 --- /dev/null +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/addonmanifest_fitask.go @@ -0,0 +1,52 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by fitask. DO NOT EDIT. + +package bootstrapchannelbuilder + +import ( + "k8s.io/kops/upup/pkg/fi" +) + +// AddonManifest + +var _ fi.HasLifecycle = &AddonManifest{} + +// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle +func (o *AddonManifest) GetLifecycle() fi.Lifecycle { + return o.Lifecycle +} + +// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle +func (o *AddonManifest) SetLifecycle(lifecycle fi.Lifecycle) { + o.Lifecycle = lifecycle +} + +var _ fi.HasName = &AddonManifest{} + +// GetName returns the Name of the object, implementing fi.HasName +func (o *AddonManifest) GetName() *string { + return o.Name +} + +// String is the stringer function for the task, producing readable output using fi.TaskAsString +func (o *AddonManifest) String() string { + return fi.CloudupTaskAsString(o) +} diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/addonmanifest_test.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/addonmanifest_test.go new file mode 100644 index 0000000000000..0637c28c9dbe7 --- /dev/null +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/addonmanifest_test.go @@ -0,0 +1,160 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrapchannelbuilder + +import ( + "context" + "strings" + "testing" + + channelsapi "k8s.io/kops/channels/pkg/api" + "k8s.io/kops/pkg/assets" + "k8s.io/kops/upup/pkg/fi" +) + +type recordingAddonRenderer struct { + calls int + rendered []byte +} + +func (r *recordingAddonRenderer) RenderTemplate(name string, source []byte, tasks map[string]fi.CloudupTask) ([]byte, error) { + r.calls++ + if r.rendered != nil { + return r.rendered, nil + } + return source, nil +} + +func (r *recordingAddonRenderer) CloudControllerConfigArgv() ([]string, error) { + return nil, nil +} + +func TestAddonManifestNormalizeSkipsRenderForRawSources(t *testing.T) { + ctx, err := fi.NewCloudupContext(context.Background(), fi.DeletionProcessingModeDeleteIncludingDeferred, nil, nil, nil, nil, nil, nil, map[string]fi.CloudupTask{}) + if err != nil { + t.Fatalf("building cloudup context: %v", err) + } + + rawManifest := []byte("apiVersion: v1\nkind: ConfigMap\ndata:\n literal: \"{{ .Values.image\"\n") + renderer := &recordingAddonRenderer{ + rendered: []byte("apiVersion: v1\nkind: ConfigMap\ndata:\n literal: rendered\n"), + } + addon := &AddonManifest{ + Name: fi.PtrTo("raw-addon"), + Location: fi.PtrTo("addons/raw-addon.yaml"), + source: fi.NewBytesResource(rawManifest), + skipRender: true, + addonRenderer: renderer, + addonSpec: testAddonSpec("raw-addon"), + skipRemap: true, + } + + if err := addon.Normalize(ctx); err != nil { + t.Fatalf("normalizing raw addon: %v", err) + } + if renderer.calls != 0 { + t.Fatalf("renderer calls = %d, want 0", renderer.calls) + } + + actual, err := fi.ResourceAsString(addon.Contents) + if err != nil { + t.Fatalf("reading addon contents: %v", err) + } + if actual != strings.TrimSpace(string(rawManifest)) { + t.Fatalf("addon contents = %q, want %q", actual, strings.TrimSpace(string(rawManifest))) + } +} + +func TestAddonManifestNormalizeRendersTemplateSources(t *testing.T) { + ctx, err := fi.NewCloudupContext(context.Background(), fi.DeletionProcessingModeDeleteIncludingDeferred, nil, nil, nil, nil, nil, nil, map[string]fi.CloudupTask{}) + if err != nil { + t.Fatalf("building cloudup context: %v", err) + } + + renderer := &recordingAddonRenderer{ + rendered: []byte("apiVersion: v1\nkind: ConfigMap\ndata:\n literal: rendered\n"), + } + addon := &AddonManifest{ + Name: fi.PtrTo("template-addon"), + Location: fi.PtrTo("addons/template-addon.yaml"), + source: fi.NewBytesResource([]byte("apiVersion: v1\nkind: ConfigMap\ndata:\n literal: {{ .Value }}\n")), + addonRenderer: renderer, + addonSpec: testAddonSpec("template-addon"), + skipRemap: true, + } + + if err := addon.Normalize(ctx); err != nil { + t.Fatalf("normalizing template addon: %v", err) + } + if renderer.calls != 1 { + t.Fatalf("renderer calls = %d, want 1", renderer.calls) + } + + actual, err := fi.ResourceAsString(addon.Contents) + if err != nil { + t.Fatalf("reading addon contents: %v", err) + } + if actual != "apiVersion: v1\nkind: ConfigMap\ndata:\n literal: rendered" { + t.Fatalf("addon contents = %q", actual) + } +} + +func TestAddonCollectImagesSkipsRenderForRawSources(t *testing.T) { + assetBuilder := assets.NewAssetBuilder(nil, nil, false) + rawManifest := []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: raw\ndata:\n helm: \"{{ .Values.image\"\n") + renderer := &recordingAddonRenderer{ + rendered: []byte("not: valid: yaml\n"), + } + addon := &Addon{ + Spec: testAddonSpec("raw-addon"), + Source: fi.NewBytesResource(rawManifest), + SkipRender: true, + } + + if err := addon.CollectImages(assetBuilder, renderer); err != nil { + t.Fatalf("collecting raw addon images: %v", err) + } + if renderer.calls != 0 { + t.Fatalf("renderer calls = %d, want 0", renderer.calls) + } +} + +func TestAddonCollectImagesRendersTemplateSources(t *testing.T) { + assetBuilder := assets.NewAssetBuilder(nil, nil, false) + renderer := &recordingAddonRenderer{ + rendered: []byte("apiVersion: v1\nkind: Pod\nmetadata:\n name: rendered\nspec:\n containers:\n - name: container\n image: registry.k8s.io/pause:3.9\n"), + } + addon := &Addon{ + Spec: testAddonSpec("template-addon"), + Source: fi.NewBytesResource([]byte("{{ template }}")), + } + + if err := addon.CollectImages(assetBuilder, renderer); err != nil { + t.Fatalf("collecting template addon images: %v", err) + } + if renderer.calls != 1 { + t.Fatalf("renderer calls = %d, want 1", renderer.calls) + } +} + +func testAddonSpec(name string) *channelsapi.AddonSpec { + return &channelsapi.AddonSpec{ + Name: fi.PtrTo(name), + Selector: map[string]string{"k8s-addon": name}, + Manifest: fi.PtrTo(name + ".yaml"), + } +} diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannel.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannel.go new file mode 100644 index 0000000000000..fb82a0cbd12a6 --- /dev/null +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannel.go @@ -0,0 +1,125 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrapchannelbuilder + +import ( + "fmt" + + channelsapi "k8s.io/kops/channels/pkg/api" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/terraform" + "k8s.io/kops/upup/pkg/fi/fitasks" + "k8s.io/kops/upup/pkg/fi/utils" +) + +// +kops:fitask +type BootstrapChannel struct { + Name *string + Lifecycle fi.Lifecycle + Location *string + Contents fi.Resource + PublicACL *bool + + addonManifests []*AddonManifest +} + +var ( + _ fi.CloudupTaskNormalize = (*BootstrapChannel)(nil) + _ fi.CloudupHasDependencies = (*BootstrapChannel)(nil) +) + +func (a *BootstrapChannel) GetDependencies(tasks map[string]fi.CloudupTask) []fi.CloudupTask { + dependencies := make([]fi.CloudupTask, 0, len(a.addonManifests)) + for _, manifest := range a.addonManifests { + dependencies = append(dependencies, manifest) + } + return dependencies +} + +func (a *BootstrapChannel) Normalize(c *fi.CloudupContext) error { + addonsObject := &channelsapi.Addons{} + addonsObject.Kind = "Addons" + addonsObject.ObjectMeta.Name = "bootstrap" + + for _, manifest := range a.addonManifests { + if manifest.addonSpec == nil { + return fmt.Errorf("addon manifest %q did not have a spec", fi.ValueOf(manifest.Name)) + } + if manifest.addonSpec.ManifestHash == "" { + return fmt.Errorf("addon %q manifest hash was not populated", fi.ValueOf(manifest.addonSpec.Name)) + } + addonsObject.Spec.Addons = append(addonsObject.Spec.Addons, manifest.addonSpec) + } + + if err := addonsObject.Verify(); err != nil { + return err + } + + addonsYAML, err := utils.YamlMarshal(addonsObject) + if err != nil { + return fmt.Errorf("error serializing addons yaml: %v", err) + } + + a.Contents = fi.NewBytesResource(addonsYAML) + return nil +} + +func (a *BootstrapChannel) Find(c *fi.CloudupContext) (*BootstrapChannel, error) { + managedFile := a.toManagedFile() + actual, err := managedFile.Find(c) + if err != nil || actual == nil { + return nil, err + } + a.PublicACL = managedFile.PublicACL + + return &BootstrapChannel{ + Name: a.Name, + Lifecycle: a.Lifecycle, + Location: a.Location, + Contents: actual.Contents, + PublicACL: actual.PublicACL, + }, nil +} + +func (a *BootstrapChannel) Run(c *fi.CloudupContext) error { + return fi.CloudupDefaultDeltaRunMethod(a, c) +} + +func (a *BootstrapChannel) CheckChanges(actual, expected, changes *BootstrapChannel) error { + return expected.toManagedFile().CheckChanges(actual.toManagedFile(), expected.toManagedFile(), changes.toManagedFile()) +} + +func (a *BootstrapChannel) Render(c *fi.CloudupContext, actual, expected, changes *BootstrapChannel) error { + return expected.toManagedFile().Render(c, actual.toManagedFile(), expected.toManagedFile(), changes.toManagedFile()) +} + +func (a *BootstrapChannel) RenderTerraform(c *fi.CloudupContext, t *terraform.TerraformTarget, actual, expected, changes *BootstrapChannel) error { + return expected.toManagedFile().RenderTerraform(c, t, actual.toManagedFile(), expected.toManagedFile(), changes.toManagedFile()) +} + +func (a *BootstrapChannel) toManagedFile() *fitasks.ManagedFile { + if a == nil { + return nil + } + return &fitasks.ManagedFile{ + Name: a.Name, + Lifecycle: a.Lifecycle, + Location: a.Location, + Contents: a.Contents, + PublicACL: a.PublicACL, + } +} diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannel_fitask.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannel_fitask.go new file mode 100644 index 0000000000000..cbdf2a5cd781e --- /dev/null +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannel_fitask.go @@ -0,0 +1,52 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by fitask. DO NOT EDIT. + +package bootstrapchannelbuilder + +import ( + "k8s.io/kops/upup/pkg/fi" +) + +// BootstrapChannel + +var _ fi.HasLifecycle = &BootstrapChannel{} + +// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle +func (o *BootstrapChannel) GetLifecycle() fi.Lifecycle { + return o.Lifecycle +} + +// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle +func (o *BootstrapChannel) SetLifecycle(lifecycle fi.Lifecycle) { + o.Lifecycle = lifecycle +} + +var _ fi.HasName = &BootstrapChannel{} + +// GetName returns the Name of the object, implementing fi.HasName +func (o *BootstrapChannel) GetName() *string { + return o.Name +} + +// String is the stringer function for the task, producing readable output using fi.TaskAsString +func (o *BootstrapChannel) String() string { + return fi.CloudupTaskAsString(o) +} diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index df3714b2451d3..726b1815a24a8 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -18,7 +18,6 @@ package bootstrapchannelbuilder import ( "fmt" - "strings" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -45,8 +44,6 @@ import ( "k8s.io/kops/pkg/templates" "k8s.io/kops/pkg/wellknownoperators" "k8s.io/kops/upup/pkg/fi" - "k8s.io/kops/upup/pkg/fi/fitasks" - "k8s.io/kops/upup/pkg/fi/utils" "k8s.io/kops/util/pkg/vfs" ) @@ -57,10 +54,20 @@ type BootstrapChannelBuilder struct { Lifecycle fi.Lifecycle templates *templates.Templates assetBuilder *assets.AssetBuilder + + addonRenderer AddonTemplateRenderer } var _ fi.CloudupModelBuilder = (*BootstrapChannelBuilder)(nil) +// AddonTemplateRenderer renders addon manifest templates against a per-call func map +// bound to the given task graph, and exposes a few direct template-function calls +// (e.g. CloudControllerConfigArgv) used by the channel builder itself. +type AddonTemplateRenderer interface { + RenderTemplate(name string, source []byte, tasks map[string]fi.CloudupTask) ([]byte, error) + CloudControllerConfigArgv() ([]string, error) +} + // networkingSelector is the labels set on networking addons // // The role.kubernetes.io/networking is used to label anything related to a networking addin, @@ -89,6 +96,7 @@ func NewBootstrapChannelBuilder(modelContext *model.KopsModelContext, clusterLifecycle fi.Lifecycle, assetBuilder *assets.AssetBuilder, templates *templates.Templates, addons kubemanifest.ObjectList, + addonRenderer AddonTemplateRenderer, ) *BootstrapChannelBuilder { return &BootstrapChannelBuilder{ KopsModelContext: modelContext, @@ -96,64 +104,25 @@ func NewBootstrapChannelBuilder(modelContext *model.KopsModelContext, assetBuilder: assetBuilder, templates: templates, ClusterAddons: addons, + addonRenderer: addonRenderer, } } -// Build is responsible for adding the addons to the channel +// Build is responsible for adding the addons to the channel. func (b *BootstrapChannelBuilder) Build(c *fi.CloudupModelBuilderContext) error { addons, serviceAccounts, err := b.buildAddons(c) if err != nil { return err } - for _, a := range addons.Items { - key := *a.Spec.Name - if a.Spec.Id != "" { - key = key + "-" + a.Spec.Id - } - name := b.Cluster.ObjectMeta.Name + "-addons-" + key - manifestPath := "addons/" + *a.Spec.Manifest - klog.V(4).Infof("Addon %q", name) - + for _, addon := range addons.Items { + manifestPath := "addons/" + *addon.Spec.Manifest manifestResource := b.templates.Find(manifestPath) if manifestResource == nil { return fmt.Errorf("unable to find manifest %s", manifestPath) } - - manifestBytes, err := fi.ResourceAsBytes(manifestResource) - if err != nil { - return fmt.Errorf("error reading manifest %s: %v", manifestPath, err) - } - - // Go through any transforms that are best expressed as code - remapped, err := addonmanifests.RemapAddonManifest(a.Spec, b.KopsModelContext, b.assetBuilder, manifestBytes, serviceAccounts) - if err != nil { - klog.Infof("invalid manifest: %s", string(manifestBytes)) - return fmt.Errorf("error remapping manifest %s: %v", manifestPath, err) - } - manifestBytes = remapped - - // Trim whitespace - manifestBytes = []byte(strings.TrimSpace(string(manifestBytes))) - - a.ManifestData = manifestBytes - - rawManifest := string(manifestBytes) - klog.V(4).Infof("Manifest %v", rawManifest) - - manifestHash, err := utils.HashString(rawManifest) - klog.V(4).Infof("hash %s", manifestHash) - if err != nil { - return fmt.Errorf("error hashing manifest: %v", err) - } - a.Spec.ManifestHash = manifestHash - - c.AddTask(&fitasks.ManagedFile{ - Contents: fi.NewBytesResource(manifestBytes), - Lifecycle: b.Lifecycle, - Location: fi.PtrTo(manifestPath), - Name: fi.PtrTo(name), - }) + addon.Source = manifestResource + addon.SkipRender = !b.templates.IsTemplate(manifestPath) } if featureflag.UseAddonOperators.Enabled() { @@ -168,43 +137,8 @@ func (b *BootstrapChannelBuilder) Build(c *fi.CloudupModelBuilderContext) error } b.ClusterAddons = clusterAddons - for _, a := range addonPackages { - key := *a.Spec.Name - if a.Spec.Id != "" { - key = key + "-" + a.Spec.Id - } - name := b.Cluster.ObjectMeta.Name + "-addons-" + key - manifestPath := "addons/" + *a.Spec.Manifest - - // Go through any transforms that are best expressed as code - manifestBytes, err := addonmanifests.RemapAddonManifest(&a.Spec, b.KopsModelContext, b.assetBuilder, a.Manifest, serviceAccounts) - if err != nil { - klog.Infof("invalid manifest: %s", string(a.Manifest)) - return fmt.Errorf("error remapping manifest %s: %v", manifestPath, err) - } - - // Trim whitespace - manifestBytes = []byte(strings.TrimSpace(string(manifestBytes))) - - rawManifest := string(manifestBytes) - klog.V(4).Infof("Manifest %v", rawManifest) - - manifestHash, err := utils.HashString(rawManifest) - klog.V(4).Infof("hash %s", manifestHash) - if err != nil { - return fmt.Errorf("error hashing manifest: %v", err) - } - a.Spec.ManifestHash = manifestHash - - c.AddTask(&fitasks.ManagedFile{ - Contents: fi.NewBytesResource(manifestBytes), - Lifecycle: b.Lifecycle, - Location: fi.PtrTo(manifestPath), - Name: fi.PtrTo(name), - }) - - addon := addons.Add(&a.Spec) - addon.ManifestData = manifestBytes + for _, pkg := range addonPackages { + addons.AddWithSource(&pkg.Spec, fi.NewBytesResource(pkg.Manifest)) } } @@ -230,71 +164,84 @@ func (b *BootstrapChannelBuilder) Build(c *fi.CloudupModelBuilderContext) error key := "cluster-addons.kops.k8s.io" location := key + "/default.yaml" - a := &channelsapi.AddonSpec{ + addon := addons.Add(&channelsapi.AddonSpec{ Name: fi.PtrTo(key), Selector: map[string]string{"k8s-addon": key}, Manifest: fi.PtrTo(location), - } - - name := b.Cluster.ObjectMeta.Name + "-addons-" + key - manifestPath := "addons/" + *a.Manifest + }) + addon.SkipRemap = true manifestBytes, err := applyAdditionalObjectsToCluster.ToYAML() if err != nil { return fmt.Errorf("error serializing addons: %v", err) } + addon.Source = fi.NewBytesResource(manifestBytes) + addon.SkipRender = true + } - // Trim whitespace - manifestBytes = []byte(strings.TrimSpace(string(manifestBytes))) - - rawManifest := string(manifestBytes) + preRegisterAddonImages := b.shouldPreRegisterAddonImages() - manifestHash, err := utils.HashString(rawManifest) - if err != nil { - return fmt.Errorf("error hashing manifest: %v", err) + var addonTasks []*AddonManifest + for _, addon := range addons.Items { + if preRegisterAddonImages { + if err := addon.CollectImages(b.assetBuilder, b.addonRenderer); err != nil { + klog.Warningf("unable to pre-register addon images for %q: %v", addonKey(addon.Spec), err) + } } - a.ManifestHash = manifestHash - c.AddTask(&fitasks.ManagedFile{ - Contents: fi.NewBytesResource(manifestBytes), + task := &AddonManifest{ + Name: fi.PtrTo(b.Cluster.ObjectMeta.Name + "-addons-" + addonKey(addon.Spec)), Lifecycle: b.Lifecycle, - Location: fi.PtrTo(manifestPath), - Name: fi.PtrTo(name), - }) - - addons.Add(a) + Location: fi.PtrTo("addons/" + *addon.Spec.Manifest), + + addonRenderer: b.addonRenderer, + source: addon.Source, + addonSpec: addon.Spec, + buildPrune: addon.BuildPrune, + skipRemap: addon.SkipRemap, + skipRender: addon.SkipRender, + modelContext: b.KopsModelContext, + assetBuilder: b.assetBuilder, + serviceAccounts: serviceAccounts, + } + c.AddTask(task) + addonTasks = append(addonTasks, task) } - if err := b.addPruneDirectives(addons); err != nil { - return err - } + c.AddTask(&BootstrapChannel{ + Name: fi.PtrTo(b.Cluster.ObjectMeta.Name + "-addons-bootstrap"), + Lifecycle: b.Lifecycle, + Location: fi.PtrTo("addons/bootstrap-channel.yaml"), + addonManifests: addonTasks, + }) - addonsObject := &channelsapi.Addons{} - addonsObject.Kind = "Addons" - addonsObject.ObjectMeta.Name = "bootstrap" - for _, addon := range addons.Items { - addonsObject.Spec.Addons = append(addonsObject.Spec.Addons, addon.Spec) - } + return nil +} - if err := addonsObject.Verify(); err != nil { - return err +func (b *BootstrapChannelBuilder) shouldPreRegisterAddonImages() bool { + if b.assetBuilder == nil || b.Cluster == nil || b.Cluster.Spec.CloudProvider.AWS == nil { + return false } - addonsYAML, err := utils.YamlMarshal(addonsObject) - if err != nil { - return fmt.Errorf("error serializing addons yaml: %v", err) + if b.Cluster.Spec.CloudProvider.AWS.WarmPool != nil { + return true } - name := b.Cluster.ObjectMeta.Name + "-addons-bootstrap" + for _, ig := range b.AllInstanceGroups { + if ig != nil && ig.Spec.WarmPool != nil { + return true + } + } - c.AddTask(&fitasks.ManagedFile{ - Contents: fi.NewBytesResource(addonsYAML), - Lifecycle: b.Lifecycle, - Location: fi.PtrTo("addons/bootstrap-channel.yaml"), - Name: fi.PtrTo(name), - }) + return false +} - return nil +func addonKey(spec *channelsapi.AddonSpec) string { + key := *spec.Name + if spec.Id != "" { + key = key + "-" + spec.Id + } + return key } type AddonList struct { @@ -302,8 +249,14 @@ type AddonList struct { } func (a *AddonList) Add(spec *channelsapi.AddonSpec) *Addon { + return a.AddWithSource(spec, nil) +} + +func (a *AddonList) AddWithSource(spec *channelsapi.AddonSpec, source fi.Resource) *Addon { addon := &Addon{ - Spec: spec, + Spec: spec, + Source: source, + SkipRender: source != nil, } a.Items = append(a.Items, addon) return addon @@ -313,11 +266,40 @@ type Addon struct { // Spec is the spec that will (eventually) be passed to the channels binary. Spec *channelsapi.AddonSpec - // ManifestData is the object data loaded from the manifest. - ManifestData []byte + // Source is the manifest template or static bytes used to build the addon file. + Source fi.Resource // BuildPrune is set if we should automatically build prune specifiers, based on the manifest. BuildPrune bool + + // SkipRemap bypasses label stamping, service-account role injection, and asset image remapping. + SkipRemap bool + + // SkipRender is true when Source is already a rendered manifest. + SkipRender bool +} + +// CollectImages renders template sources under stubbed task funcs and scans +// the resulting YAML for image references to pre-register with the builder. +// Best-effort only (used for AWS WarmPool image prewarm): a failure here +// warns rather than breaks the build. +func (a *Addon) CollectImages(assetBuilder *assets.AssetBuilder, renderer AddonTemplateRenderer) error { + if a == nil || assetBuilder == nil || a.Source == nil { + return nil + } + + manifestBytes, err := fi.ResourceAsBytes(a.Source) + if err != nil { + return err + } + if !a.SkipRender && renderer != nil { + manifestBytes, err = renderer.RenderTemplate(addonKey(a.Spec), manifestBytes, nil) + if err != nil { + return fmt.Errorf("rendering addon %q for image discovery: %w", addonKey(a.Spec), err) + } + } + _, err = assetBuilder.RemapManifest(manifestBytes) + return err } func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) (*AddonList, map[types.NamespacedName]iam.Subject, error) { @@ -901,15 +883,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) klog.Infof("replacing arguments in externally provided cloud-controller-manager") - fnAny, ok := b.templates.TemplateFunctions["CloudControllerConfigArgv"] - if !ok { - return nil, nil, fmt.Errorf("unable to find TemplateFunction CloudControllerConfigArgv") - } - fn, ok := fnAny.(func() ([]string, error)) - if !ok { - return nil, nil, fmt.Errorf("unexpected type for TemplateFunction CloudControllerConfigArgv: %T", fnAny) - } - args, err := fn() + args, err := b.addonRenderer.CloudControllerConfigArgv() if err != nil { return nil, nil, fmt.Errorf("in TemplateFunction CloudControllerConfigArgv: %w", err) } diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/pruning.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/pruning.go index fb0b282c34e2e..a9c86b05f0c70 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/pruning.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/pruning.go @@ -30,28 +30,13 @@ import ( "k8s.io/kops/pkg/model/components/addonmanifests" ) -func (b *BootstrapChannelBuilder) addPruneDirectives(addons *AddonList) error { - for _, addon := range addons.Items { - if !addon.BuildPrune { - continue - } - - id := *addon.Spec.Name - - if err := b.addPruneDirectivesForAddon(addon); err != nil { - return fmt.Errorf("failed to configure pruning for %s: %w", id, err) - } - } - return nil -} - -func (b *BootstrapChannelBuilder) addPruneDirectivesForAddon(addon *Addon) error { - addon.Spec.Prune = &channelsapi.PruneSpec{} +func buildPruneDirectives(spec *channelsapi.AddonSpec, manifestData []byte) error { + spec.Prune = &channelsapi.PruneSpec{} // We add these labels to all objects we manage, so we reuse them for pruning. selectorMap := map[string]string{ "app.kubernetes.io/managed-by": "kops", - addonmanifests.KopsAddonLabelKey: *addon.Spec.Name, + addonmanifests.KopsAddonLabelKey: *spec.Name, } selector, err := labels.ValidatedSelectorFromSet(selectorMap) if err != nil { @@ -91,7 +76,7 @@ func (b *BootstrapChannelBuilder) addPruneDirectivesForAddon(addon *Addon) error } // Parse the manifest; we use this to scope pruning to namespaces - objects, err := kubemanifest.LoadObjectsFrom(addon.ManifestData) + objects, err := kubemanifest.LoadObjectsFrom(manifestData) if err != nil { return fmt.Errorf("failed to parse manifest: %w", err) } @@ -147,7 +132,7 @@ func (b *BootstrapChannelBuilder) addPruneDirectivesForAddon(addon *Addon) error pruneSpec.LabelSelector = selector.String() - addon.Spec.Prune.Kinds = append(addon.Spec.Prune.Kinds, pruneSpec) + spec.Prune.Kinds = append(spec.Prune.Kinds, pruneSpec) } return nil diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go index 718fb1909b916..2b598b2804e06 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go @@ -18,6 +18,7 @@ package cloudup import ( "context" + "io" "os" "path" "testing" @@ -35,7 +36,6 @@ import ( "k8s.io/kops/upup/models" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/bootstrapchannelbuilder" - "k8s.io/kops/upup/pkg/fi/fitasks" "k8s.io/kops/util/pkg/vfs" ) @@ -111,7 +111,7 @@ func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) { } cluster = fullSpec - templates, err := templates.LoadTemplates(ctx, cluster, models.NewAssetPath("cloudup/resources")) + templates, err := templates.LoadTemplates(ctx, models.NewAssetPath("cloudup/resources")) if err != nil { t.Fatalf("error building templates for %q: %v", key, err) } @@ -155,11 +155,11 @@ func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) { kopsModel.AllInstanceGroups = kopsModel.InstanceGroups - tf := &TemplateFunctions{ - KopsModelContext: kopsModel, - cloud: cloud, + addonRenderer := &addonTemplateRenderer{ + modelContext: &kopsModel, + cloud: cloud, + secretStore: secretStore, } - tf.AddTo(templates.TemplateFunctions, secretStore) bcb := bootstrapchannelbuilder.NewBootstrapChannelBuilder( &kopsModel, @@ -167,6 +167,7 @@ func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) { assets.NewAssetBuilder(vfs.Context, cluster.Spec.Assets, false), templates, nil, + addonRenderer, ) context := &fi.CloudupModelBuilderContext{ @@ -176,16 +177,44 @@ func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) { if err != nil { t.Fatalf("error from BootstrapChannelBuilder Build: %v", err) } + if context.Tasks["ManagedFile/"+cluster.ObjectMeta.Name+"-addons-bootstrap"] != nil { + t.Fatalf("bootstrap addon manifest should be created as an BootstrapChannel task") + } + if context.Tasks["BootstrapChannel/"+cluster.ObjectMeta.Name+"-addons-bootstrap"] == nil { + t.Fatalf("bootstrap addons channel task not found") + } + + target := fi.NewCloudupDryRunTarget(assets.NewAssetBuilder(vfs.Context, cluster.Spec.Assets, false), false, io.Discard) + cloudupContext, err := fi.NewCloudupContext(ctx, fi.DeletionProcessingModeDeleteIncludingDeferred, target, cluster, cloud, nil, secretStore, basePath, context.Tasks) + if err != nil { + t.Fatalf("error building cloudup context: %v", err) + } { name := cluster.ObjectMeta.Name + "-addons-bootstrap" - manifestTask := context.Tasks["ManagedFile/"+name] + manifestTask := context.Tasks["BootstrapChannel/"+name] if manifestTask == nil { t.Fatalf("manifest task not found (%q)", name) } - manifestFileTask := manifestTask.(*fitasks.ManagedFile) - actualManifest, err := fi.ResourceAsString(manifestFileTask.Contents) + bootstrapChannelTask, ok := manifestTask.(*bootstrapchannelbuilder.BootstrapChannel) + if !ok { + t.Fatalf("expected BootstrapChannel task, got %T", manifestTask) + } + for _, dependency := range bootstrapChannelTask.GetDependencies(context.Tasks) { + addonManifestTask, ok := dependency.(*bootstrapchannelbuilder.AddonManifest) + if !ok { + t.Fatalf("expected AddonManifest dependency, got %T", dependency) + } + if err := addonManifestTask.Normalize(cloudupContext); err != nil { + t.Fatalf("error normalizing addon manifest %q: %v", fi.ValueOf(addonManifestTask.Name), err) + } + } + if err := bootstrapChannelTask.Normalize(cloudupContext); err != nil { + t.Fatalf("error normalizing addons channel: %v", err) + } + + actualManifest, err := fi.ResourceAsString(bootstrapChannelTask.Contents) if err != nil { t.Fatalf("error getting manifest as string: %v", err) } @@ -196,7 +225,7 @@ func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) { for _, k := range addonManifests { name := cluster.ObjectMeta.Name + "-addons-" + k - manifestTask := context.Tasks["ManagedFile/"+name] + manifestTask := context.Tasks["AddonManifest/"+name] if manifestTask == nil { for k := range context.Tasks { t.Logf("found task %s", k) @@ -204,8 +233,12 @@ func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) { t.Fatalf("manifest task not found (%q)", name) } - manifestFileTask := manifestTask.(*fitasks.ManagedFile) - actualManifest, err := fi.ResourceAsString(manifestFileTask.Contents) + addonManifestTask, ok := manifestTask.(*bootstrapchannelbuilder.AddonManifest) + if !ok { + t.Fatalf("expected AddonManifest task, got %T", manifestTask) + } + + actualManifest, err := fi.ResourceAsString(addonManifestTask.Contents) if err != nil { t.Fatalf("error getting manifest as string: %v", err) } diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 39da372d12e87..5d3dcd1aa1b24 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -28,6 +28,7 @@ When defining a new function: package cloudup import ( + "bytes" "encoding/base64" "encoding/json" "fmt" @@ -78,6 +79,57 @@ type TemplateFunctions struct { model.KopsModelContext cloud fi.Cloud + + tasks map[string]fi.CloudupTask +} + +// addonTemplateRenderer constructs a fresh TemplateFunctions per addon render, +// so each addon sees its own task-bound func map. +type addonTemplateRenderer struct { + modelContext *model.KopsModelContext + cloud fi.Cloud + secretStore fi.SecretStore +} + +func (r *addonTemplateRenderer) newTemplateFunctions(tasks map[string]fi.CloudupTask) *TemplateFunctions { + return &TemplateFunctions{ + KopsModelContext: *r.modelContext, + cloud: r.cloud, + tasks: tasks, + } +} + +// RenderTemplate parses and executes an addon template source against a func map +// derived from a per-call *TemplateFunctions bound to the given task graph. +// When tasks is nil, task-based functions return empty stubs so templates still +// render — used for Build-time image discovery before the task graph exists. +func (r *addonTemplateRenderer) RenderTemplate(name string, source []byte, tasks map[string]fi.CloudupTask) ([]byte, error) { + tf := r.newTemplateFunctions(tasks) + funcMap := template.FuncMap{} + if err := tf.AddTo(funcMap, r.secretStore); err != nil { + return nil, err + } + if tasks == nil { + funcMap["Task"] = func(typeName, name string) (fi.CloudupTask, error) { return nil, nil } + funcMap["HasTask"] = func(typeName, name string) bool { return false } + funcMap["TasksByType"] = func(typeName string) ([]fi.CloudupTask, error) { return nil, nil } + } + + t := template.New(name).Funcs(funcMap).Option("missingkey=zero") + if _, err := t.Parse(string(source)); err != nil { + return nil, fmt.Errorf("error parsing template %q: %w", name, err) + } + + var buf bytes.Buffer + if err := t.ExecuteTemplate(&buf, name, r.modelContext.Cluster.Spec); err != nil { + return nil, fmt.Errorf("error executing template %q: %w", name, err) + } + return buf.Bytes(), nil +} + +// CloudControllerConfigArgv returns the cloud controller argv without binding any task graph. +func (r *addonTemplateRenderer) CloudControllerConfigArgv() ([]string, error) { + return r.newTemplateFunctions(nil).CloudControllerConfigArgv() } // AddTo defines the available functions we can use in our YAML models. @@ -124,6 +176,10 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS dest["GetInstanceGroup"] = tf.GetInstanceGroup dest["GetNodeInstanceGroups"] = tf.GetNodeInstanceGroups dest["GetClusterAutoscalerNodeGroups"] = tf.GetClusterAutoscalerNodeGroups + dest["Task"] = tf.Task + dest["HasTask"] = tf.HasTask + dest["TasksByType"] = tf.TasksByType + dest["TaskKey"] = tf.TaskKey dest["HasHighlyAvailableControlPlane"] = tf.HasHighlyAvailableControlPlane dest["ControlPlaneControllerReplicas"] = tf.ControlPlaneControllerReplicas dest["APIServerNodeRole"] = tf.APIServerNodeRole @@ -455,6 +511,75 @@ func (tf *TemplateFunctions) ToYAML(data interface{}) string { return string(encoded) } +func (tf *TemplateFunctions) taskMap() (map[string]fi.CloudupTask, error) { + if tf.tasks == nil { + return nil, fmt.Errorf("template tasks are not available during this render phase") + } + return tf.tasks, nil +} + +// Task returns a task by type and name, for example Task "IAMRole" "nodes.example.com". +func (tf *TemplateFunctions) Task(typeName, name string) (fi.CloudupTask, error) { + tasks, err := tf.taskMap() + if err != nil { + return nil, err + } + + key := typeName + "/" + name + task := tasks[key] + if task == nil { + return nil, fmt.Errorf("task %q not found", key) + } + return task, nil +} + +// HasTask reports whether the named task exists in the final task graph. +func (tf *TemplateFunctions) HasTask(typeName, name string) bool { + if tf.tasks == nil { + return false + } + return tf.tasks[typeName+"/"+name] != nil +} + +// TasksByType returns tasks of a specific type in deterministic task-key order. +func (tf *TemplateFunctions) TasksByType(typeName string) ([]fi.CloudupTask, error) { + tasks, err := tf.taskMap() + if err != nil { + return nil, err + } + + prefix := typeName + "/" + var keys []string + for key := range tasks { + if strings.HasPrefix(key, prefix) { + keys = append(keys, key) + } + } + sort.Strings(keys) + + matches := make([]fi.CloudupTask, 0, len(keys)) + for _, key := range keys { + matches = append(matches, tasks[key]) + } + return matches, nil +} + +// TaskKey returns the canonical task key used in the task map. +func (tf *TemplateFunctions) TaskKey(task fi.CloudupTask) (string, error) { + if task == nil { + return "", fmt.Errorf("task is nil") + } + hasName, ok := task.(fi.HasName) + if !ok { + return "", fmt.Errorf("task %T does not implement HasName", task) + } + name := fi.ValueOf(hasName.GetName()) + if name == "" { + return "", fmt.Errorf("task %T did not have a name", task) + } + return fi.TypeNameForTask(task) + "/" + name, nil +} + // SharedVPC is a simple helper function which makes the templates for a shared VPC clearer func (tf *TemplateFunctions) SharedVPC() bool { return tf.Cluster.SharedVPC() diff --git a/upup/pkg/fi/cloudup/template_functions_test.go b/upup/pkg/fi/cloudup/template_functions_test.go index 64dd0cd50f3af..5a0aba294dba2 100644 --- a/upup/pkg/fi/cloudup/template_functions_test.go +++ b/upup/pkg/fi/cloudup/template_functions_test.go @@ -17,14 +17,17 @@ limitations under the License. package cloudup import ( + "bytes" "fmt" "reflect" "testing" + "text/template" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/fitasks" ) func Test_TemplateFunctions_CloudControllerConfigArgv(t *testing.T) { @@ -441,3 +444,69 @@ func TestHasHighlyAvailableControlPlane(t *testing.T) { }) } } + +func TestTemplateFunctions_TaskHelpers(t *testing.T) { + tf := &TemplateFunctions{} + tf.Cluster = &kops.Cluster{} + tf.tasks = map[string]fi.CloudupTask{ + "ManagedFile/zeta": &fitasks.ManagedFile{ + Name: fi.PtrTo("zeta"), + Location: fi.PtrTo("addons/zeta.yaml"), + }, + "ManagedFile/alpha": &fitasks.ManagedFile{ + Name: fi.PtrTo("alpha"), + Location: fi.PtrTo("addons/alpha.yaml"), + }, + } + + if !tf.HasTask("ManagedFile", "alpha") { + t.Fatalf("expected alpha task to exist") + } + if tf.HasTask("ManagedFile", "missing") { + t.Fatalf("did not expect missing task to exist") + } + + task, err := tf.Task("ManagedFile", "alpha") + if err != nil { + t.Fatalf("Task returned error: %v", err) + } + if key, err := tf.TaskKey(task); err != nil || key != "ManagedFile/alpha" { + t.Fatalf("unexpected task key %q, err=%v", key, err) + } + + tasks, err := tf.TasksByType("ManagedFile") + if err != nil { + t.Fatalf("TasksByType returned error: %v", err) + } + var gotKeys []string + for _, task := range tasks { + key, err := tf.TaskKey(task) + if err != nil { + t.Fatalf("TaskKey returned error: %v", err) + } + gotKeys = append(gotKeys, key) + } + expectedKeys := []string{"ManagedFile/alpha", "ManagedFile/zeta"} + if !reflect.DeepEqual(gotKeys, expectedKeys) { + t.Fatalf("unexpected task order %v", gotKeys) + } + + funcMap := template.FuncMap{} + if err := tf.AddTo(funcMap, nil); err != nil { + t.Fatalf("AddTo returned error: %v", err) + } + + tmpl, err := template.New("tasks").Funcs(funcMap).Parse(`{{ TaskKey (Task "ManagedFile" "alpha") }}|{{ range $task := TasksByType "ManagedFile" }}{{ TaskKey $task }};{{ end }}`) + if err != nil { + t.Fatalf("error parsing template: %v", err) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, tf.Cluster.Spec); err != nil { + t.Fatalf("error executing template: %v", err) + } + + if actual := buf.String(); actual != "ManagedFile/alpha|ManagedFile/alpha;ManagedFile/zeta;" { + t.Fatalf("unexpected rendered template %q", actual) + } +}