diff --git a/.chloggen/standalone-mode.yaml b/.chloggen/standalone-mode.yaml new file mode 100644 index 0000000000..8ca2aa9a32 --- /dev/null +++ b/.chloggen/standalone-mode.yaml @@ -0,0 +1,18 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action) +component: opamp-bridge + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: OpAMP Bridge standalone mode + +# One or more tracking issues related to the change +issues: [4913] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Standalone mode for OpAMP Bridge allows users to manage collector configuration from a remote + OpAMP server without the need to deploy full Otel Operator. diff --git a/Makefile b/Makefile index 6f23e1c139..07b8d50305 100644 --- a/Makefile +++ b/Makefile @@ -347,6 +347,28 @@ deploy-no-crds: set-image-controller undeploy-no-crds: set-image-controller $(KUSTOMIZE) build config/no-crds | kubectl delete --ignore-not-found=$(ignore-not-found) -f - +##@ Standalone OpAMP Bridge (no operator / CRDs required) + +STANDALONE_BRIDGE_MANIFESTS ?= cmd/operator-opamp-bridge/manifests/standalone + +# Deploy the standalone OpAMP bridge into the current Kubernetes context. +# Does not require the operator, CRDs, or cert-manager. +.PHONY: deploy-standalone-bridge +deploy-standalone-bridge: kustomize + cd $(STANDALONE_BRIDGE_MANIFESTS) && $(KUSTOMIZE) edit set image operator-opamp-bridge=${OPERATOROPAMPBRIDGE_IMG} + $(KUSTOMIZE) build $(STANDALONE_BRIDGE_MANIFESTS) | kubectl apply -f - + kubectl rollout status deployment/otel-opamp-bridge-standalone -n opentelemetry-opamp-bridge --timeout=120s + +# Undeploy the standalone OpAMP bridge from the current Kubernetes context. +.PHONY: undeploy-standalone-bridge +undeploy-standalone-bridge: kustomize + $(KUSTOMIZE) build $(STANDALONE_BRIDGE_MANIFESTS) | kubectl delete --ignore-not-found=true -f - + +# Build, load, and deploy the standalone bridge to a kind cluster. +# Assumes a kind cluster is already running (use start-kind first). +.PHONY: deploy-standalone-bridge-kind +deploy-standalone-bridge-kind: load-image-operator-opamp-bridge deploy-standalone-bridge + # Generates the released manifests .PHONY: release-artifacts release-artifacts: set-image-controller @@ -463,7 +485,7 @@ e2e-multi-instrumentation: chainsaw # OpAMPBridge CR end-to-tests .PHONY: e2e-opampbridge e2e-opampbridge: chainsaw - $(CHAINSAW) test --test-dir ./tests/e2e-opampbridge --report-name e2e-opampbridge + OPERATOROPAMPBRIDGE_IMG=$(OPERATOROPAMPBRIDGE_IMG) $(CHAINSAW) test --test-dir ./tests/e2e-opampbridge --report-name e2e-opampbridge # end-to-end-test for testing pdb support .PHONY: e2e-pdb diff --git a/bundle/community/manifests/opentelemetry-operator.clusterserviceversion.yaml b/bundle/community/manifests/opentelemetry-operator.clusterserviceversion.yaml index ca544ea08a..8257f4d55d 100644 --- a/bundle/community/manifests/opentelemetry-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/opentelemetry-operator.clusterserviceversion.yaml @@ -99,7 +99,7 @@ metadata: categories: Logging & Tracing,Monitoring,Observability certified: "false" containerImage: ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator - createdAt: "2026-06-13T17:24:50Z" + createdAt: "2026-06-17T17:01:16Z" description: Provides the OpenTelemetry components, including the Collector operators.operatorframework.io/builder: operator-sdk-v1.29.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 diff --git a/bundle/openshift/manifests/opentelemetry-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/opentelemetry-operator.clusterserviceversion.yaml index 198307e803..84c96552af 100644 --- a/bundle/openshift/manifests/opentelemetry-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/opentelemetry-operator.clusterserviceversion.yaml @@ -99,7 +99,7 @@ metadata: categories: Logging & Tracing,Monitoring,Observability certified: "false" containerImage: ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator - createdAt: "2026-06-13T17:24:51Z" + createdAt: "2026-06-17T17:01:17Z" description: Provides the OpenTelemetry components, including the Collector operators.operatorframework.io/builder: operator-sdk-v1.29.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 diff --git a/cmd/operator-opamp-bridge/README.md b/cmd/operator-opamp-bridge/README.md index 3da9986a65..828d9b9014 100644 --- a/cmd/operator-opamp-bridge/README.md +++ b/cmd/operator-opamp-bridge/README.md @@ -20,6 +20,67 @@ There are two main ways to install the OpAMP Bridge: ## Usage +### Standalone mode + +Standalone mode lets the bridge manage Collector configuration stored in Kubernetes `ConfigMap` resources, without creating `OpenTelemetryCollector` CRDs. This is useful when the Collector workload is managed outside the operator, but the config still needs to be reported to and updated from an OpAMP server. + +Start the bridge with `mode: standalone` in its config file, or pass `--mode=standalone`: + +```yaml +endpoint: "" +mode: standalone +healthListenAddr: ":8081" +capabilities: + AcceptsRemoteConfig: true + ReportsEffectiveConfig: true + ReportsRemoteConfig: true +standalone: + agents: + - namespace: default + type: otel-collector + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: my-collector + config: + collector: + kind: configmap + name: collector-config + key: collector.yaml +``` + +In this mode, the bridge creates one OpAMP client connection for each entry under `standalone.agents`. Each key under an agent's `config` section is the OpAMP config file name reported to the server. The value describes the local Kubernetes resource that backs that file. Config resources are resolved in the workload namespace. In the example above, the OpAMP server sees a config file named `collector`, and the bridge maps it locally to `ConfigMap/default/collector-config`, key `collector.yaml`. + +After applying a config update, the bridge restarts the configured workload by updating the workload pod template's `kubectl.kubernetes.io/restartedAt` annotation. Supported workload refs are `apps/v1` `Deployment`, `DaemonSet`, and `StatefulSet`. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: collector-config + namespace: default +data: + collector.yaml: | + receivers: + otlp: + protocols: + grpc: + http: + exporters: + otlphttp: + endpoint: http://example-collector:4318 + service: + pipelines: + traces: + receivers: [otlp] + exporters: [otlphttp] +``` + + +The bridge will not create or delete ConfigMaps in standalone mode. Remote config updates are only applied to the configured local resource and key. + +Standalone mode needs RBAC for ConfigMaps and configured workload types. The repository includes a starter manifest at [`manifests/standalone/rbac.yaml`](manifests/standalone/rbac.yaml). + ### OpAMPBridge CRD The [OpAMPBridge](../../docs/api/opampbridges.md) CRD is used to create an OpAMP Bridge instance. diff --git a/cmd/operator-opamp-bridge/internal/agent/agent.go b/cmd/operator-opamp-bridge/internal/agent/agent.go index 6008a4131c..e144a9a122 100644 --- a/cmd/operator-opamp-bridge/internal/agent/agent.go +++ b/cmd/operator-opamp-bridge/internal/agent/agent.go @@ -19,9 +19,7 @@ import ( "github.com/open-telemetry/opamp-go/client/types" "github.com/open-telemetry/opamp-go/protobufs" "k8s.io/utils/clock" - "sigs.k8s.io/yaml" - "github.com/open-telemetry/opentelemetry-operator/apis/v1beta1" "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/config" "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/metrics" "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/operator" @@ -31,7 +29,7 @@ import ( type Agent struct { logger logr.Logger - appliedKeys map[kubeResourceKey]bool + appliedKeys map[string]bool clock clock.Clock startTime uint64 lastHash []byte @@ -61,7 +59,7 @@ func NewAgent(logger logr.Logger, applier operator.ConfigApplier, cfg *config.Co applier: applier, proxy: p, logger: logger, - appliedKeys: map[kubeResourceKey]bool{}, + appliedKeys: map[string]bool{}, instanceId: cfg.GetInstanceId(), agentDescription: cfg.GetDescription(), remoteConfigEnabled: cfg.RemoteConfigEnabled(), @@ -81,7 +79,7 @@ func NewAgent(logger logr.Logger, applier operator.ConfigApplier, cfg *config.Co // getHealth is called every heartbeat interval to report health. func (agent *Agent) getHealth() *protobufs.ComponentHealth { - healthMap, err := agent.generateCollectorPoolHealth() + health, err := agent.applier.GetHealth() if err != nil { return &protobufs.ComponentHealth{ Healthy: false, @@ -97,124 +95,80 @@ func (agent *Agent) getHealth() *protobufs.ComponentHealth { LastError: err.Error(), } } - return &protobufs.ComponentHealth{ - Healthy: true, - StartTimeUnixNano: agent.startTime, - StatusTimeUnixNano: statusTime, - LastError: "", - ComponentHealthMap: healthMap, + componentHealth, err := agent.componentHealthFromInternal(health, statusTime, true) + if err != nil { + return &protobufs.ComponentHealth{ + Healthy: false, + StartTimeUnixNano: agent.startTime, + LastError: err.Error(), + } } + agent.addProxyHealth(componentHealth) + return componentHealth } -// generateCollectorPoolHealth allows the bridge to report the status of the collector pools it owns. -// TODO: implement enhanced health messaging using the collector's new healthcheck extension: -// https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/26661 -func (agent *Agent) generateCollectorPoolHealth() (map[string]*protobufs.ComponentHealth, error) { - cols, err := agent.applier.ListInstances() - if err != nil { - return nil, err +// componentHealthFromInternal converts the bridge's internal recursive Health tree into the OpAMP protobuf shape. +// statusTime is applied to every component in the tree; root controls whether the bridge start time or the +// component-specific start time is reported. +func (agent *Agent) componentHealthFromInternal(health operator.Health, statusTime uint64, root bool) (*protobufs.ComponentHealth, error) { + startTime := uint64(0) + if root { + startTime = agent.startTime + } else if !health.StartTime.IsZero() { + var err error + startTime, err = timeToUnixNanoUnsigned(health.StartTime) + if err != nil { + return nil, err + } + } + componentHealthMap := map[string]*protobufs.ComponentHealth{} + for key, child := range health.Children { + childHealth, err := agent.componentHealthFromInternal(child, statusTime, false) + if err != nil { + return nil, err + } + componentHealthMap[key] = childHealth } + return &protobufs.ComponentHealth{ + Healthy: health.Healthy, + StartTimeUnixNano: startTime, + StatusTimeUnixNano: statusTime, + Status: health.Status, + LastError: health.LastError, + ComponentHealthMap: componentHealthMap, + }, nil +} + +// addProxyHealth merges directly connected collector agents into the reported health tree. +// rootHealth is mutated in place: proxy agents are attached beneath matching pods by hostname, and any +// unmatched proxy agents are reported as root-level components. +func (agent *Agent) addProxyHealth(rootHealth *protobufs.ComponentHealth) { proxyHealth := agent.proxy.GetHealth() agentsByHostName := agent.proxy.GetAgentsByHostname() - healthMap := map[string]*protobufs.ComponentHealth{} proxiesUsed := make(map[uuid.UUID]struct{}, len(agentsByHostName)) - for _, col := range cols { - key := newKubeResourceKey(col.GetNamespace(), col.GetName()) - podMap, err := agent.generateCollectorHealth(agent.getCollectorSelector(col), col.GetNamespace()) - if err != nil { - return nil, err + for collectorName, collector := range rootHealth.ComponentHealthMap { + namespace, _, ok := strings.Cut(collectorName, "/") + if !ok { + continue } - isPoolHealthy := true - for podName, pod := range podMap { + for podName, pod := range collector.ComponentHealthMap { // we need to remove the prefix here as we don't have the namespace from the hostname. - podNameWithoutNamespace := strings.TrimPrefix(podName, col.GetNamespace()+"/") - isPoolHealthy = isPoolHealthy && pod.Healthy + podNameWithoutNamespace := strings.TrimPrefix(podName, namespace+"/") if uid, ok := agentsByHostName[podNameWithoutNamespace]; ok { - podMap[podName].ComponentHealthMap[uid.String()] = proxyHealth[uid] + if pod.ComponentHealthMap == nil { + pod.ComponentHealthMap = map[string]*protobufs.ComponentHealth{} + } + pod.ComponentHealthMap[uid.String()] = proxyHealth[uid] proxiesUsed[uid] = struct{}{} } } - podStartTime, err := timeToUnixNanoUnsigned(col.ObjectMeta.GetCreationTimestamp().Time) - if err != nil { - return nil, err - } - statusTime, err := agent.getCurrentTimeUnixNano() - if err != nil { - return nil, err - } - healthMap[key.String()] = &protobufs.ComponentHealth{ - StartTimeUnixNano: podStartTime, - StatusTimeUnixNano: statusTime, - Status: col.Status.Scale.StatusReplicas, - ComponentHealthMap: podMap, - Healthy: isPoolHealthy, - } } - for instance, health := range proxyHealth { + for instance, instanceHealth := range proxyHealth { if _, ok := proxiesUsed[instance]; ok { continue } - healthMap[instance.String()] = health - } - return healthMap, nil -} - -// getCollectorSelector destructures the collectors scale selector if present, it uses the labelmap from the operator. -func (*Agent) getCollectorSelector(col v1beta1.OpenTelemetryCollector) map[string]string { - if col.Status.Scale.Selector != "" { - selMap := map[string]string{} - for kvPair := range strings.SplitSeq(col.Status.Scale.Selector, ",") { - kv := strings.Split(kvPair, "=") - // skip malformed pairs - if len(kv) != 2 { - continue - } - selMap[kv[0]] = kv[1] - } - return selMap - } - return map[string]string{ - "app.kubernetes.io/managed-by": "opentelemetry-operator", - "app.kubernetes.io/instance": fmt.Sprintf("%s.%s", col.GetNamespace(), col.GetName()), - "app.kubernetes.io/part-of": "opentelemetry", - "app.kubernetes.io/component": "opentelemetry-collector", - } -} - -func (agent *Agent) generateCollectorHealth(selectorLabels map[string]string, namespace string) (map[string]*protobufs.ComponentHealth, error) { - statusTime, err := agent.getCurrentTimeUnixNano() - if err != nil { - return nil, err - } - pods, err := agent.applier.GetCollectorPods(selectorLabels, namespace) - if err != nil { - return nil, err - } - healthMap := map[string]*protobufs.ComponentHealth{} - for _, item := range pods.Items { - key := newKubeResourceKey(item.GetNamespace(), item.GetName()) - healthy := true - if item.Status.Phase != "Running" { - healthy = false - } - var startTime uint64 - if item.Status.StartTime != nil { - startTime, err = timeToUnixNanoUnsigned(item.Status.StartTime.Time) - if err != nil { - return nil, err - } - } else { - healthy = false - } - healthMap[key.String()] = &protobufs.ComponentHealth{ - StartTimeUnixNano: startTime, - StatusTimeUnixNano: statusTime, - Status: string(item.Status.Phase), - Healthy: healthy, - ComponentHealthMap: map[string]*protobufs.ComponentHealth{}, - } + rootHealth.ComponentHealthMap[instance.String()] = instanceHealth } - return healthMap, nil } // onConnect is called when an agent is successfully connected to a server. @@ -362,6 +316,11 @@ func (agent *Agent) runHeartbeat() { } } +// UpdateHealth sends the current health state to the OpAMP server. +func (agent *Agent) UpdateHealth() error { + return agent.opampClient.SetHealth(agent.getHealth()) +} + // updateAgentIdentity receives a new instancedId from the remote server and updates the agent's instanceID field. // The meter will be reinitialized by the onMessage function. func (agent *Agent) updateAgentIdentity(instanceId uuid.UUID) { @@ -381,20 +340,23 @@ func (agent *Agent) getEffectiveConfig(context.Context) (*protobufs.EffectiveCon } instanceMap := map[string]*protobufs.AgentConfigFile{} for _, instance := range instances { - // Skip collectors pending deletion if instance.GetDeletionTimestamp() != nil { continue } - col := instance - marshaled, err := yaml.Marshal(&col) - if err != nil { - agent.logger.Error(err, "failed to marhsal config") - return nil, err - } - mapKey := newKubeResourceKey(instance.GetNamespace(), instance.GetName()) - instanceMap[mapKey.String()] = &protobufs.AgentConfigFile{ - Body: marshaled, - ContentType: "yaml", + for key, file := range instance.GetConfigMap() { + if file.Body == nil { + agent.logger.Error(errors.New("nil effective config"), "failed to get effective config", + "name", instance.GetName(), "namespace", instance.GetNamespace(), "key", key) + continue + } + contentType := file.ContentType + if contentType == "" { + contentType = "yaml" + } + instanceMap[key] = &protobufs.AgentConfigFile{ + Body: file.Body, + ContentType: contentType, + } } } for id, instance := range agent.proxy.GetConfigurations() { @@ -427,37 +389,35 @@ func (agent *Agent) initMeter(settings *protobufs.TelemetryConnectionSettings) { // applyRemoteConfig receives a remote configuration from a remote server of the following form: // -// map[name/namespace] -> collector CRD spec +// map[resource key] -> AgentConfigFile body // -// For every key in the received remote configuration, the agent attempts to apply it to the connected -// Kubernetes cluster. If an agent fails to apply a collector CRD, it will continue to the next entry. The agent will -// store the received configuration hash regardless of application status as per the OpAMP spec. +// For every key in the received remote configuration, the agent attempts to apply it via the configured +// applier. If an entry fails to apply, the agent continues to the next entry. The agent stores the +// received configuration hash regardless of application status, as per the OpAMP spec. // // INVARIANT: The caller must verify that config isn't nil _and_ the configuration has changed between calls. func (agent *Agent) applyRemoteConfig(config *protobufs.AgentRemoteConfig) (*protobufs.RemoteConfigStatus, error) { var errs []error // Apply changes from the received config map for key, file := range config.Config.GetConfigMap() { - if key == "" || len(file.Body) == 0 { + if key == "" { + errs = append(errs, errors.New("remote config entry has empty name")) continue } - colKey, err := kubeResourceFromKey(key) - if err != nil { - errs = append(errs, err) + if len(file.Body) == 0 { + errs = append(errs, fmt.Errorf("remote config entry %q has empty body", key)) continue } - err = agent.applier.Apply(colKey.name, colKey.namespace, file) - if err != nil { + if err := agent.applier.Apply(key, file); err != nil { errs = append(errs, err) continue } - agent.appliedKeys[colKey] = true + agent.appliedKeys[key] = true } // Check if anything was deleted - for collectorKey := range agent.appliedKeys { - if _, ok := config.Config.GetConfigMap()[collectorKey.String()]; !ok { - err := agent.applier.Delete(collectorKey.name, collectorKey.namespace) - if err != nil { + for key := range agent.appliedKeys { + if _, ok := config.Config.GetConfigMap()[key]; !ok { + if err := agent.applier.Delete(key); err != nil { errs = append(errs, err) } } diff --git a/cmd/operator-opamp-bridge/internal/agent/agent_test.go b/cmd/operator-opamp-bridge/internal/agent/agent_test.go index d98dc88e9d..c06cb127ec 100644 --- a/cmd/operator-opamp-bridge/internal/agent/agent_test.go +++ b/cmd/operator-opamp-bridge/internal/agent/agent_test.go @@ -5,6 +5,7 @@ package agent import ( "context" + "errors" "fmt" "os" "slices" @@ -275,6 +276,166 @@ func getFakeApplier(t *testing.T, conf *config.Config, lists ...runtimeClient.Ob return operator.NewClient("test-bridge", l, c.Build(), conf.GetComponentsAllowed()) } +type mockHealthApplier struct { + health operator.Health + err error +} + +func (*mockHealthApplier) Apply(string, *protobufs.AgentConfigFile) error { + return nil +} + +func (*mockHealthApplier) Delete(string) error { + return nil +} + +func (*mockHealthApplier) ListInstances() ([]operator.CollectorInstance, error) { + return nil, nil +} + +func (m *mockHealthApplier) GetHealth() (operator.Health, error) { + return m.health, m.err +} + +type recordingConfigApplier struct { + applied map[string][]byte +} + +func (r *recordingConfigApplier) Apply(name string, configFile *protobufs.AgentConfigFile) error { + if r.applied == nil { + r.applied = map[string][]byte{} + } + r.applied[name] = configFile.Body + return nil +} + +func (*recordingConfigApplier) Delete(string) error { + return nil +} + +func (*recordingConfigApplier) ListInstances() ([]operator.CollectorInstance, error) { + return nil, nil +} + +func (*recordingConfigApplier) GetHealth() (operator.Health, error) { + return operator.Health{Healthy: true, Children: map[string]operator.Health{}}, nil +} + +func TestAgent_UpdateHealth(t *testing.T) { + mockClient := &mockOpampClient{} + conf := config.NewConfig(logr.Discard()) + applier := getFakeApplier(t, conf) + agent := NewAgent(logr.Discard(), applier, conf, mockClient, newMockProxy(nil, nil, nil)) + + require.NoError(t, agent.UpdateHealth()) + assert.NotNil(t, mockClient.lastHealth) + assert.True(t, mockClient.lastHealth.Healthy) +} + +func TestAgentApplyRemoteConfigRejectsEmptyRemoteName(t *testing.T) { + applier := &recordingConfigApplier{} + agent := NewAgent(logr.Discard(), applier, config.NewConfig(logr.Discard()), &mockOpampClient{}, newMockProxy(nil, nil, nil)) + body := []byte("receivers: {}\n") + + status, err := agent.applyRemoteConfig(&protobufs.AgentRemoteConfig{ + Config: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{ + "": { + Body: body, + }, + }, + }, + ConfigHash: []byte("hash"), + }) + + require.Error(t, err) + require.Equal(t, protobufs.RemoteConfigStatuses_RemoteConfigStatuses_FAILED, status.Status) + assert.Contains(t, status.ErrorMessage, "remote config entry has empty name") + assert.Empty(t, applier.applied) +} + +func TestAgentApplyRemoteConfigRejectsEmptyBody(t *testing.T) { + applier := &recordingConfigApplier{} + agent := NewAgent(logr.Discard(), applier, config.NewConfig(logr.Discard()), &mockOpampClient{}, newMockProxy(nil, nil, nil)) + + status, err := agent.applyRemoteConfig(&protobufs.AgentRemoteConfig{ + Config: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{ + "collector": {}, + }, + }, + ConfigHash: []byte("hash"), + }) + + require.Error(t, err) + require.Equal(t, protobufs.RemoteConfigStatuses_RemoteConfigStatuses_FAILED, status.Status) + assert.Contains(t, status.ErrorMessage, `remote config entry "collector" has empty body`) + assert.Empty(t, applier.applied) +} + +func TestAgent_getHealthFromApplierHealth(t *testing.T) { + fakeClock := testingclock.NewFakeClock(time.Now()) + startTime, err := timeToUnixNanoUnsigned(fakeClock.Now()) + require.NoError(t, err) + childStart := fakeClock.Now().Add(time.Second) + childStartUnix, err := timeToUnixNanoUnsigned(childStart) + require.NoError(t, err) + applier := &mockHealthApplier{ + health: operator.Health{ + Healthy: true, + Status: "root", + Children: map[string]operator.Health{ + "child": { + Healthy: false, + Status: "child-status", + LastError: "child-error", + StartTime: childStart, + Children: map[string]operator.Health{}, + }, + }, + }, + } + agent := NewAgent(logr.Discard(), applier, config.NewConfig(logr.Discard()), &mockOpampClient{}, newMockProxy(nil, nil, nil)) + agent.clock = fakeClock + agent.startTime = startTime + + got := agent.getHealth() + + assert.Equal(t, &protobufs.ComponentHealth{ + Healthy: true, + StartTimeUnixNano: startTime, + StatusTimeUnixNano: startTime, + Status: "root", + ComponentHealthMap: map[string]*protobufs.ComponentHealth{ + "child": { + Healthy: false, + StartTimeUnixNano: childStartUnix, + StatusTimeUnixNano: startTime, + Status: "child-status", + LastError: "child-error", + ComponentHealthMap: map[string]*protobufs.ComponentHealth{}, + }, + }, + }, got) +} + +func TestAgent_getHealthFromApplierError(t *testing.T) { + fakeClock := testingclock.NewFakeClock(time.Now()) + startTime, err := timeToUnixNanoUnsigned(fakeClock.Now()) + require.NoError(t, err) + agent := NewAgent(logr.Discard(), &mockHealthApplier{err: errors.New("health failed")}, config.NewConfig(logr.Discard()), &mockOpampClient{}, newMockProxy(nil, nil, nil)) + agent.clock = fakeClock + agent.startTime = startTime + + got := agent.getHealth() + + assert.Equal(t, &protobufs.ComponentHealth{ + Healthy: false, + StartTimeUnixNano: startTime, + LastError: "health failed", + }, got) +} + func TestAgent_getHealth(t *testing.T) { fakeClock := testingclock.NewFakeClock(time.Now()) mockInstanceId := uuid.New() diff --git a/cmd/operator-opamp-bridge/internal/agent/kube_resource_key.go b/cmd/operator-opamp-bridge/internal/agent/kube_resource_key.go deleted file mode 100644 index 9b2ede4c88..0000000000 --- a/cmd/operator-opamp-bridge/internal/agent/kube_resource_key.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package agent - -import ( - "errors" - "fmt" - "strings" -) - -type kubeResourceKey struct { - name string - namespace string -} - -func newKubeResourceKey(namespace, name string) kubeResourceKey { - return kubeResourceKey{name: name, namespace: namespace} -} - -func kubeResourceFromKey(key string) (kubeResourceKey, error) { - s := strings.Split(key, "/") - // We expect map keys to be of the form name/namespace - if len(s) != 2 { - return kubeResourceKey{}, errors.New("invalid key") - } - return newKubeResourceKey(s[0], s[1]), nil -} - -func (k kubeResourceKey) String() string { - return fmt.Sprintf("%s/%s", k.namespace, k.name) -} diff --git a/cmd/operator-opamp-bridge/internal/config/config.go b/cmd/operator-opamp-bridge/internal/config/config.go index f5f5b1a8b9..e7f5ead9b8 100644 --- a/cmd/operator-opamp-bridge/internal/config/config.go +++ b/cmd/operator-opamp-bridge/internal/config/config.go @@ -7,9 +7,11 @@ import ( "errors" "fmt" "io/fs" + "maps" "net/url" "os" "runtime" + "strings" "time" "github.com/go-logr/logr" @@ -17,7 +19,7 @@ import ( opampclient "github.com/open-telemetry/opamp-go/client" "github.com/open-telemetry/opamp-go/protobufs" "github.com/spf13/pflag" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" @@ -34,7 +36,9 @@ import ( ) const ( - agentType = "io.opentelemetry.operator-opamp-bridge" + agentType = "io.opentelemetry.operator-opamp-bridge" + operatorMode = "operator" + standaloneMode = "standalone" ) var ( @@ -79,6 +83,7 @@ type Config struct { // kubernetes configuration file. KubeConfigFilePath string `yaml:"kubeConfigFilePath,omitempty"` ListenAddr string `yaml:"listenAddr,omitempty"` + HealthListenAddr string `yaml:"healthListenAddr,omitempty"` ClusterConfig *rest.Config `yaml:"-"` RootLogger logr.Logger `yaml:"-"` instanceId uuid.UUID `yaml:"-"` @@ -92,6 +97,11 @@ type Config struct { HeartbeatInterval time.Duration `yaml:"heartbeatInterval,omitempty"` Name string `yaml:"name,omitempty"` AgentDescription AgentDescription `yaml:"description,omitempty"` + Standalone StandaloneConfig `yaml:"standalone,omitempty"` + + // Mode selects the operating mode: "operator" (default) uses OpenTelemetryCollector CRDs, + // "standalone" manages static Kubernetes config sources from this config. + Mode string `yaml:"mode,omitempty"` } // AgentDescription is copied from the OpAMP Extension in the collector. @@ -102,13 +112,49 @@ type AgentDescription struct { NonIdentifyingAttributes map[string]string `yaml:"non_identifying_attributes"` } +type StandaloneConfig struct { + Agents []StandaloneAgentConfig `yaml:"agents,omitempty"` +} + +type StandaloneAgentConfig struct { + Namespace string `yaml:"namespace"` + Type string `yaml:"type"` + WorkloadRef StandaloneWorkloadRef `yaml:"workloadRef"` + Config map[string]StandaloneConfigEntry `yaml:"config"` +} + +type StandaloneWorkloadRef struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Name string `yaml:"name"` +} + +type StandaloneConfigEntryKind string + +const ( + StandaloneConfigEntryKindConfigMap StandaloneConfigEntryKind = "configmap" +) + +func (k StandaloneConfigEntryKind) IsValid() bool { + // use switch case as we add more supported values + return k == StandaloneConfigEntryKindConfigMap +} + +type StandaloneConfigEntry struct { + Kind StandaloneConfigEntryKind `yaml:"kind"` + Name string `yaml:"name"` + Key string `yaml:"key"` +} + func NewConfig(logger logr.Logger) *Config { return &Config{ instanceId: mustGetInstanceId(), Name: opampBridgeName, ListenAddr: defaultServerListenAddr, + HealthListenAddr: defaultHealthListenAddr, HeartbeatInterval: defaultHeartbeatInterval, KubeConfigFilePath: defaultKubeConfigPath, + Mode: defaultMode, RootLogger: logger, } } @@ -158,7 +204,10 @@ func (c *Config) GetAgentScheme() string { return uri.Scheme } -func (*Config) GetAgentType() string { +func (c *Config) GetAgentType() string { + if c.Name != opampBridgeName && c.Mode == standaloneMode { + return c.Name + } return agentType } @@ -177,18 +226,144 @@ func (c *Config) GetDescription() *protobufs.AgentDescription { keyValuePair("service.instance.id", c.GetInstanceId().String()), keyValuePair("service.version", c.GetAgentVersion()), }, - NonIdentifyingAttributes: append( - c.AgentDescription.nonIdentifyingAttributes(), - keyValuePair("os.family", runtime.GOOS), - keyValuePair("host.name", hostname), - ), + NonIdentifyingAttributes: c.AgentDescription.nonIdentifyingAttributes(map[string]string{ + "os.family": runtime.GOOS, + "host.name": hostname, + }), + } +} + +func NewStandaloneAgentConfig(base *Config, agent StandaloneAgentConfig) *Config { + nonIdentifyingAttributes := map[string]string{} + maps.Copy(nonIdentifyingAttributes, base.AgentDescription.NonIdentifyingAttributes) + nonIdentifyingAttributes["k8s.namespace.name"] = agent.Namespace + nonIdentifyingAttributes["k8s.workload.name"] = agent.WorkloadRef.Name + nonIdentifyingAttributes["k8s.workload.type"] = agent.WorkloadRef.Kind + nonIdentifyingAttributes["opentelemetry.io/agent.type"] = agent.Type + nonIdentifyingAttributes["host.name"] = agent.WorkloadRef.Name + + headers := Headers{} + maps.Copy(headers, base.Headers) + + capabilities := map[Capability]bool{} + maps.Copy(capabilities, base.Capabilities) + + return &Config{ + KubeConfigFilePath: base.KubeConfigFilePath, + ListenAddr: base.ListenAddr, + HealthListenAddr: base.HealthListenAddr, + ClusterConfig: base.ClusterConfig, + RootLogger: base.RootLogger, + instanceId: uuid.NewSHA1(uuid.NameSpaceURL, fmt.Appendf(nil, "%s/%s/%s/%s", agent.Namespace, agent.WorkloadRef.Kind, agent.WorkloadRef.Name, agent.Type)), + ComponentsAllowed: cloneStringSliceMap(base.ComponentsAllowed), + Endpoint: base.Endpoint, + Headers: headers, + Capabilities: capabilities, + HeartbeatInterval: base.HeartbeatInterval, + Name: agent.WorkloadRef.Name, + AgentDescription: AgentDescription{ + NonIdentifyingAttributes: nonIdentifyingAttributes, + }, + Mode: standaloneMode, } } -func (ad *AgentDescription) nonIdentifyingAttributes() []*protobufs.KeyValue { - toReturn := make([]*protobufs.KeyValue, len(ad.NonIdentifyingAttributes)) +func cloneStringSliceMap(in map[string][]string) map[string][]string { + if in == nil { + return nil + } + out := make(map[string][]string, len(in)) + for key, values := range in { + out[key] = append([]string(nil), values...) + } + return out +} + +func (c *Config) Validate() error { + // Normalize empty mode (e.g. when Config is constructed directly without + // NewConfig, or when mode is set to an explicit empty string) to the + // documented default. This keeps logs/state consistent with the flag default. + if c.Mode == "" { + c.Mode = defaultMode + } + switch c.Mode { + case operatorMode, standaloneMode: + default: + return fmt.Errorf("invalid mode %q: must be %q or %q", c.Mode, operatorMode, standaloneMode) + } + if !c.IsStandaloneMode() { + return nil + } + if len(c.Standalone.Agents) == 0 { + return errors.New("standalone mode requires at least one configured agent") + } + agents := map[string]struct{}{} + for _, agent := range c.Standalone.Agents { + if strings.TrimSpace(agent.WorkloadRef.APIVersion) == "" { + return errors.New("standalone agent workloadRef.apiVersion is required") + } + if !strings.EqualFold(agent.WorkloadRef.APIVersion, "apps/v1") { + return fmt.Errorf("standalone agent workloadRef.apiVersion %q is unsupported", agent.WorkloadRef.APIVersion) + } + if strings.TrimSpace(agent.WorkloadRef.Kind) == "" { + return errors.New("standalone agent workloadRef.kind is required") + } + if strings.TrimSpace(agent.WorkloadRef.Name) == "" { + return errors.New("standalone agent workloadRef.name is required") + } + if strings.TrimSpace(agent.Namespace) == "" { + return fmt.Errorf("standalone agent %q namespace is required", agent.WorkloadRef.Name) + } + if strings.TrimSpace(agent.Type) == "" { + return fmt.Errorf("standalone agent %q type is required", agent.WorkloadRef.Name) + } + if !supportedStandaloneWorkloadKind(agent.WorkloadRef.Kind) { + return fmt.Errorf("standalone agent %q has unsupported workloadRef.kind %q", agent.WorkloadRef.Name, agent.WorkloadRef.Kind) + } + agentKey := fmt.Sprintf("%s/%s/%s", agent.Namespace, strings.ToLower(agent.WorkloadRef.Kind), agent.WorkloadRef.Name) + if _, ok := agents[agentKey]; ok { + return fmt.Errorf("duplicate standalone agent workload %q", agentKey) + } + agents[agentKey] = struct{}{} + if len(agent.Config) == 0 { + return fmt.Errorf("standalone agent %q requires at least one config entry", agent.WorkloadRef.Name) + } + for remoteName, entry := range agent.Config { + if strings.TrimSpace(remoteName) == "" { + return fmt.Errorf("standalone agent %q config remote name is required", agent.WorkloadRef.Name) + } + if !entry.Kind.IsValid() { + return fmt.Errorf("standalone agent %q config %q has unsupported kind %q", agent.WorkloadRef.Name, remoteName, entry.Kind) + } + if strings.TrimSpace(entry.Name) == "" { + return fmt.Errorf("standalone agent %q config %q name is required", agent.WorkloadRef.Name, remoteName) + } + if strings.TrimSpace(entry.Key) == "" { + return fmt.Errorf("standalone agent %q config %q key is required", agent.WorkloadRef.Name, remoteName) + } + } + } + return nil +} + +func supportedStandaloneWorkloadKind(workloadKind string) bool { + switch strings.ToLower(workloadKind) { + case "deployment", "daemonset", "statefulset": + return true + default: + return false + } +} + +// nonIdentifyingAttributes overlays configured non-identifying attributes on top of defaults for OpAMP reporting. +func (ad *AgentDescription) nonIdentifyingAttributes(defaults map[string]string) []*protobufs.KeyValue { + attrs := map[string]string{} + maps.Copy(attrs, defaults) + maps.Copy(attrs, ad.NonIdentifyingAttributes) + + toReturn := make([]*protobufs.KeyValue, len(attrs)) i := 0 - for k, v := range ad.NonIdentifyingAttributes { + for k, v := range attrs { toReturn[i] = keyValuePair(k, v) i++ } @@ -226,15 +401,25 @@ func (c *Config) RemoteConfigEnabled() bool { } func (c *Config) GetKubernetesClient() (client.Client, error) { - err := schemeBuilder.AddToScheme(scheme.Scheme) - if err != nil { - return nil, err + if c.Mode != standaloneMode { + err := schemeBuilder.AddToScheme(scheme.Scheme) + if err != nil { + return nil, err + } } return client.New(c.ClusterConfig, client.Options{ Scheme: scheme.Scheme, }) } +func (c *Config) IsStandaloneMode() bool { + return c.Mode == standaloneMode +} + +func (c *Config) GetRestConfig() *rest.Config { + return c.ClusterConfig +} + func Load(logger logr.Logger, args []string) (*Config, error) { flagSet := GetFlagSet(pflag.ExitOnError) err := flagSet.Parse(args) @@ -260,6 +445,9 @@ func Load(logger logr.Logger, args []string) (*Config, error) { if err != nil { return nil, err } + if err := cfg.Validate(); err != nil { + return nil, err + } return cfg, nil } @@ -291,6 +479,11 @@ func LoadFromCLI(target *Config, flagSet *pflag.FlagSet) error { } else if changed { target.ListenAddr = listenAddr } + if healthListenAddr, changed, err := getHealthListenAddr(flagSet); err != nil { + return err + } else if changed { + target.HealthListenAddr = healthListenAddr + } if heartbeatInterval, changed, err := getHeartbeatInterval(flagSet); err != nil { return err } else if changed { @@ -301,16 +494,21 @@ func LoadFromCLI(target *Config, flagSet *pflag.FlagSet) error { } else if changed { target.Name = name } + if mode, changed, err := getMode(flagSet); err != nil { + return err + } else if changed { + target.Mode = mode + } return nil } func LoadFromFile(cfg *Config, configFile string) error { - yamlFile, err := os.ReadFile(configFile) - if err != nil { - return err + yamlFile, readErr := os.ReadFile(configFile) + if readErr != nil { + return readErr } envExpandedYaml := []byte(os.ExpandEnv(string(yamlFile))) - if err = yaml.Unmarshal(envExpandedYaml, cfg); err != nil { + if err := yaml.Unmarshal(envExpandedYaml, cfg); err != nil { return fmt.Errorf("error unmarshaling YAML: %w", err) } return nil diff --git a/cmd/operator-opamp-bridge/internal/config/config_test.go b/cmd/operator-opamp-bridge/internal/config/config_test.go index c31353279b..cba098e584 100644 --- a/cmd/operator-opamp-bridge/internal/config/config_test.go +++ b/cmd/operator-opamp-bridge/internal/config/config_test.go @@ -5,6 +5,9 @@ package config import ( "fmt" + "os" + "path/filepath" + "strings" "testing" "time" @@ -25,6 +28,7 @@ func TestConfigLoadPriority(t *testing.T) { assert.NoError(t, err) assert.Equal(t, defaultServerListenAddr, cfg.ListenAddr, "use default value") + assert.Equal(t, defaultHealthListenAddr, cfg.HealthListenAddr, "use default value") assert.Equal(t, defaultHeartbeatInterval, cfg.HeartbeatInterval, "use default value") assert.Equal(t, opampBridgeName, cfg.Name, "use default value") }) @@ -38,6 +42,7 @@ func TestConfigLoadPriority(t *testing.T) { assert.NoError(t, err) assert.Equal(t, defaultServerListenAddr, cfg.ListenAddr, "use default value") + assert.Equal(t, defaultHealthListenAddr, cfg.HealthListenAddr, "use default value") assert.Equal(t, 10*time.Second, cfg.HeartbeatInterval, "command-line priority is higher than config, overwrite time.Duration value") assert.Equal(t, "http-test-bridge", cfg.Name, "config file priority is higher than default string value") }) @@ -48,16 +53,34 @@ func TestConfigLoadPriority(t *testing.T) { "--" + configFilePathFlagName + "=./testdata/agenthttpbasic.yaml", "--" + kubeConfigPathFlagName + "=./testdata/kubeconfig.yaml", "--" + nameFlagName + "=" + testOpAMPBridgeName, + "--" + healthListenAddrFlagName + "=:8082", } cfg, err := Load(GetLogger(), args) assert.NoError(t, err) assert.Equal(t, defaultServerListenAddr, cfg.ListenAddr, "use default value") + assert.Equal(t, ":8082", cfg.HealthListenAddr, "command-line priority is higher than config, overwrite health address") assert.Equal(t, 45*time.Second, cfg.HeartbeatInterval, "config file priority is higher than default time.Duration value") assert.Equal(t, testOpAMPBridgeName, cfg.Name, "command-line priority is higher than config, overwrite string value") }) } +func TestLoadFromFileHealthListenAddr(t *testing.T) { + cfg := NewConfig(logr.Discard()) + configFile := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(configFile, []byte(` +endpoint: ws://127.0.0.1:4320/v1/opamp +healthListenAddr: ":9090" +capabilities: + ReportsHealth: true +`), 0o600)) + + require.NoError(t, LoadFromFile(cfg, configFile)) + + assert.Equal(t, ":9090", cfg.HealthListenAddr) + assert.Equal(t, defaultServerListenAddr, cfg.ListenAddr) +} + func TestLoadFromFile(t *testing.T) { type args struct { file string @@ -80,10 +103,12 @@ func TestLoadFromFile(t *testing.T) { instanceId: instanceId, RootLogger: logr.Discard(), ListenAddr: defaultServerListenAddr, + HealthListenAddr: defaultHealthListenAddr, KubeConfigFilePath: defaultKubeConfigPath, Endpoint: "ws://127.0.0.1:4320/v1/opamp", HeartbeatInterval: defaultHeartbeatInterval, Name: opampBridgeName, + Mode: defaultMode, Capabilities: map[Capability]bool{ AcceptsRemoteConfig: true, ReportsEffectiveConfig: true, @@ -112,9 +137,11 @@ func TestLoadFromFile(t *testing.T) { RootLogger: logr.Discard(), Endpoint: "http://127.0.0.1:4320/v1/opamp", ListenAddr: defaultServerListenAddr, + HealthListenAddr: defaultHealthListenAddr, KubeConfigFilePath: defaultKubeConfigPath, HeartbeatInterval: 45 * time.Second, Name: "http-test-bridge", + Mode: defaultMode, Capabilities: map[Capability]bool{ AcceptsRemoteConfig: true, ReportsEffectiveConfig: true, @@ -143,9 +170,11 @@ func TestLoadFromFile(t *testing.T) { RootLogger: logr.Discard(), Endpoint: "ws://127.0.0.1:4320/v1/opamp", ListenAddr: defaultServerListenAddr, + HealthListenAddr: defaultHealthListenAddr, KubeConfigFilePath: defaultKubeConfigPath, HeartbeatInterval: defaultHeartbeatInterval, Name: opampBridgeName, + Mode: defaultMode, Capabilities: map[Capability]bool{ AcceptsRemoteConfig: true, ReportsEffectiveConfig: true, @@ -184,7 +213,14 @@ func TestLoadFromFile(t *testing.T) { want: &Config{}, needErr: true, wantErr: func(t assert.TestingT, err error, i ...any) bool { - return assert.ErrorContains(t, err, "error unmarshaling YAML", i...) + if err == nil { + return assert.Fail(t, "expected YAML error, got nil", i...) + } + msg := err.Error() + if !strings.Contains(msg, "error unmarshaling YAML") { + return assert.Fail(t, fmt.Sprintf("unexpected error %q", msg), i...) + } + return true }, }, { @@ -207,9 +243,11 @@ func TestLoadFromFile(t *testing.T) { "my-env-variable-2": "my-env-variable-2-value", }, ListenAddr: defaultServerListenAddr, + HealthListenAddr: defaultHealthListenAddr, KubeConfigFilePath: defaultKubeConfigPath, HeartbeatInterval: defaultHeartbeatInterval, Name: opampBridgeName, + Mode: defaultMode, Capabilities: map[Capability]bool{ AcceptsRemoteConfig: true, ReportsEffectiveConfig: true, @@ -247,9 +285,11 @@ func TestLoadFromFile(t *testing.T) { }, }, ListenAddr: defaultServerListenAddr, + HealthListenAddr: defaultHealthListenAddr, KubeConfigFilePath: defaultKubeConfigPath, HeartbeatInterval: defaultHeartbeatInterval, Name: opampBridgeName, + Mode: defaultMode, Capabilities: map[Capability]bool{ AcceptsRemoteConfig: true, ReportsEffectiveConfig: true, @@ -290,6 +330,35 @@ func TestLoadFromFile(t *testing.T) { } } +func TestLoadFromFileRejectsDuplicateStandaloneConfigKeys(t *testing.T) { + cfg := []byte(` +mode: standalone +standalone: + agents: + - namespace: default + type: otel-collector + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: collector + config: + collector: + kind: configmap + name: collector-config + key: collector.yaml + collector: + kind: configmap + name: other-config + key: other.yaml +`) + configPath := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(configPath, cfg, 0o600)) + + err := LoadFromFile(NewConfig(logr.Discard()), configPath) + require.Error(t, err) + assert.ErrorContains(t, err, `mapping key "collector" already defined`) +} + func TestGetDescription(t *testing.T) { got := NewConfig(logr.Discard()) instanceId := uuid.New() @@ -320,3 +389,89 @@ func TestGetDescriptionNoneSet(t *testing.T) { }}) assert.Len(t, desc.NonIdentifyingAttributes, 2) } + +func TestNewConfigSetsDefaultMode(t *testing.T) { + cfg := NewConfig(logr.Discard()) + assert.Equal(t, operatorMode, cfg.Mode, "NewConfig should seed Mode with the documented default so logs/state match the flag default") + assert.False(t, cfg.IsStandaloneMode()) +} + +func TestValidateRejectsUnknownMode(t *testing.T) { + t.Run("typo on mode is rejected", func(t *testing.T) { + cfg := NewConfig(logr.Discard()) + cfg.Mode = "standlon" + err := cfg.Validate() + require.Error(t, err) + assert.ErrorContains(t, err, `invalid mode "standlon"`) + }) + + t.Run("case-sensitive mode is rejected", func(t *testing.T) { + cfg := NewConfig(logr.Discard()) + cfg.Mode = "Standalone" + err := cfg.Validate() + require.Error(t, err) + assert.ErrorContains(t, err, `invalid mode "Standalone"`) + }) + + t.Run("operator mode validates without error", func(t *testing.T) { + cfg := NewConfig(logr.Discard()) + cfg.Mode = operatorMode + assert.NoError(t, cfg.Validate()) + }) + + t.Run("empty mode is normalized to the documented default", func(t *testing.T) { + cfg := NewConfig(logr.Discard()) + cfg.Mode = "" + require.NoError(t, cfg.Validate()) + assert.Equal(t, operatorMode, cfg.Mode) + }) + + t.Run("invalid mode does not fall through to operator behavior", func(t *testing.T) { + cfg := NewConfig(logr.Discard()) + cfg.Mode = "standlon" + cfg.Standalone = StandaloneConfig{Agents: []StandaloneAgentConfig{}} + // Previously, an unknown mode would short-circuit Validate() at the + // `IsStandaloneMode()` guard, silently behaving as operator. Make sure + // the error is surfaced instead. + err := cfg.Validate() + require.Error(t, err) + assert.NotContains(t, err.Error(), "standalone mode requires at least one configured agent") + }) +} + +func TestNewStandaloneAgentConfigUsesWorkloadRefNameAsHostName(t *testing.T) { + cfg := NewConfig(logr.Discard()) + cfg.Mode = standaloneMode + cfg.Headers = Headers{"x-test-header": "header-value"} + cfg.Capabilities = map[Capability]bool{AcceptsRemoteConfig: true} + cfg.ComponentsAllowed = map[string][]string{"receivers": {"otlp"}} + cfg.AgentDescription.NonIdentifyingAttributes = map[string]string{"environment": "test"} + + agentCfg := NewStandaloneAgentConfig(cfg, StandaloneAgentConfig{ + Namespace: "default", + Type: "otel-collector", + WorkloadRef: StandaloneWorkloadRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "collector-workload", + }, + }) + + desc := agentCfg.GetDescription() + assert.Contains(t, desc.NonIdentifyingAttributes, &protobufs.KeyValue{Key: "host.name", Value: &protobufs.AnyValue{ + Value: &protobufs.AnyValue_StringValue{StringValue: "collector-workload"}, + }}) + assert.NotContains(t, desc.NonIdentifyingAttributes, &protobufs.KeyValue{Key: "host.name", Value: &protobufs.AnyValue{ + Value: &protobufs.AnyValue_StringValue{StringValue: hostname}, + }}) + + agentCfg.Headers["x-test-header"] = "changed" + agentCfg.Capabilities[AcceptsRemoteConfig] = false + agentCfg.ComponentsAllowed["receivers"][0] = "prometheus" + agentCfg.AgentDescription.NonIdentifyingAttributes["environment"] = "changed" + + assert.Equal(t, "header-value", cfg.Headers["x-test-header"]) + assert.True(t, cfg.Capabilities[AcceptsRemoteConfig]) + assert.Equal(t, []string{"otlp"}, cfg.ComponentsAllowed["receivers"]) + assert.Equal(t, "test", cfg.AgentDescription.NonIdentifyingAttributes["environment"]) +} diff --git a/cmd/operator-opamp-bridge/internal/config/flags.go b/cmd/operator-opamp-bridge/internal/config/flags.go index 14e01b0a16..241d6c55d7 100644 --- a/cmd/operator-opamp-bridge/internal/config/flags.go +++ b/cmd/operator-opamp-bridge/internal/config/flags.go @@ -21,10 +21,14 @@ const ( configFilePathFlagName = "config-file" listenAddrFlagName = "listen-addr" defaultServerListenAddr = ":8080" + healthListenAddrFlagName = "health-listen-addr" + defaultHealthListenAddr = ":8081" kubeConfigPathFlagName = "kubeconfig-path" heartbeatIntervalFlagName = "heartbeat-interval" nameFlagName = "name" + modeFlagName = "mode" defaultHeartbeatInterval = 30 * time.Second + defaultMode = operatorMode ) var defaultKubeConfigPath = filepath.Join(homedir.HomeDir(), ".kube", "config") @@ -35,10 +39,12 @@ var zapCmdLineOpts zap.Options func GetFlagSet(errorHandling pflag.ErrorHandling) *pflag.FlagSet { flagSet := pflag.NewFlagSet(opampBridgeName, errorHandling) flagSet.String(configFilePathFlagName, defaultConfigFilePath, "The path to the config file.") - flagSet.String(listenAddrFlagName, defaultServerListenAddr, "The address where this service serves.") + flagSet.String(listenAddrFlagName, defaultServerListenAddr, "The address where this service serves OpAMP proxy traffic.") + flagSet.String(healthListenAddrFlagName, defaultHealthListenAddr, "The address where this service serves health checks.") flagSet.String(kubeConfigPathFlagName, defaultKubeConfigPath, "absolute path to the KubeconfigPath file.") flagSet.Duration(heartbeatIntervalFlagName, defaultHeartbeatInterval, "The interval to use for sending a heartbeat. Setting it to 0 disables the heartbeat.") flagSet.String(nameFlagName, opampBridgeName, "The name of the bridge to use for querying managed collectors.") + flagSet.String(modeFlagName, defaultMode, `Operating mode: "operator" (default, uses CRDs) or "standalone" (manages ConfigMaps for Deployments/DaemonSets).`) zapFlagSet := flag.NewFlagSet("", flag.ErrorHandling(errorHandling)) zapCmdLineOpts.BindFlags(zapFlagSet) flagSet.AddGoFlagSet(zapFlagSet) @@ -65,6 +71,16 @@ func getListenAddr(flagSet *pflag.FlagSet) (value string, changed bool, err erro return getFlagValueAndChanged[string](flagSet, listenAddrFlagName) } +func getHealthListenAddr(flagSet *pflag.FlagSet) (value string, changed bool, err error) { + return getFlagValueAndChanged[string](flagSet, healthListenAddrFlagName) +} + +func getMode(flagSet *pflag.FlagSet) (value string, changed bool, err error) { + return getFlagValueAndChanged[string](flagSet, modeFlagName) +} + +// getFlagValueAndChanged returns a typed flag value only when the user explicitly set the flag. +// changed is false when the flag should leave the existing config value untouched. func getFlagValueAndChanged[T any](flagSet *pflag.FlagSet, flagName string) (value T, changed bool, err error) { var zero T if changed = flagSet.Changed(flagName); !changed { diff --git a/cmd/operator-opamp-bridge/internal/config/flags_test.go b/cmd/operator-opamp-bridge/internal/config/flags_test.go index 727224fcf6..9e14c9cc11 100644 --- a/cmd/operator-opamp-bridge/internal/config/flags_test.go +++ b/cmd/operator-opamp-bridge/internal/config/flags_test.go @@ -18,6 +18,7 @@ func TestGetFlagSet(t *testing.T) { // Check if each flag exists assert.NotNil(t, fs.Lookup(configFilePathFlagName), "Flag %s not found", configFilePathFlagName) assert.NotNil(t, fs.Lookup(listenAddrFlagName), "Flag %s not found", listenAddrFlagName) + assert.NotNil(t, fs.Lookup(healthListenAddrFlagName), "Flag %s not found", healthListenAddrFlagName) assert.NotNil(t, fs.Lookup(kubeConfigPathFlagName), "Flag %s not found", kubeConfigPathFlagName) } @@ -65,6 +66,15 @@ func TestFlagGetters(t *testing.T) { return value, err }, }, + { + name: "GetHealthListenAddr", + flagArgs: []string{"--" + healthListenAddrFlagName, ":8082"}, + expectedValue: ":8082", + getterFunc: func(fs *pflag.FlagSet) (any, error) { + value, _, err := getHealthListenAddr(fs) + return value, err + }, + }, { name: "GetHeartbeatInterval", flagArgs: []string{"--" + heartbeatIntervalFlagName, "45s"}, diff --git a/cmd/operator-opamp-bridge/internal/healthcheck/server.go b/cmd/operator-opamp-bridge/internal/healthcheck/server.go new file mode 100644 index 0000000000..3d2f6bbd18 --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/healthcheck/server.go @@ -0,0 +1,66 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package healthcheck + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "time" + + "github.com/go-logr/logr" +) + +const Path = "/healthz" + +type Server struct { + logger logr.Logger + server *http.Server + addr string +} + +func NewServer(logger logr.Logger, listenAddr string) *Server { + mux := http.NewServeMux() + mux.HandleFunc(Path, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + return &Server{ + logger: logger, + server: &http.Server{ + Addr: listenAddr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + }, + } +} + +func (s *Server) Start(ctx context.Context) error { + listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", s.server.Addr) + if err != nil { + return fmt.Errorf("failed to start health listener on %q: %w", s.server.Addr, err) + } + + s.addr = listener.Addr().String() + go func() { + s.logger.Info("starting health listener", "address", s.addr) + if err := s.server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + s.logger.Error(err, "health listener stopped with error") + } + }() + return nil +} + +func (s *Server) Stop(ctx context.Context) error { + if s == nil || s.server == nil { + return nil + } + return s.server.Shutdown(ctx) +} + +func (s *Server) Addr() string { + return s.addr +} diff --git a/cmd/operator-opamp-bridge/internal/healthcheck/server_test.go b/cmd/operator-opamp-bridge/internal/healthcheck/server_test.go new file mode 100644 index 0000000000..3fd5db5c71 --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/healthcheck/server_test.go @@ -0,0 +1,48 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package healthcheck + +import ( + "context" + "net" + "net/http" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServerStartsHealthEndpoint(t *testing.T) { + server := NewServer(logr.Discard(), "127.0.0.1:0") + + require.NoError(t, server.Start(t.Context())) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + require.NoError(t, server.Stop(ctx)) + }) + + client := http.Client{Timeout: time.Second} + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://"+server.Addr()+Path, http.NoBody) + require.NoError(t, err) + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + assert.Equal(t, http.StatusOK, resp.StatusCode) +} + +func TestServerReturnsBindError(t *testing.T) { + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + + server := NewServer(logr.Discard(), listener.Addr().String()) + + err = server.Start(t.Context()) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to start health listener") +} diff --git a/cmd/operator-opamp-bridge/internal/manager/manager.go b/cmd/operator-opamp-bridge/internal/manager/manager.go new file mode 100644 index 0000000000..4d8b86106b --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/manager/manager.go @@ -0,0 +1,169 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package manager + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/go-logr/logr" + opampclient "github.com/open-telemetry/opamp-go/client" + + opampagent "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/agent" + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/healthcheck" + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/proxy" +) + +type Runtime struct { + Name string + OpAMPAgent *opampagent.Agent + Client opampclient.OpAMPClient +} + +type KubernetesClient interface { + Start(context.Context) error +} + +type Option func(*Manager) + +type Manager struct { + log logr.Logger + runtimes []Runtime + healthServer *healthcheck.Server + opampProxy *proxy.OpAMPProxy + kubernetesClient KubernetesClient + permissionReviewClient PermissionReviewClient + listRequiredPermissions func() ([]Permission, error) + cancelKubernetesClient context.CancelFunc + shutdownOnce sync.Once +} + +func WithLogger(log logr.Logger) Option { + return func(m *Manager) { + m.log = log + } +} + +func WithRuntimes(runtimes []Runtime) Option { + return func(m *Manager) { + m.runtimes = runtimes + } +} + +func WithHealthServer(healthServer *healthcheck.Server) Option { + return func(m *Manager) { + m.healthServer = healthServer + } +} + +func WithOpAMPProxy(opampProxy *proxy.OpAMPProxy) Option { + return func(m *Manager) { + m.opampProxy = opampProxy + } +} + +func WithKubernetesClient(kubernetesClient KubernetesClient) Option { + return func(m *Manager) { + m.kubernetesClient = kubernetesClient + } +} + +func WithPermissionReviewClient(permissionReviewClient PermissionReviewClient) Option { + return func(m *Manager) { + m.permissionReviewClient = permissionReviewClient + } +} + +func WithRequiredPermissions(listRequiredPermissions func() ([]Permission, error)) Option { + return func(m *Manager) { + m.listRequiredPermissions = listRequiredPermissions + } +} + +func New(options ...Option) (*Manager, error) { + manager := &Manager{} + for _, option := range options { + option(manager) + } + if err := manager.validate(); err != nil { + return nil, err + } + return manager, nil +} + +func (m *Manager) validate() error { + if m.listRequiredPermissions != nil && m.permissionReviewClient == nil { + return errors.New("permission review client is required") + } + for i, runtime := range m.runtimes { + if runtime.OpAMPAgent == nil { + return fmt.Errorf("runtime %d opamp agent is required", i) + } + } + return nil +} + +func (m *Manager) Start(ctx context.Context) error { + if m.listRequiredPermissions != nil { + requiredPermissions, err := m.listRequiredPermissions() + if err != nil { + return err + } + if err := CheckPermissions(ctx, m.permissionReviewClient, requiredPermissions); err != nil { + return err + } + } + for _, runtime := range m.runtimes { + if err := runtime.OpAMPAgent.Start(); err != nil { + m.Shutdown(ctx) + return err + } + } + if m.kubernetesClient != nil { + kubernetesClientCtx, cancelKubernetesClient := context.WithCancel(ctx) + m.cancelKubernetesClient = cancelKubernetesClient + if err := m.kubernetesClient.Start(kubernetesClientCtx); err != nil { + cancelKubernetesClient() + m.cancelKubernetesClient = nil + m.Shutdown(ctx) + return err + } + } + if m.opampProxy != nil { + if err := m.opampProxy.Start(); err != nil { + m.Shutdown(ctx) + return err + } + } + if m.healthServer != nil { + if err := m.healthServer.Start(ctx); err != nil { + m.Shutdown(ctx) + return err + } + } + return nil +} + +func (m *Manager) Shutdown(ctx context.Context) { + m.shutdownOnce.Do(func() { + if m.cancelKubernetesClient != nil { + m.cancelKubernetesClient() + } + for _, runtime := range m.runtimes { + runtime.OpAMPAgent.Shutdown() + } + if m.opampProxy != nil { + if err := m.opampProxy.Stop(ctx); err != nil { + m.log.Error(err, "failed to stop OpAMP proxy") + } + } + if m.healthServer != nil { + if err := m.healthServer.Stop(ctx); err != nil { + m.log.Error(err, "failed to stop health listener") + } + } + }) +} diff --git a/cmd/operator-opamp-bridge/internal/manager/manager_test.go b/cmd/operator-opamp-bridge/internal/manager/manager_test.go new file mode 100644 index 0000000000..5933d31880 --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/manager/manager_test.go @@ -0,0 +1,166 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package manager + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/go-logr/logr" + "github.com/open-telemetry/opamp-go/client/types" + "github.com/open-telemetry/opamp-go/protobufs" + "github.com/stretchr/testify/require" + + opampagent "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/agent" + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/config" + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/operator" + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/proxy" +) + +func TestManager_StartStartsRuntimesBeforeKubernetesClient(t *testing.T) { + cfg := config.NewConfig(logr.Discard()) + cfg.Endpoint = "ws://example.test/v1/opamp" + cfg.Capabilities = map[config.Capability]bool{ + config.ReportsHealth: true, + } + opampClient := &recordingOpAMPClient{} + agent := opampagent.NewAgent(logr.Discard(), healthyApplier{}, cfg, opampClient, proxy.NoopServer{}) + kubernetesClient := &orderCheckingKubernetesClient{opampClient: opampClient} + manager, err := New( + WithLogger(logr.Discard()), + WithRuntimes([]Runtime{{Name: "test", OpAMPAgent: agent, Client: opampClient}}), + WithKubernetesClient(kubernetesClient), + ) + require.NoError(t, err) + + require.NoError(t, manager.Start(context.Background())) + t.Cleanup(func() { + manager.Shutdown(t.Context()) + }) + require.True(t, kubernetesClient.started) +} + +func TestManager_NewRequiresPermissionReviewClientWithRequiredPermissions(t *testing.T) { + manager, err := New( + WithRequiredPermissions(func() ([]Permission, error) { + return []Permission{{Verb: "get", Resource: "pods"}}, nil + }), + ) + + require.Nil(t, manager) + require.EqualError(t, err, "permission review client is required") +} + +func TestManager_NewRequiresRuntimeOpAMPAgent(t *testing.T) { + manager, err := New( + WithRuntimes([]Runtime{{Name: "test"}}), + ) + + require.Nil(t, manager) + require.EqualError(t, err, "runtime 0 opamp agent is required") +} + +type orderCheckingKubernetesClient struct { + opampClient *recordingOpAMPClient + started bool +} + +func (c *orderCheckingKubernetesClient) Start(context.Context) error { + if !c.opampClient.Started() { + return errors.New("kubernetes client started before opamp client") + } + c.started = true + return nil +} + +type healthyApplier struct{} + +func (healthyApplier) Apply(string, *protobufs.AgentConfigFile) error { + return nil +} + +func (healthyApplier) Delete(string) error { + return nil +} + +func (healthyApplier) ListInstances() ([]operator.CollectorInstance, error) { + return nil, nil +} + +func (healthyApplier) GetHealth() (operator.Health, error) { + return operator.Health{Healthy: true, Children: map[string]operator.Health{}}, nil +} + +type recordingOpAMPClient struct { + mu sync.Mutex + started bool +} + +func (c *recordingOpAMPClient) Started() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.started +} + +func (c *recordingOpAMPClient) Start(context.Context, types.StartSettings) error { + c.mu.Lock() + defer c.mu.Unlock() + c.started = true + return nil +} + +func (c *recordingOpAMPClient) Stop(context.Context) error { + c.mu.Lock() + defer c.mu.Unlock() + c.started = false + return nil +} + +func (*recordingOpAMPClient) SetAgentDescription(*protobufs.AgentDescription) error { + return nil +} + +func (*recordingOpAMPClient) AgentDescription() *protobufs.AgentDescription { + return nil +} + +func (*recordingOpAMPClient) SetHealth(*protobufs.ComponentHealth) error { + return nil +} + +func (*recordingOpAMPClient) UpdateEffectiveConfig(context.Context) error { + return nil +} + +func (*recordingOpAMPClient) SetRemoteConfigStatus(*protobufs.RemoteConfigStatus) error { + return nil +} + +func (*recordingOpAMPClient) SetPackageStatuses(*protobufs.PackageStatuses) error { + return nil +} + +func (*recordingOpAMPClient) RequestConnectionSettings(*protobufs.ConnectionSettingsRequest) error { + return nil +} + +func (*recordingOpAMPClient) SetCustomCapabilities(*protobufs.CustomCapabilities) error { + return nil +} + +func (*recordingOpAMPClient) SetFlags(protobufs.AgentToServerFlags) {} + +func (*recordingOpAMPClient) SendCustomMessage(*protobufs.CustomMessage) (chan struct{}, error) { + return nil, nil +} + +func (*recordingOpAMPClient) SetAvailableComponents(*protobufs.AvailableComponents) error { + return nil +} + +func (*recordingOpAMPClient) SetCapabilities(*protobufs.AgentCapabilities) error { + return nil +} diff --git a/cmd/operator-opamp-bridge/internal/manager/permissions.go b/cmd/operator-opamp-bridge/internal/manager/permissions.go new file mode 100644 index 0000000000..3fdf8f80ad --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/manager/permissions.go @@ -0,0 +1,88 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package manager + +import ( + "context" + "errors" + "fmt" + "strings" + + authorizationv1 "k8s.io/api/authorization/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type PermissionReviewClient interface { + Create(context.Context, client.Object, ...client.CreateOption) error +} + +type Permission struct { + Verb string + APIGroup string + Resource string + Namespace string + Name string +} + +// CheckPermissions verifies every requested Kubernetes permission using SelfSubjectAccessReview. +// perms describes the verb, resource, namespace, and optional object name that the bridge must be allowed to access. +func CheckPermissions(ctx context.Context, k8sClient PermissionReviewClient, perms []Permission) error { + if len(perms) == 0 { + return nil + } + if k8sClient == nil { + return errors.New("permission review client is required") + } + var errs []error + for _, perm := range perms { + if err := checkPermission(ctx, k8sClient, perm); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// checkPermission checks if the process has the given permission. It returns an error if the perm +// is missing otherwise returns nil. +func checkPermission(ctx context.Context, k8sClient PermissionReviewClient, perm Permission) error { + review := &authorizationv1.SelfSubjectAccessReview{ + Spec: authorizationv1.SelfSubjectAccessReviewSpec{ + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Verb: perm.Verb, + Group: perm.APIGroup, + Resource: perm.Resource, + Namespace: perm.Namespace, + Name: perm.Name, + }, + }, + } + if err := k8sClient.Create(ctx, review); err != nil { + return fmt.Errorf("failed to check %s permission for %s: %w", perm.Verb, perm.description(), err) + } + if review.Status.Allowed { + return nil + } + reason := review.Status.Reason + if review.Status.EvaluationError != "" { + reason = strings.TrimSpace(reason + ": " + review.Status.EvaluationError) + } + if reason == "" { + reason = "access denied" + } + return fmt.Errorf("missing %s permission for %s: %s", perm.Verb, perm.description(), reason) +} + +func (p Permission) description() string { + resource := p.Resource + if p.APIGroup != "" { + resource = p.APIGroup + "/" + resource + } + if p.Namespace == "" && p.Name == "" { + return resource + } + if p.Name == "" { + return fmt.Sprintf("%s in namespace %s", resource, p.Namespace) + } + return fmt.Sprintf("%s %s/%s", resource, p.Namespace, p.Name) +} diff --git a/cmd/operator-opamp-bridge/internal/operator/client.go b/cmd/operator-opamp-bridge/internal/operator/client.go index 57313d4684..8bd3cd433f 100644 --- a/cmd/operator-opamp-bridge/internal/operator/client.go +++ b/cmd/operator-opamp-bridge/internal/operator/client.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/go-logr/logr" "github.com/open-telemetry/opamp-go/protobufs" @@ -29,20 +30,21 @@ const ( ) type ConfigApplier interface { - // Apply receives a name and namespace to apply an OpenTelemetryCollector CRD that is contained in the configmap. - Apply(name, namespace string, configmap *protobufs.AgentConfigFile) error + // Apply receives an OpAMP remote config entry name and applies the corresponding configuration. + // Implementations define the accepted key format, but keys should match entries returned by + // CollectorInstance.GetConfigMap. + Apply(key string, configmap *protobufs.AgentConfigFile) error - // Delete attempts to delete an OpenTelemetryCollector object given a name and namespace. - Delete(name, namespace string) error + // Delete attempts to delete the resource identified by an OpAMP remote config entry name. + // Implementations define the accepted key format, but keys should match entries returned by + // CollectorInstance.GetConfigMap. + Delete(key string) error - // ListInstances retrieves all OpenTelemetryCollector CRDs created by the operator-opamp-bridge agent. - ListInstances() ([]v1beta1.OpenTelemetryCollector, error) + // ListInstances retrieves all collector instances managed by the bridge. + ListInstances() ([]CollectorInstance, error) - // GetInstance retrieves an OpenTelemetryCollector CRD given a name and namespace. - GetInstance(name, namespace string) (*v1beta1.OpenTelemetryCollector, error) - - // GetCollectorPods retrieves all pods that match the given collector's selector labels and namespace. - GetCollectorPods(selectorLabels map[string]string, namespace string) (*v1.PodList, error) + // GetHealth retrieves the health of resources managed by the bridge. + GetHealth() (Health, error) } type Client struct { @@ -65,7 +67,12 @@ func NewClient(name string, log logr.Logger, c client.Client, componentsAllowed } } -func (c Client) Apply(name, namespace string, configmap *protobufs.AgentConfigFile) error { +func (c Client) Apply(key string, configmap *protobufs.AgentConfigFile) error { + resource, err := kubeResourceFromKey(key) + if err != nil { + return err + } + name, namespace := resource.name, resource.namespace c.log.Info("Received new config", "name", name, "namespace", namespace) if len(configmap.Body) == 0 { @@ -73,7 +80,7 @@ func (c Client) Apply(name, namespace string, configmap *protobufs.AgentConfigFi } var collector v1beta1.OpenTelemetryCollector - err := yaml.Unmarshal(configmap.Body, &collector) + err = yaml.Unmarshal(configmap.Body, &collector) if err != nil { return errors.NewBadRequest(fmt.Sprintf("failed to unmarshal config into v1beta1 API Version: %v", err)) } @@ -191,12 +198,16 @@ func (c Client) update(ctx context.Context, o, n *v1beta1.OpenTelemetryCollector return c.k8sClient.Update(ctx, n) } -func (c Client) Delete(name, namespace string) error { +func (c Client) Delete(key string) error { + resource, err := kubeResourceFromKey(key) + if err != nil { + return err + } ctx := context.Background() result := v1beta1.OpenTelemetryCollector{} - err := c.k8sClient.Get(ctx, client.ObjectKey{ - Namespace: namespace, - Name: name, + err = c.k8sClient.Get(ctx, client.ObjectKey{ + Namespace: resource.namespace, + Name: resource.name, }, &result) if err != nil { if errors.IsNotFound(err) { @@ -207,7 +218,79 @@ func (c Client) Delete(name, namespace string) error { return c.k8sClient.Delete(ctx, &result) } -func (c Client) ListInstances() ([]v1beta1.OpenTelemetryCollector, error) { +// ListInstances returns all collectors that are visible to OpAMP as effective config entries. +func (c Client) ListInstances() ([]CollectorInstance, error) { + collectors, err := c.listOpenTelemetryCollectors() + if err != nil { + return nil, err + } + result := make([]CollectorInstance, len(collectors)) + for i := range collectors { + result[i] = collectors[i] + } + return result, nil +} + +// GetHealth reports bridge root health with managed collector health as children. +func (c Client) GetHealth() (Health, error) { + collectors, err := c.listOpenTelemetryCollectors() + if err != nil { + return Health{}, err + } + healthMap := map[string]Health{} + for _, col := range collectors { + podMap, err := c.generateCollectorHealth(col.selectorLabels(), col.GetNamespace()) + if err != nil { + return Health{}, err + } + isPoolHealthy := true + for _, pod := range podMap { + isPoolHealthy = isPoolHealthy && pod.Healthy + } + healthMap[NewKubeResourceKey(col.GetNamespace(), col.GetName()).String()] = Health{ + StartTime: col.Col.GetCreationTimestamp().Time, + Status: col.Col.Status.Scale.StatusReplicas, + Children: podMap, + Healthy: isPoolHealthy, + } + } + return Health{ + Healthy: true, + Children: healthMap, + }, nil +} + +// generateCollectorHealth reports pod health for one collector selected by labels within a namespace. +func (c Client) generateCollectorHealth(selectorLabels map[string]string, namespace string) (map[string]Health, error) { + pods, err := c.getCollectorPods(selectorLabels, namespace) + if err != nil { + return nil, err + } + healthMap := map[string]Health{} + for _, item := range pods.Items { + key := NewKubeResourceKey(item.GetNamespace(), item.GetName()) + healthy := true + if item.Status.Phase != "Running" { + healthy = false + } + var startTime time.Time + if item.Status.StartTime != nil { + startTime = item.Status.StartTime.Time + } else { + healthy = false + } + healthMap[key.String()] = Health{ + StartTime: startTime, + Status: string(item.Status.Phase), + Healthy: healthy, + Children: map[string]Health{}, + } + } + return healthMap, nil +} + +// listOpenTelemetryCollectors returns collectors managed by this bridge plus reporting-only collectors. +func (c Client) listOpenTelemetryCollectors() ([]CRDInstance, error) { ctx := context.Background() var instances []v1beta1.OpenTelemetryCollector @@ -239,7 +322,11 @@ func (c Client) ListInstances() ([]v1beta1.OpenTelemetryCollector, error) { setTypedMeta(&instances[i]) } - return instances, nil + result := make([]CRDInstance, len(instances)) + for i := range instances { + result[i] = newCRDInstance(instances[i]) + } + return result, nil } func (c Client) GetInstance(name, namespace string) (*v1beta1.OpenTelemetryCollector, error) { @@ -260,7 +347,7 @@ func (c Client) GetInstance(name, namespace string) (*v1beta1.OpenTelemetryColle return &result, nil } -func (c Client) GetCollectorPods(selectorLabels map[string]string, namespace string) (*v1.PodList, error) { +func (c Client) getCollectorPods(selectorLabels map[string]string, namespace string) (*v1.PodList, error) { ctx := context.Background() podList := &v1.PodList{} err := c.k8sClient.List(ctx, podList, client.MatchingLabels(selectorLabels), client.InNamespace(namespace)) diff --git a/cmd/operator-opamp-bridge/internal/operator/client_test.go b/cmd/operator-opamp-bridge/internal/operator/client_test.go index ca6b1fa51b..7776747261 100644 --- a/cmd/operator-opamp-bridge/internal/operator/client_test.go +++ b/cmd/operator-opamp-bridge/internal/operator/client_test.go @@ -156,7 +156,7 @@ func TestClient_Apply(t *testing.T) { Body: colConfig, ContentType: "yaml", } - applyErr := c.Apply(tt.args.name, tt.args.namespace, configmap) + applyErr := c.Apply(NewKubeResourceKey(tt.args.namespace, tt.args.name).String(), configmap) if tt.wantErr { assert.Error(t, applyErr) assert.ErrorContains(t, applyErr, tt.errContains) @@ -199,7 +199,7 @@ func TestClient_ApplyUpdate(t *testing.T) { ContentType: "yaml", } // Apply a valid initial configuration - err = c.Apply(name, namespace, configmap) + err = c.Apply(NewKubeResourceKey(namespace, name).String(), configmap) require.NoError(t, err, "Should apply base config") // Confirm there are now two collector instances, reporting and managed @@ -220,7 +220,7 @@ func TestClient_ApplyUpdate(t *testing.T) { // Try updating with an invalid configuration configmap.Body = []byte("empty, invalid!") - err = c.Apply(name, namespace, configmap) + err = c.Apply(NewKubeResourceKey(namespace, name).String(), configmap) assert.Error(t, err, "Should be unable to update with invalid config") // Update successfully with a valid configuration @@ -230,7 +230,7 @@ func TestClient_ApplyUpdate(t *testing.T) { Body: newColConfig, ContentType: "yaml", } - err = c.Apply(name, namespace, newConfigMap) + err = c.Apply(NewKubeResourceKey(namespace, name).String(), newConfigMap) require.NoError(t, err, "Should be able to update collector") // Get the updated collector @@ -246,8 +246,12 @@ func TestClient_ApplyUpdate(t *testing.T) { allInstances, err = c.ListInstances() require.NoError(t, err, "Should be able to list all collectors") assert.Len(t, allInstances, 2) - assert.Contains(t, allInstances, reportingCol) - assert.Contains(t, allInstances, *updatedInstance) + instanceNames := make([]string, len(allInstances)) + for i, inst := range allInstances { + instanceNames[i] = inst.GetNamespace() + "/" + inst.GetName() + } + assert.Contains(t, instanceNames, reportingCol.GetNamespace()+"/"+reportingCol.GetName()) + assert.Contains(t, instanceNames, updatedInstance.GetNamespace()+"/"+updatedInstance.GetName()) } func TestClient_Delete(t *testing.T) { @@ -262,7 +266,7 @@ func TestClient_Delete(t *testing.T) { ContentType: "yaml", } // Apply a valid initial configuration - err = c.Apply(name, namespace, configmap) + err = c.Apply(NewKubeResourceKey(namespace, name).String(), configmap) require.NoError(t, err, "Should apply base config") // Get the newly created collector @@ -274,7 +278,7 @@ func TestClient_Delete(t *testing.T) { require.Len(t, instance.Spec.Config.Service.Pipelines, 1, "Should have a pipeline") // Delete it - err = c.Delete(name, namespace) + err = c.Delete(NewKubeResourceKey(namespace, name).String()) require.NoError(t, err, "Should be able to delete a collector") // Check there's nothing left @@ -291,7 +295,7 @@ func loadConfig(file string) ([]byte, error) { return yamlFile, nil } -func TestClient_GetCollectorPods(t *testing.T) { +func TestClient_getCollectorPods(t *testing.T) { mockPodList := &v1.PodList{ Items: []v1.Pod{ { @@ -359,11 +363,11 @@ func TestClient_GetCollectorPods(t *testing.T) { t.Run(tt.name, func(t *testing.T) { fakeClient := getFakeClient(t, mockPodList) c := NewClient(bridgeName, clientLogger, fakeClient, nil) - got, err := c.GetCollectorPods(tt.args.selector, tt.args.namespace) - if !tt.wantErr(t, err, fmt.Sprintf("GetCollectorPods(%v)", tt.args.selector)) { + got, err := c.getCollectorPods(tt.args.selector, tt.args.namespace) + if !tt.wantErr(t, err, fmt.Sprintf("getCollectorPods(%v)", tt.args.selector)) { return } - assert.Equalf(t, tt.want, got, "GetCollectorPods(%v)", tt.args.selector) + assert.Equalf(t, tt.want, got, "getCollectorPods(%v)", tt.args.selector) }) } } diff --git a/cmd/operator-opamp-bridge/internal/operator/crd_instance.go b/cmd/operator-opamp-bridge/internal/operator/crd_instance.go new file mode 100644 index 0000000000..d892a8f91e --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/operator/crd_instance.go @@ -0,0 +1,73 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package operator + +import ( + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" + + "github.com/open-telemetry/opentelemetry-operator/apis/v1beta1" +) + +var _ CollectorInstance = CRDInstance{} + +// CRDInstance wraps an OpenTelemetryCollector CRD to implement CollectorInstance. +type CRDInstance struct { + Col v1beta1.OpenTelemetryCollector +} + +func newCRDInstance(col v1beta1.OpenTelemetryCollector) CRDInstance { + return CRDInstance{Col: col} +} + +func (c CRDInstance) GetName() string { + return c.Col.GetName() +} + +func (c CRDInstance) GetNamespace() string { + return c.Col.GetNamespace() +} + +func (c CRDInstance) GetDeletionTimestamp() *metav1.Time { + return c.Col.GetDeletionTimestamp() +} + +// selectorLabels returns the collector's status selector, or the operator's standard collector labels as a fallback. +func (c CRDInstance) selectorLabels() map[string]string { + if c.Col.Status.Scale.Selector != "" { + selMap := map[string]string{} + for kvPair := range strings.SplitSeq(c.Col.Status.Scale.Selector, ",") { + kv := strings.Split(kvPair, "=") + if len(kv) != 2 { + continue + } + selMap[kv[0]] = kv[1] + } + return selMap + } + return map[string]string{ + "app.kubernetes.io/managed-by": "opentelemetry-operator", + "app.kubernetes.io/instance": fmt.Sprintf("%s.%s", c.Col.GetNamespace(), c.Col.GetName()), + "app.kubernetes.io/part-of": "opentelemetry", + "app.kubernetes.io/component": "opentelemetry-collector", + } +} + +// GetConfigMap serializes the collector CRD into the OpAMP effective config map using namespace/name as the key. +func (c CRDInstance) GetConfigMap() map[string]ConfigFile { + key := NewKubeResourceKey(c.GetNamespace(), c.GetName()).String() + marshaled, err := yaml.Marshal(&c.Col) + if err != nil { + return map[string]ConfigFile{key: {}} + } + return map[string]ConfigFile{ + key: { + Body: marshaled, + ContentType: "yaml", + }, + } +} diff --git a/cmd/operator-opamp-bridge/internal/operator/instance.go b/cmd/operator-opamp-bridge/internal/operator/instance.go new file mode 100644 index 0000000000..8178bfb99e --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/operator/instance.go @@ -0,0 +1,34 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package operator + +import ( + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ConfigFile is an internal representation of an effective OpAMP config file. +type ConfigFile struct { + Body []byte + ContentType string +} + +// Health is an internal representation of OpAMP component health. +type Health struct { + Healthy bool + Status string + LastError string + StartTime time.Time + Children map[string]Health +} + +// CollectorInstance represents a collector managed by the bridge, abstracting +// the underlying Kubernetes resource (CRD or Deployment/DaemonSet). +type CollectorInstance interface { + GetName() string + GetNamespace() string + GetDeletionTimestamp() *metav1.Time + GetConfigMap() map[string]ConfigFile +} diff --git a/cmd/operator-opamp-bridge/internal/operator/kube_resource_key.go b/cmd/operator-opamp-bridge/internal/operator/kube_resource_key.go new file mode 100644 index 0000000000..2688d1d495 --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/operator/kube_resource_key.go @@ -0,0 +1,32 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package operator + +import ( + "errors" + "fmt" + "strings" +) + +type KubeResourceKey struct { + name string + namespace string +} + +func NewKubeResourceKey(namespace, name string) KubeResourceKey { + return KubeResourceKey{name: name, namespace: namespace} +} + +func kubeResourceFromKey(key string) (KubeResourceKey, error) { + s := strings.Split(key, "/") + // We expect map keys to be of the form namespace/name. + if len(s) != 2 { + return KubeResourceKey{}, errors.New("invalid key") + } + return NewKubeResourceKey(s[0], s[1]), nil +} + +func (k KubeResourceKey) String() string { + return fmt.Sprintf("%s/%s", k.namespace, k.name) +} diff --git a/cmd/operator-opamp-bridge/internal/agent/kube_resource_key_test.go b/cmd/operator-opamp-bridge/internal/operator/kube_resource_key_test.go similarity index 58% rename from cmd/operator-opamp-bridge/internal/agent/kube_resource_key_test.go rename to cmd/operator-opamp-bridge/internal/operator/kube_resource_key_test.go index 0c31655f22..61d5098f19 100644 --- a/cmd/operator-opamp-bridge/internal/agent/kube_resource_key_test.go +++ b/cmd/operator-opamp-bridge/internal/operator/kube_resource_key_test.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package agent +package operator import ( "fmt" @@ -10,14 +10,14 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_collectorKeyFromKey(t *testing.T) { +func TestKubeResourceFromKey(t *testing.T) { type args struct { key string } tests := []struct { name string args args - want kubeResourceKey + want KubeResourceKey wantErr assert.ErrorAssertionFunc }{ { @@ -25,7 +25,7 @@ func Test_collectorKeyFromKey(t *testing.T) { args: args{ key: "namespace/good", }, - want: kubeResourceKey{ + want: KubeResourceKey{ name: "good", namespace: "namespace", }, @@ -36,7 +36,7 @@ func Test_collectorKeyFromKey(t *testing.T) { args: args{ key: "badnamespace", }, - want: kubeResourceKey{}, + want: KubeResourceKey{}, wantErr: assert.Error, }, { @@ -44,7 +44,7 @@ func Test_collectorKeyFromKey(t *testing.T) { args: args{ key: "too/many/slashes", }, - want: kubeResourceKey{}, + want: KubeResourceKey{}, wantErr: assert.Error, }, } @@ -59,29 +59,7 @@ func Test_collectorKeyFromKey(t *testing.T) { } } -func Test_collectorKey_String(t *testing.T) { - type fields struct { - name string - namespace string - } - tests := []struct { - name string - fields fields - want string - }{ - { - name: "can make a key", - fields: fields{ - name: "good", - namespace: "namespace", - }, - want: "namespace/good", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - k := newKubeResourceKey(tt.fields.namespace, tt.fields.name) - assert.Equalf(t, tt.want, k.String(), "String()") - }) - } +func TestKubeResourceKeyString(t *testing.T) { + key := NewKubeResourceKey("namespace", "good") + assert.Equal(t, "namespace/good", key.String()) } diff --git a/cmd/operator-opamp-bridge/internal/operatorbridge/permissions.go b/cmd/operator-opamp-bridge/internal/operatorbridge/permissions.go new file mode 100644 index 0000000000..cf86d5d481 --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/operatorbridge/permissions.go @@ -0,0 +1,19 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package operatorbridge + +import bridgemanager "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/manager" + +// ListRequiredPermissions returns the Kubernetes permissions needed to run the bridge in operator mode. +func ListRequiredPermissions() ([]bridgemanager.Permission, error) { + return []bridgemanager.Permission{ + {Verb: "get", APIGroup: "opentelemetry.io", Resource: "opentelemetrycollectors"}, + {Verb: "list", APIGroup: "opentelemetry.io", Resource: "opentelemetrycollectors"}, + {Verb: "create", APIGroup: "opentelemetry.io", Resource: "opentelemetrycollectors"}, + {Verb: "update", APIGroup: "opentelemetry.io", Resource: "opentelemetrycollectors"}, + {Verb: "delete", APIGroup: "opentelemetry.io", Resource: "opentelemetrycollectors"}, + {Verb: "get", Resource: "pods"}, + {Verb: "list", Resource: "pods"}, + }, nil +} diff --git a/cmd/operator-opamp-bridge/internal/proxy/noop.go b/cmd/operator-opamp-bridge/internal/proxy/noop.go new file mode 100644 index 0000000000..fcd064dd22 --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/proxy/noop.go @@ -0,0 +1,28 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package proxy + +import ( + "github.com/google/uuid" + "github.com/open-telemetry/opamp-go/protobufs" +) + +// NoopServer satisfies the agent's proxy dependency when no OpAMP proxy is running. +type NoopServer struct{} + +func (NoopServer) GetAgentsByHostname() map[string]uuid.UUID { + return map[string]uuid.UUID{} +} + +func (NoopServer) GetConfigurations() map[uuid.UUID]*protobufs.EffectiveConfig { + return map[uuid.UUID]*protobufs.EffectiveConfig{} +} + +func (NoopServer) GetHealth() map[uuid.UUID]*protobufs.ComponentHealth { + return map[uuid.UUID]*protobufs.ComponentHealth{} +} + +func (NoopServer) HasUpdates() <-chan struct{} { + return make(chan struct{}) +} diff --git a/cmd/operator-opamp-bridge/internal/proxy/server.go b/cmd/operator-opamp-bridge/internal/proxy/server.go index 2d86cd6972..f1a4c720e6 100644 --- a/cmd/operator-opamp-bridge/internal/proxy/server.go +++ b/cmd/operator-opamp-bridge/internal/proxy/server.go @@ -112,7 +112,7 @@ func (s *OpAMPProxy) onDisconnect(conn types.Connection) { } delete(s.connections, conn) // Tell listeners to get updates. - s.updatesChan <- struct{}{} + s.signalUpdate() } func (s *OpAMPProxy) onMessage(_ context.Context, conn types.Connection, msg *protobufs.AgentToServer) *protobufs.ServerToAgent { @@ -146,12 +146,19 @@ func (s *OpAMPProxy) onMessage(_ context.Context, conn types.Connection, msg *pr } s.mux.Unlock() if agentUpdated { - s.updatesChan <- struct{}{} + s.signalUpdate() } // Send the response back to the Agent. return response } +func (s *OpAMPProxy) signalUpdate() { + select { + case s.updatesChan <- struct{}{}: + default: + } +} + // GetConfigurations implements Server. func (s *OpAMPProxy) GetConfigurations() map[uuid.UUID]*protobufs.EffectiveConfig { s.mux.RLock() diff --git a/cmd/operator-opamp-bridge/internal/standalone/client.go b/cmd/operator-opamp-bridge/internal/standalone/client.go new file mode 100644 index 0000000000..cf943837d7 --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/standalone/client.go @@ -0,0 +1,469 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package standalone + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/go-logr/logr" + "github.com/open-telemetry/opamp-go/protobufs" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/rest" + toolscache "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/config" + bridgemanager "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/manager" + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/operator" +) + +const ( + restartAnnotation = "kubectl.kubernetes.io/restartedAt" +) + +// Client implements operator.ConfigApplier for standalone mode. +// ConfigMaps are the primary managed objects. +type Client struct { + log logr.Logger + k8sClient client.Client + restCfg *rest.Config + cmCache cache.Cache + onUpdate func() + healthMux sync.RWMutex + healthUpdaters map[workloadKey]func() error +} + +type workloadKey struct { + namespace string + workloadType string + workloadName string +} + +// NewClient creates a standalone OpAMP bridge Client that works directly on ConfigMaps +// without the need for CRDs or the operator. +func NewClient(log logr.Logger, c client.Client, restCfg *rest.Config, onUpdate func()) *Client { + return &Client{ + log: log, + k8sClient: c, + restCfg: restCfg, + onUpdate: onUpdate, + healthUpdaters: map[workloadKey]func() error{}, + } +} + +// Start creates an informer cache for ConfigMaps so configured agents can +// refresh their effective config when local data changes. +func (c *Client) Start(ctx context.Context) error { + ca, err := cache.New(c.restCfg, cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &v1.ConfigMap{}: {}, + &appsv1.Deployment{}: {}, + &appsv1.DaemonSet{}: {}, + &appsv1.StatefulSet{}: {}, + }, + }) + if err != nil { + return fmt.Errorf("failed to create standalone cache: %w", err) + } + + configMapInformer, err := ca.GetInformer(ctx, &v1.ConfigMap{}) + if err != nil { + return fmt.Errorf("failed to get ConfigMap informer: %w", err) + } + configMapHandler := toolscache.ResourceEventHandlerFuncs{ + AddFunc: func(_ any) { c.notifyUpdate() }, + UpdateFunc: func(_, _ any) { c.notifyUpdate() }, + DeleteFunc: func(_ any) { c.notifyUpdate() }, + } + if _, err = configMapInformer.AddEventHandler(configMapHandler); err != nil { + return fmt.Errorf("failed to add ConfigMap event handler: %w", err) + } + if err := c.addWorkloadEventHandler(ctx, ca, &appsv1.Deployment{}, "deployment"); err != nil { + return err + } + if err := c.addWorkloadEventHandler(ctx, ca, &appsv1.DaemonSet{}, "daemonset"); err != nil { + return err + } + if err := c.addWorkloadEventHandler(ctx, ca, &appsv1.StatefulSet{}, "statefulset"); err != nil { + return err + } + + go func() { + if err := ca.Start(ctx); err != nil { + if errors.Is(err, context.Canceled) { + return + } + c.log.Error(err, "standalone cache stopped with error") + } + }() + + if !ca.WaitForCacheSync(ctx) { + return errors.New("timed out waiting for ConfigMap cache to sync") + } + + c.cmCache = ca + c.log.Info("standalone informer cache synced") + return nil +} + +// addWorkloadEventHandler registers an informer handler for one supported workload kind. +// workload is the Kubernetes object type to watch, and workloadType is the lowercase type used in standalone config. +func (c *Client) addWorkloadEventHandler(ctx context.Context, ca cache.Cache, workload client.Object, workloadType string) error { + informer, err := ca.GetInformer(ctx, workload) + if err != nil { + return fmt.Errorf("failed to get %s informer: %w", workloadType, err) + } + + handler := toolscache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { c.notifyWorkloadHealthUpdate(workloadType, obj) }, + UpdateFunc: func(_, newObj any) { c.notifyWorkloadHealthUpdate(workloadType, newObj) }, + DeleteFunc: func(obj any) { c.notifyWorkloadHealthUpdate(workloadType, obj) }, + } + if _, err = informer.AddEventHandler(handler); err != nil { + return fmt.Errorf("failed to add %s event handler: %w", workloadType, err) + } + return nil +} + +func (c *Client) notifyUpdate() { + if c.onUpdate != nil { + c.onUpdate() + } +} + +// notifyHealthUpdate invokes the registered OpAMP health update for a configured workload. +func (c *Client) notifyHealthUpdate(namespace, workloadType, workloadName string) { + key := newWorkloadKey(namespace, workloadType, workloadName) + c.healthMux.RLock() + updateHealth := c.healthUpdaters[key] + c.healthMux.RUnlock() + if updateHealth == nil { + return + } + if err := updateHealth(); err != nil { + c.log.Error(err, "failed to update health after workload change", "workloadType", workloadType, "name", workloadName, "namespace", namespace) + } +} + +// RegisterHealthUpdater wires workload informer events to the matching OpAMP agent. +func (c *Client) RegisterHealthUpdater(agent config.StandaloneAgentConfig, updateHealth func() error) { + c.healthMux.Lock() + defer c.healthMux.Unlock() + c.healthUpdaters[newWorkloadKey(agent.Namespace, agent.WorkloadRef.Kind, agent.WorkloadRef.Name)] = updateHealth +} + +func newWorkloadKey(namespace, workloadType, workloadName string) workloadKey { + return workloadKey{ + namespace: namespace, + workloadType: strings.ToLower(workloadType), + workloadName: workloadName, + } +} + +// notifyWorkloadHealthUpdate extracts a workload from an informer event and updates its matching agent health. +func (c *Client) notifyWorkloadHealthUpdate(workloadType string, obj any) { + workload := eventObject(obj) + if workload == nil { + return + } + c.notifyHealthUpdate(workload.GetNamespace(), workloadType, workload.GetName()) +} + +// eventObject normalizes informer event objects, including delete tombstones, into controller-runtime objects. +func eventObject(obj any) client.Object { + if object, ok := obj.(client.Object); ok { + return object + } + tombstone, ok := obj.(toolscache.DeletedFinalStateUnknown) + if !ok { + return nil + } + object, _ := tombstone.Obj.(client.Object) + return object +} + +// CheckPermissions verifies the Kubernetes access standalone mode needs before starting agents. +func (c *Client) CheckPermissions(ctx context.Context, agents []config.StandaloneAgentConfig, remoteConfigEnabled bool) error { + perms, err := ListRequiredPermissions(agents, remoteConfigEnabled) + if err != nil { + return err + } + if err := bridgemanager.CheckPermissions(ctx, c.k8sClient, perms); err != nil { + return fmt.Errorf("standalone permission check failed: %w", err) + } + return nil +} + +// ListRequiredPermissions builds the Kubernetes permissions needed for standalone watches and configured agents. +// remoteConfigEnabled adds update permissions for managed ConfigMaps and workloads because applying config triggers rollouts. +func ListRequiredPermissions(agents []config.StandaloneAgentConfig, remoteConfigEnabled bool) ([]bridgemanager.Permission, error) { + perms := []bridgemanager.Permission{} + for _, rule := range []struct { + apiGroup string + resource string + }{ + {resource: "configmaps"}, + {apiGroup: "apps", resource: "deployments"}, + {apiGroup: "apps", resource: "daemonsets"}, + {apiGroup: "apps", resource: "statefulsets"}, + } { + for _, verb := range []string{"list", "watch"} { + perms = append(perms, bridgemanager.Permission{Verb: verb, APIGroup: rule.apiGroup, Resource: rule.resource}) + } + } + + for _, agent := range agents { + workloadResource, err := standaloneWorkloadResource(agent.WorkloadRef.Kind) + if err != nil { + return nil, err + } + perms = append(perms, bridgemanager.Permission{Verb: "get", APIGroup: "apps", Resource: workloadResource, Namespace: agent.Namespace, Name: agent.WorkloadRef.Name}) + if remoteConfigEnabled { + perms = append(perms, bridgemanager.Permission{Verb: "update", APIGroup: "apps", Resource: workloadResource, Namespace: agent.Namespace, Name: agent.WorkloadRef.Name}) + } + for _, entry := range agent.Config { + if entry.Kind != config.StandaloneConfigEntryKindConfigMap { + continue + } + perms = append(perms, bridgemanager.Permission{Verb: "get", Resource: "configmaps", Namespace: agent.Namespace, Name: entry.Name}) + if remoteConfigEnabled { + perms = append(perms, bridgemanager.Permission{Verb: "update", Resource: "configmaps", Namespace: agent.Namespace, Name: entry.Name}) + } + } + } + + return perms, nil +} + +func standaloneWorkloadResource(workloadType string) (string, error) { + switch strings.ToLower(workloadType) { + case "deployment": + return "deployments", nil + case "daemonset": + return "daemonsets", nil + case "statefulset": + return "statefulsets", nil + default: + return "", fmt.Errorf("unsupported workload type %q", workloadType) + } +} + +// getConfigMapFile reads the specified config file from a k8s configmap and returns it as raw bytes. +func (c *Client) getConfigMapFile(namespace string, entry config.StandaloneConfigEntry) ([]byte, error) { + cm := &v1.ConfigMap{} + if err := c.k8sClient.Get(context.Background(), client.ObjectKey{Name: entry.Name, Namespace: namespace}, cm); err != nil { + return nil, fmt.Errorf("failed to get ConfigMap %s/%s: %w", namespace, entry.Name, err) + } + body, ok := cm.Data[entry.Key] + if !ok { + return nil, fmt.Errorf("ConfigMap %s/%s does not contain key %q", namespace, entry.Name, entry.Key) + } + return []byte(body), nil +} + +// applyConfigMapFile is called when an opamp server pushes config for an agent. It validates the config, updates the local +// k8s configmap and triggers a rolling restart of the relevant workload. +func (c *Client) applyConfigMapFile(namespace, workloadType, workloadName string, entry config.StandaloneConfigEntry, configFile *protobufs.AgentConfigFile) error { + if len(configFile.Body) == 0 { + return errors.New("invalid config to apply: config is empty") + } + if err := validateCollectorConfigEntry(string(configFile.Body)); err != nil { + return fmt.Errorf("invalid collector config: %w", err) + } + + existing := &v1.ConfigMap{} + err := c.k8sClient.Get(context.Background(), client.ObjectKey{Name: entry.Name, Namespace: namespace}, existing) + if apierrors.IsNotFound(err) { + return fmt.Errorf("standalone mode does not support creating ConfigMap %s/%s", namespace, entry.Name) + } else if err != nil { + return fmt.Errorf("failed to get ConfigMap %s/%s: %w", namespace, entry.Name, err) + } + + if existing.Data == nil { + existing.Data = map[string]string{} + } + existing.Data[entry.Key] = string(configFile.Body) + if updateErr := c.k8sClient.Update(context.Background(), existing); updateErr != nil { + return fmt.Errorf("failed to update ConfigMap %s/%s: %w", namespace, entry.Name, updateErr) + } + c.log.Info("Updated ConfigMap key", "name", entry.Name, "namespace", namespace, "key", entry.Key) + + if err := c.triggerRollout(context.Background(), namespace, workloadType, workloadName); err != nil { + return fmt.Errorf("failed to trigger rollout for %s/%s: %w", namespace, workloadName, err) + } + return nil +} + +// triggerRollout updates the workload pod template restart annotation so Kubernetes rolls out the new ConfigMap content. +func (c *Client) triggerRollout(ctx context.Context, namespace, workloadType, workloadName string) error { + restartVal := time.Now().Format(time.RFC3339) + + switch strings.ToLower(workloadType) { + case "deployment": + deploy := &appsv1.Deployment{} + if err := c.k8sClient.Get(ctx, client.ObjectKey{Name: workloadName, Namespace: namespace}, deploy); err != nil { + return fmt.Errorf("failed to get Deployment %s/%s for rollout: %w", namespace, workloadName, err) + } + if deploy.Spec.Template.Annotations == nil { + deploy.Spec.Template.Annotations = map[string]string{} + } + deploy.Spec.Template.Annotations[restartAnnotation] = restartVal + if err := c.k8sClient.Update(ctx, deploy); err != nil { + return fmt.Errorf("failed to trigger rollout for Deployment %s/%s: %w", namespace, workloadName, err) + } + case "daemonset": + ds := &appsv1.DaemonSet{} + if err := c.k8sClient.Get(ctx, client.ObjectKey{Name: workloadName, Namespace: namespace}, ds); err != nil { + return fmt.Errorf("failed to get DaemonSet %s/%s for rollout: %w", namespace, workloadName, err) + } + if ds.Spec.Template.Annotations == nil { + ds.Spec.Template.Annotations = map[string]string{} + } + ds.Spec.Template.Annotations[restartAnnotation] = restartVal + if err := c.k8sClient.Update(ctx, ds); err != nil { + return fmt.Errorf("failed to trigger rollout for DaemonSet %s/%s: %w", namespace, workloadName, err) + } + case "statefulset": + sts := &appsv1.StatefulSet{} + if err := c.k8sClient.Get(ctx, client.ObjectKey{Name: workloadName, Namespace: namespace}, sts); err != nil { + return fmt.Errorf("failed to get StatefulSet %s/%s for rollout: %w", namespace, workloadName, err) + } + if sts.Spec.Template.Annotations == nil { + sts.Spec.Template.Annotations = map[string]string{} + } + sts.Spec.Template.Annotations[restartAnnotation] = restartVal + if err := c.k8sClient.Update(ctx, sts); err != nil { + return fmt.Errorf("failed to trigger rollout for StatefulSet %s/%s: %w", namespace, workloadName, err) + } + default: + return fmt.Errorf("unsupported workload type %q", workloadType) + } + + c.log.Info("Triggered workload rollout", "workloadType", workloadType, "name", workloadName, "namespace", namespace) + return nil +} + +type workloadReplicaStatus struct { + ready int32 + desired int32 +} + +func (s workloadReplicaStatus) Healthy() bool { + return s.desired > 0 && s.ready == s.desired +} + +func (s workloadReplicaStatus) String() string { + return strconv.Itoa(int(s.ready)) + "/" + strconv.Itoa(int(s.desired)) +} + +func desiredReplicas(replicas *int32) int32 { + if replicas == nil { + return 1 + } + return *replicas +} + +// getWorkloadStatusReplicas reads ready and desired replica counts for the configured standalone workload. +func (c *Client) getWorkloadStatusReplicas(ctx context.Context, namespace, workloadType, workloadName string) (workloadReplicaStatus, error) { + switch strings.ToLower(workloadType) { + case "deployment": + deploy := &appsv1.Deployment{} + if err := c.k8sClient.Get(ctx, client.ObjectKey{Name: workloadName, Namespace: namespace}, deploy); err != nil { + return workloadReplicaStatus{}, fmt.Errorf("failed to get Deployment %s/%s status.replicas: %w", namespace, workloadName, err) + } + return workloadReplicaStatus{ready: deploy.Status.ReadyReplicas, desired: desiredReplicas(deploy.Spec.Replicas)}, nil + case "daemonset": + ds := &appsv1.DaemonSet{} + if err := c.k8sClient.Get(ctx, client.ObjectKey{Name: workloadName, Namespace: namespace}, ds); err != nil { + return workloadReplicaStatus{}, fmt.Errorf("failed to get DaemonSet %s/%s status.replicas: %w", namespace, workloadName, err) + } + return workloadReplicaStatus{ready: ds.Status.NumberReady, desired: ds.Status.DesiredNumberScheduled}, nil + case "statefulset": + sts := &appsv1.StatefulSet{} + if err := c.k8sClient.Get(ctx, client.ObjectKey{Name: workloadName, Namespace: namespace}, sts); err != nil { + return workloadReplicaStatus{}, fmt.Errorf("failed to get StatefulSet %s/%s status.replicas: %w", namespace, workloadName, err) + } + return workloadReplicaStatus{ready: sts.Status.ReadyReplicas, desired: desiredReplicas(sts.Spec.Replicas)}, nil + default: + return workloadReplicaStatus{}, fmt.Errorf("unsupported workload type %q", workloadType) + } +} + +// ScopedApplier returns a ConfigApplier limited to one configured standalone agent. +func (c *Client) ScopedApplier(agent config.StandaloneAgentConfig) operator.ConfigApplier { + return &scopedApplier{ + client: c, + agent: agent, + } +} + +type scopedApplier struct { + client *Client + agent config.StandaloneAgentConfig +} + +var _ operator.ConfigApplier = &scopedApplier{} + +// Apply writes the named remote config entry for this standalone agent. +// name must match one entry from the agent's standalone config map. +func (s *scopedApplier) Apply(name string, configFile *protobufs.AgentConfigFile) error { + entry, ok := s.agent.Config[name] + if !ok { + return fmt.Errorf("standalone agent %q does not manage config %q", s.agent.WorkloadRef.Name, name) + } + if entry.Kind != config.StandaloneConfigEntryKindConfigMap { + return fmt.Errorf("unsupported standalone config kind %q", entry.Kind) + } + return s.client.applyConfigMapFile(s.agent.Namespace, s.agent.WorkloadRef.Kind, s.agent.WorkloadRef.Name, entry, configFile) +} + +func (*scopedApplier) Delete(name string) error { + return fmt.Errorf("standalone mode does not support deleting ConfigMap %s", name) +} + +// ListInstances reports this standalone workload's current ConfigMap data as effective OpAMP config. +func (s *scopedApplier) ListInstances() ([]operator.CollectorInstance, error) { + configMap := make(map[string]operator.ConfigFile, len(s.agent.Config)) + for remoteName, entry := range s.agent.Config { + if entry.Kind != config.StandaloneConfigEntryKindConfigMap { + continue + } + body, err := s.client.getConfigMapFile(s.agent.Namespace, entry) + if err != nil { + return nil, err + } + configMap[remoteName] = operator.ConfigFile{ + Body: body, + ContentType: "yaml", + } + } + if len(configMap) == 0 { + return []operator.CollectorInstance{}, nil + } + return []operator.CollectorInstance{ + newStandaloneCollectorInstance(s.agent.WorkloadRef.Name, s.agent.Namespace, configMap), + }, nil +} + +// GetHealth reports the standalone workload's replica readiness as OpAMP component health. +func (s *scopedApplier) GetHealth() (operator.Health, error) { + statusReplicas, err := s.client.getWorkloadStatusReplicas(context.Background(), s.agent.Namespace, s.agent.WorkloadRef.Kind, s.agent.WorkloadRef.Name) + if err != nil { + return operator.Health{}, err + } + return operator.Health{ + Healthy: statusReplicas.Healthy(), + Status: statusReplicas.String(), + Children: map[string]operator.Health{}, + }, nil +} diff --git a/cmd/operator-opamp-bridge/internal/standalone/client_test.go b/cmd/operator-opamp-bridge/internal/standalone/client_test.go new file mode 100644 index 0000000000..ff3a12734a --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/standalone/client_test.go @@ -0,0 +1,449 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package standalone + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/open-telemetry/opamp-go/protobufs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + authorizationv1 "k8s.io/api/authorization/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/config" + bridgemanager "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/manager" + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/operator" +) + +const validCollectorConfig = `receivers: + otlp: + protocols: + grpc: +exporters: + debug: +service: + pipelines: + traces: + receivers: + - otlp + exporters: + - debug +` + +func getFakeK8sClient(t *testing.T, objs ...client.Object) client.Client { + scheme := runtime.NewScheme() + require.NoError(t, v1.AddToScheme(scheme)) + require.NoError(t, appsv1.AddToScheme(scheme)) + require.NoError(t, authorizationv1.AddToScheme(scheme)) + builder := fake.NewClientBuilder().WithScheme(scheme) + for _, obj := range objs { + builder = builder.WithObjects(obj) + } + return builder.Build() +} + +func getPermissionReviewClient(t *testing.T, allowed func(authorizationv1.ResourceAttributes) bool) client.Client { + scheme := runtime.NewScheme() + require.NoError(t, v1.AddToScheme(scheme)) + require.NoError(t, appsv1.AddToScheme(scheme)) + require.NoError(t, authorizationv1.AddToScheme(scheme)) + return fake.NewClientBuilder(). + WithScheme(scheme). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + review, ok := obj.(*authorizationv1.SelfSubjectAccessReview) + if !ok { + return c.Create(ctx, obj, opts...) + } + review.Status.Allowed = allowed(*review.Spec.ResourceAttributes) + if !review.Status.Allowed { + review.Status.Reason = "denied by test" + } + return nil + }, + }). + Build() +} + +func testConfigMap() *v1.ConfigMap { + return &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "collector-config", + Namespace: "default", + }, + Data: map[string]string{ + "collector.yaml": validCollectorConfig, + "extra.yaml": "extra: true", + }, + } +} + +func testAgentConfig() config.StandaloneAgentConfig { + return config.StandaloneAgentConfig{ + Namespace: "default", + Type: "otel-collector", + WorkloadRef: config.StandaloneWorkloadRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "standalone-collector", + }, + Config: map[string]config.StandaloneConfigEntry{ + "collector": { + Kind: config.StandaloneConfigEntryKindConfigMap, + Name: "collector-config", + Key: "collector.yaml", + }, + }, + } +} + +func testDeployment() *appsv1.Deployment { + replicas := ptr.To[int32](10) + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "standalone-collector", + Namespace: "default", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: replicas, + Template: v1.PodTemplateSpec{}, + }, + Status: appsv1.DeploymentStatus{ + ReadyReplicas: 5, + Replicas: 5, + }, + } +} + +func testStatefulSet() *appsv1.StatefulSet { + replicas := ptr.To[int32](4) + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "standalone-collector", + Namespace: "default", + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: replicas, + }, + Status: appsv1.StatefulSetStatus{ + ReadyReplicas: 3, + Replicas: 3, + }, + } +} + +func testDaemonSet() *appsv1.DaemonSet { + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "standalone-collector", + Namespace: "default", + }, + Status: appsv1.DaemonSetStatus{ + NumberReady: 7, + DesiredNumberScheduled: 9, + }, + } +} + +func newTestClient(k8s client.Client) *Client { + return NewClient(logr.Discard(), k8s, nil, nil) +} + +func TestScopedApplierListInstancesReturnsWorkloadWithConfiguredConfigMap(t *testing.T) { + cm := testConfigMap() + c := newTestClient(getFakeK8sClient(t, cm, testDeployment())) + agentCfg := testAgentConfig() + agentCfg.Config["extra"] = config.StandaloneConfigEntry{ + Kind: config.StandaloneConfigEntryKindConfigMap, + Name: "collector-config", + Key: "extra.yaml", + } + + instances, err := c.ScopedApplier(agentCfg).ListInstances() + require.NoError(t, err) + require.Len(t, instances, 1) + + assert.Equal(t, "standalone-collector", instances[0].GetName()) + assert.Equal(t, "default", instances[0].GetNamespace()) + configMap := instances[0].GetConfigMap() + require.Len(t, configMap, 2) + assert.Equal(t, validCollectorConfig, string(configMap["collector"].Body)) + assert.Equal(t, "yaml", configMap["collector"].ContentType) + assert.Equal(t, "extra: true", string(configMap["extra"].Body)) + assert.Equal(t, "yaml", configMap["extra"].ContentType) +} + +func TestClientGetWorkloadStatusReplicas(t *testing.T) { + tests := []struct { + name string + workloadType string + workload client.Object + want string + wantHealthy bool + }{ + { + name: "deployment uses spec replicas during scale up", + workloadType: "deployment", + workload: testDeployment(), + want: "5/10", + wantHealthy: false, + }, + { + name: "deployment defaults desired replicas to one", + workloadType: "deployment", + workload: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "standalone-collector", + Namespace: "default", + }, + Status: appsv1.DeploymentStatus{ + ReadyReplicas: 1, + Replicas: 1, + }, + }, + want: "1/1", + wantHealthy: true, + }, + { + name: "daemonset", + workloadType: "daemonset", + workload: testDaemonSet(), + want: "7/9", + wantHealthy: false, + }, + { + name: "statefulset uses spec replicas during scale up", + workloadType: "statefulset", + workload: testStatefulSet(), + want: "3/4", + wantHealthy: false, + }, + { + name: "statefulset defaults desired replicas to one", + workloadType: "statefulset", + workload: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "standalone-collector", + Namespace: "default", + }, + Status: appsv1.StatefulSetStatus{ + ReadyReplicas: 1, + Replicas: 1, + }, + }, + want: "1/1", + wantHealthy: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := newTestClient(getFakeK8sClient(t, tt.workload)) + + status, err := c.getWorkloadStatusReplicas(context.Background(), "default", tt.workloadType, "standalone-collector") + require.NoError(t, err) + assert.Equal(t, tt.want, status.String()) + assert.Equal(t, tt.wantHealthy, status.Healthy()) + }) + } +} + +func TestScopedApplierGetHealthReturnsRootWorkloadHealth(t *testing.T) { + healthyDeploy := testDeployment() + healthyDeploy.Status.ReadyReplicas = 10 + healthyDeploy.Status.Replicas = 10 + tests := []struct { + name string + workload client.Object + want operator.Health + }{ + { + name: "unhealthy rollout", + workload: testDeployment(), + want: operator.Health{ + Healthy: false, + Status: "5/10", + Children: map[string]operator.Health{}, + }, + }, + { + name: "healthy rollout", + workload: healthyDeploy, + want: operator.Health{ + Healthy: true, + Status: "10/10", + Children: map[string]operator.Health{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := newTestClient(getFakeK8sClient(t, tt.workload)) + + got, err := c.ScopedApplier(testAgentConfig()).GetHealth() + + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestClientNotifyWorkloadHealthUpdate(t *testing.T) { + c := newTestClient(getFakeK8sClient(t)) + called := false + c.RegisterHealthUpdater(testAgentConfig(), func() error { + called = true + return nil + }) + + c.notifyWorkloadHealthUpdate("deployment", testDeployment()) + + assert.True(t, called) +} + +func TestClientCheckPermissionsAllowsConfiguredStandaloneResources(t *testing.T) { + c := newTestClient(getPermissionReviewClient(t, func(_ authorizationv1.ResourceAttributes) bool { + return true + })) + + err := c.CheckPermissions(context.Background(), []config.StandaloneAgentConfig{testAgentConfig()}, true) + + require.NoError(t, err) +} + +func TestClientCheckPermissionsRejectsMissingConfiguredResourcePermission(t *testing.T) { + c := newTestClient(getPermissionReviewClient(t, func(attrs authorizationv1.ResourceAttributes) bool { + return attrs.Verb != "update" || attrs.Resource != "deployments" || attrs.Name != "standalone-collector" + })) + + err := c.CheckPermissions(context.Background(), []config.StandaloneAgentConfig{testAgentConfig()}, true) + + require.Error(t, err) + assert.Contains(t, err.Error(), "standalone permission check failed") + assert.Contains(t, err.Error(), "missing update permission for apps/deployments default/standalone-collector") +} + +func TestPermissionsCheckerRejectsDeniedPermission(t *testing.T) { + k8sClient := getPermissionReviewClient(t, func(_ authorizationv1.ResourceAttributes) bool { + return false + }) + + err := bridgemanager.CheckPermissions(context.Background(), k8sClient, []bridgemanager.Permission{{ + Verb: "update", + APIGroup: "apps", + Resource: "deployments", + Namespace: "default", + Name: "standalone-collector", + }}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "missing update permission for apps/deployments default/standalone-collector") +} + +func TestScopedApplierApplyUpdatesConfiguredConfigMapKeyAndRestartsWorkload(t *testing.T) { + cm := testConfigMap() + deploy := testDeployment() + k8s := getFakeK8sClient(t, cm, deploy) + c := newTestClient(k8s) + + updatedConfig := `receivers: + otlp: + protocols: + http: +exporters: + debug: +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] +` + err := c.ScopedApplier(testAgentConfig()).Apply("collector", &protobufs.AgentConfigFile{Body: []byte(updatedConfig)}) + require.NoError(t, err) + + updated := &v1.ConfigMap{} + require.NoError(t, k8s.Get(context.Background(), client.ObjectKey{Name: "collector-config", Namespace: "default"}, updated)) + assert.Equal(t, updatedConfig, updated.Data["collector.yaml"]) + assert.Equal(t, "extra: true", updated.Data["extra.yaml"]) + + updatedDeploy := &appsv1.Deployment{} + require.NoError(t, k8s.Get(context.Background(), client.ObjectKey{Name: "standalone-collector", Namespace: "default"}, updatedDeploy)) + assert.NotEmpty(t, updatedDeploy.Spec.Template.Annotations[restartAnnotation]) +} + +func TestScopedApplierApplyRejectsInvalidCollectorConfig(t *testing.T) { + c := newTestClient(getFakeK8sClient(t, testConfigMap(), testDeployment())) + + invalidConfig := `receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:65536 +exporters: + debug: +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] +` + err := c.ScopedApplier(testAgentConfig()).Apply("collector", &protobufs.AgentConfigFile{Body: []byte(invalidConfig)}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "collector config failed operator validation") + assert.Contains(t, err.Error(), "the OpenTelemetry config is incorrect") +} + +func TestScopedApplierApplyRejectsMissingPipelineComponent(t *testing.T) { + c := newTestClient(getFakeK8sClient(t, testConfigMap(), testDeployment())) + + invalidConfig := `receivers: + otlp: + protocols: + http: +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] +` + err := c.ScopedApplier(testAgentConfig()).Apply("collector", &protobufs.AgentConfigFile{Body: []byte(invalidConfig)}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid collector config") + assert.Contains(t, err.Error(), "no exporters configured") +} + +func TestScopedApplierApplyRejectsUnknownRemoteName(t *testing.T) { + c := newTestClient(getFakeK8sClient(t, testConfigMap())) + + err := c.ScopedApplier(testAgentConfig()).Apply("unknown", &protobufs.AgentConfigFile{Body: []byte(validCollectorConfig)}) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not manage config") +} + +func TestScopedApplierApplyDoesNotCreateConfigMap(t *testing.T) { + c := newTestClient(getFakeK8sClient(t)) + + err := c.ScopedApplier(testAgentConfig()).Apply("collector", &protobufs.AgentConfigFile{Body: []byte(validCollectorConfig)}) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not support creating ConfigMap") +} + +func TestScopedApplierDeleteUnsupported(t *testing.T) { + c := newTestClient(getFakeK8sClient(t, testConfigMap())) + + err := c.ScopedApplier(testAgentConfig()).Delete("collector") + require.Error(t, err) + assert.Contains(t, err.Error(), "does not support deleting") +} diff --git a/cmd/operator-opamp-bridge/internal/standalone/configmap_instance.go b/cmd/operator-opamp-bridge/internal/standalone/configmap_instance.go new file mode 100644 index 0000000000..02393df351 --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/standalone/configmap_instance.go @@ -0,0 +1,39 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package standalone + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/operator" +) + +var _ operator.CollectorInstance = &standaloneCollectorInstance{} + +// standaloneCollectorInstance represents a bridge-managed standalone workload. +type standaloneCollectorInstance struct { + name string + namespace string + configMap map[string]operator.ConfigFile +} + +func newStandaloneCollectorInstance(name, namespace string, configMap map[string]operator.ConfigFile) *standaloneCollectorInstance { + return &standaloneCollectorInstance{name: name, namespace: namespace, configMap: configMap} +} + +func (p *standaloneCollectorInstance) GetName() string { + return p.name +} + +func (p *standaloneCollectorInstance) GetNamespace() string { + return p.namespace +} + +func (*standaloneCollectorInstance) GetDeletionTimestamp() *metav1.Time { + return nil +} + +func (p *standaloneCollectorInstance) GetConfigMap() map[string]operator.ConfigFile { + return p.configMap +} diff --git a/cmd/operator-opamp-bridge/internal/standalone/validate.go b/cmd/operator-opamp-bridge/internal/standalone/validate.go new file mode 100644 index 0000000000..497a04f570 --- /dev/null +++ b/cmd/operator-opamp-bridge/internal/standalone/validate.go @@ -0,0 +1,96 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package standalone + +import ( + "context" + "errors" + "fmt" + "strings" + + "sigs.k8s.io/yaml" + + "github.com/open-telemetry/opentelemetry-operator/apis/v1beta1" + "github.com/open-telemetry/opentelemetry-operator/internal/webhook" +) + +// validateCollectorConfigEntry checks that a standalone config value is valid collector config. +// It returns an error if the body is not valid Otel config. This helps report bad configs back to the opamp server. +func validateCollectorConfigEntry(body string) error { + if strings.TrimSpace(body) == "" { + return errors.New("config value is empty") + } + + var cfg v1beta1.Config + if err := yaml.Unmarshal([]byte(body), &cfg); err != nil { + return fmt.Errorf("failed to parse collector config: %w", err) + } + + collector := &v1beta1.OpenTelemetryCollector{ + Spec: v1beta1.OpenTelemetryCollectorSpec{ + Mode: v1beta1.ModeDeployment, + Config: cfg, + }, + } + if _, err := (webhook.CollectorWebhook{}).Validate(context.Background(), collector); err != nil { + return fmt.Errorf("collector config failed operator validation: %w", err) + } + if err := validateCollectorConfigReferences(cfg); err != nil { + return fmt.Errorf("collector config failed references validation: %w", err) + } + + return nil +} + +func validateCollectorConfigReferences(cfg v1beta1.Config) error { + if len(cfg.Receivers.Object) == 0 { + return errors.New("no receivers configured") + } + if len(cfg.Exporters.Object) == 0 { + return errors.New("no exporters configured") + } + if len(cfg.Service.Pipelines) == 0 { + return errors.New("no pipelines configured") + } + + for pipelineName, pipeline := range cfg.Service.Pipelines { + if pipeline == nil { + return fmt.Errorf("pipeline %s is empty", pipelineName) + } + for _, receiver := range pipeline.Receivers { + if !componentExists(receiver, cfg.Receivers.Object, cfg.Connectors) { + return fmt.Errorf("pipeline %s references non-existent receiver %s", pipelineName, receiver) + } + } + for _, processor := range pipeline.Processors { + if cfg.Processors == nil || !componentExists(processor, cfg.Processors.Object, nil) { + return fmt.Errorf("pipeline %s references non-existent processor %s", pipelineName, processor) + } + } + for _, exporter := range pipeline.Exporters { + if !componentExists(exporter, cfg.Exporters.Object, cfg.Connectors) { + return fmt.Errorf("pipeline %s references non-existent exporter %s", pipelineName, exporter) + } + } + } + + for _, extension := range cfg.Service.Extensions { + if cfg.Extensions == nil || !componentExists(extension, cfg.Extensions.Object, nil) { + return fmt.Errorf("service references non-existent extension %s", extension) + } + } + + return nil +} + +func componentExists(name string, components map[string]any, connectors *v1beta1.AnyConfig) bool { + if _, ok := components[name]; ok { + return true + } + if connectors != nil { + _, ok := connectors.Object[name] + return ok + } + return false +} diff --git a/cmd/operator-opamp-bridge/main.go b/cmd/operator-opamp-bridge/main.go index f3602149bb..a6334f1bee 100644 --- a/cmd/operator-opamp-bridge/main.go +++ b/cmd/operator-opamp-bridge/main.go @@ -7,11 +7,19 @@ import ( "context" "os" "os/signal" + "time" - "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/agent" + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/client" + + opampagent "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/agent" "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/config" + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/healthcheck" + bridgemanager "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/manager" "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/operator" + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/operatorbridge" "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/proxy" + "github.com/open-telemetry/opentelemetry-operator/cmd/operator-opamp-bridge/internal/standalone" ) func main() { @@ -22,34 +30,91 @@ func main() { l.Error(configLoadErr, "Unable to load configuration") os.Exit(1) } - l.Info("Starting the Remote Configuration service") + l.Info("Starting the Remote Configuration service", "mode", cfg.Mode) kubeClient, kubeErr := cfg.GetKubernetesClient() if kubeErr != nil { l.Error(kubeErr, "Couldn't create kubernetes client") os.Exit(1) } - operatorClient := operator.NewClient(cfg.Name, l.WithName("operator-client"), kubeClient, cfg.GetComponentsAllowed()) - opampClient := cfg.CreateClient() - opampProxy := proxy.NewOpAMPProxy(l.WithName("server"), cfg.ListenAddr) - opampAgent := agent.NewAgent(l.WithName("agent"), operatorClient, cfg, opampClient, opampProxy) + // signalCtx is cancelled on interrupt, which stops the informer goroutine. + signalCtx, cancelSignal := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancelSignal() - if err := opampAgent.Start(); err != nil { - l.Error(err, "Cannot start OpAMP client") + options := commonManagerOptions(l, cfg, kubeClient) + if cfg.IsStandaloneMode() { + options = append(options, standaloneManagerOptions(l, cfg, kubeClient)...) + } else { + options = append(options, operatorManagerOptions(l, cfg, kubeClient)...) + } + manager, err := bridgemanager.New(options...) + if err != nil { + l.Error(err, "Cannot configure OpAMP bridge") os.Exit(1) } - if err := opampProxy.Start(); err != nil { - l.Error(err, "failed to start OpAMP Server") + if err := manager.Start(signalCtx); err != nil { + l.Error(err, "Cannot start OpAMP bridge") os.Exit(1) } + <-signalCtx.Done() + shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelShutdown() + manager.Shutdown(shutdownCtx) +} + +func commonManagerOptions(log logr.Logger, cfg *config.Config, c client.Client) []bridgemanager.Option { + return []bridgemanager.Option{ + bridgemanager.WithLogger(log), + bridgemanager.WithHealthServer(healthcheck.NewServer(log.WithName("healthcheck"), cfg.HealthListenAddr)), + bridgemanager.WithPermissionReviewClient(c), + } +} + +func standaloneManagerOptions(log logr.Logger, cfg *config.Config, c client.Client) []bridgemanager.Option { + runtimes := make([]bridgemanager.Runtime, 0, len(cfg.Standalone.Agents)) + standaloneClient := standalone.NewClient(log.WithName("client"), c, cfg.GetRestConfig(), func() { + for _, runtime := range runtimes { + if err := runtime.Client.UpdateEffectiveConfig(context.Background()); err != nil { + log.Error(err, "failed to update effective config", "agent", runtime.Name) + } + } + }) + for _, configuredAgent := range cfg.Standalone.Agents { + agentCfg := config.NewStandaloneAgentConfig(cfg, configuredAgent) + opampClient := agentCfg.CreateClient() + applier := standaloneClient.ScopedApplier(configuredAgent) + opampAgent := opampagent.NewAgent(log.WithName(configuredAgent.WorkloadRef.Name), applier, agentCfg, opampClient, proxy.NoopServer{}) + standaloneClient.RegisterHealthUpdater(configuredAgent, opampAgent.UpdateHealth) + runtimes = append(runtimes, bridgemanager.Runtime{ + Name: configuredAgent.WorkloadRef.Name, + Client: opampClient, + OpAMPAgent: opampAgent, + }) + } + return []bridgemanager.Option{ + bridgemanager.WithRuntimes(runtimes), + bridgemanager.WithKubernetesClient(standaloneClient), + bridgemanager.WithRequiredPermissions(func() ([]bridgemanager.Permission, error) { + return standalone.ListRequiredPermissions(cfg.Standalone.Agents, cfg.RemoteConfigEnabled()) + }), + } +} - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt) - <-interrupt - opampAgent.Shutdown() - proxyStopErr := opampProxy.Stop(context.Background()) - if proxyStopErr != nil { - l.Error(proxyStopErr, "failed to shutdown proxy server") +func operatorManagerOptions(log logr.Logger, cfg *config.Config, c client.Client) []bridgemanager.Option { + opampClient := cfg.CreateClient() + applier := operator.NewClient(cfg.Name, log.WithName("operator-client"), c, cfg.GetComponentsAllowed()) + opampProxy := proxy.NewOpAMPProxy(log.WithName("server"), cfg.ListenAddr) + opampAgent := opampagent.NewAgent(log.WithName("agent"), applier, cfg, opampClient, opampProxy) + return []bridgemanager.Option{ + bridgemanager.WithOpAMPProxy(opampProxy), + bridgemanager.WithRequiredPermissions(operatorbridge.ListRequiredPermissions), + bridgemanager.WithRuntimes([]bridgemanager.Runtime{ + { + Name: cfg.Name, + Client: opampClient, + OpAMPAgent: opampAgent, + }, + }), } } diff --git a/cmd/operator-opamp-bridge/manifests/standalone/README.md b/cmd/operator-opamp-bridge/manifests/standalone/README.md new file mode 100644 index 0000000000..6901f50ecc --- /dev/null +++ b/cmd/operator-opamp-bridge/manifests/standalone/README.md @@ -0,0 +1,7 @@ +# Standalone OpAMP Bridge Manifests + +This directory contains a Kustomize base for running the OpAMP Bridge without installing the OpenTelemetry Operator, CRDs, or cert-manager. + +It deploys the bridge into the `opentelemetry-opamp-bridge` namespace with a service account, ConfigMap-based configuration, RBAC for ConfigMaps and workload resources, and a single Deployment. + +Use `make deploy-standalone-bridge` to deploy these manifests, or run `kustomize build cmd/operator-opamp-bridge/manifests/standalone` to inspect the rendered resources. diff --git a/cmd/operator-opamp-bridge/manifests/standalone/configmap.yaml b/cmd/operator-opamp-bridge/manifests/standalone/configmap.yaml new file mode 100644 index 0000000000..44183c992b --- /dev/null +++ b/cmd/operator-opamp-bridge/manifests/standalone/configmap.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-opamp-bridge-standalone-config + namespace: opentelemetry-opamp-bridge +data: + config.yaml: | + mode: standalone + endpoint: http://host.docker.internal:8080/v1/opamp + healthListenAddr: ":8081" + heartbeatInterval: 30s + name: opamp-bridge-standalone + capabilities: + AcceptsRemoteConfig: true + ReportsEffectiveConfig: true + ReportsHealth: true + ReportsOwnMetrics: true + ReportsRemoteConfig: true + standalone: + agents: + - namespace: opentelemetry-opamp-bridge + type: otel-collector + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: otel-collector + config: + collector: + kind: configmap + name: otel-collector-conf + key: otel-collector-config + - namespace: opentelemetry-opamp-bridge + type: otel-agent + workloadRef: + apiVersion: apps/v1 + kind: DaemonSet + name: otel-agent + config: + agent: + kind: configmap + name: otel-agent-conf + key: otel-agent-config diff --git a/cmd/operator-opamp-bridge/manifests/standalone/deployment.yaml b/cmd/operator-opamp-bridge/manifests/standalone/deployment.yaml new file mode 100644 index 0000000000..c423331ad2 --- /dev/null +++ b/cmd/operator-opamp-bridge/manifests/standalone/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: otel-opamp-bridge-standalone + namespace: opentelemetry-opamp-bridge + labels: + app.kubernetes.io/name: otel-opamp-bridge-standalone + app.kubernetes.io/component: opamp-bridge +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: otel-opamp-bridge-standalone + template: + metadata: + labels: + app.kubernetes.io/name: otel-opamp-bridge-standalone + app.kubernetes.io/component: opamp-bridge + spec: + serviceAccountName: otel-opamp-bridge-standalone + containers: + - name: bridge + image: operator-opamp-bridge:latest + args: + - --config-file=/conf/config.yaml + - --mode=standalone + ports: + - name: healthz + containerPort: 8081 + protocol: TCP + volumeMounts: + - name: config + mountPath: /conf + readOnly: true + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 5 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + volumes: + - name: config + configMap: + name: otel-opamp-bridge-standalone-config \ No newline at end of file diff --git a/cmd/operator-opamp-bridge/manifests/standalone/kustomization.yaml b/cmd/operator-opamp-bridge/manifests/standalone/kustomization.yaml new file mode 100644 index 0000000000..a56eed8266 --- /dev/null +++ b/cmd/operator-opamp-bridge/manifests/standalone/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: opentelemetry-opamp-bridge + +resources: +- namespace.yaml +- serviceaccount.yaml +- rbac.yaml +- configmap.yaml +- deployment.yaml + +images: +- name: operator-opamp-bridge + newName: ghcr.io/open-telemetry/opentelemetry-operator/operator-opamp-bridge + newTag: v0.150.0-155-g89004787 diff --git a/cmd/operator-opamp-bridge/manifests/standalone/namespace.yaml b/cmd/operator-opamp-bridge/manifests/standalone/namespace.yaml new file mode 100644 index 0000000000..bc25d9b580 --- /dev/null +++ b/cmd/operator-opamp-bridge/manifests/standalone/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: opentelemetry-opamp-bridge diff --git a/cmd/operator-opamp-bridge/manifests/standalone/rbac.yaml b/cmd/operator-opamp-bridge/manifests/standalone/rbac.yaml new file mode 100644 index 0000000000..b8bf6cf24f --- /dev/null +++ b/cmd/operator-opamp-bridge/manifests/standalone/rbac.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: otel-opamp-bridge-standalone +rules: + # ConfigMaps: the bridge watches and updates the configured key; it never creates or deletes them + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "update"] + # Deployments + DaemonSets + StatefulSets: the bridge watches rollout status and updates pod template annotations to trigger rollouts + - apiGroups: ["apps"] + resources: ["deployments", "daemonsets", "statefulsets"] + verbs: ["get", "list", "watch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: otel-opamp-bridge-standalone +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: otel-opamp-bridge-standalone +subjects: + - kind: ServiceAccount + name: otel-opamp-bridge-standalone + namespace: opentelemetry-opamp-bridge diff --git a/cmd/operator-opamp-bridge/manifests/standalone/serviceaccount.yaml b/cmd/operator-opamp-bridge/manifests/standalone/serviceaccount.yaml new file mode 100644 index 0000000000..0c29ca6580 --- /dev/null +++ b/cmd/operator-opamp-bridge/manifests/standalone/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: otel-opamp-bridge-standalone + namespace: opentelemetry-opamp-bridge diff --git a/go.mod b/go.mod index 684ddc862d..78cb9f6a64 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( go.uber.org/zap v1.28.0 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 gotest.tools/v3 v3.5.2 k8s.io/api v0.36.2 k8s.io/apiextensions-apiserver v0.36.2 diff --git a/internal/controllers/testdata/build_opamp_bridge_base.yaml b/internal/controllers/testdata/build_opamp_bridge_base.yaml index fe7e8f3fd2..87ab6398db 100644 --- a/internal/controllers/testdata/build_opamp_bridge_base.yaml +++ b/internal/controllers/testdata/build_opamp_bridge_base.yaml @@ -50,7 +50,22 @@ spec: divisor: "0" resource: limits.cpu image: test + livenessProbe: + httpGet: + path: /healthz + port: healthz name: opamp-bridge-container + ports: + - containerPort: 8080 + name: opamp + protocol: TCP + - containerPort: 8081 + name: healthz + protocol: TCP + readinessProbe: + httpGet: + path: /healthz + port: healthz resources: {} volumeMounts: - mountPath: /conf diff --git a/internal/manifests/opampbridge/container.go b/internal/manifests/opampbridge/container.go index a2f1a6a468..fd79d34693 100644 --- a/internal/manifests/opampbridge/container.go +++ b/internal/manifests/opampbridge/container.go @@ -8,6 +8,7 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" "github.com/open-telemetry/opentelemetry-operator/internal/config" @@ -86,5 +87,33 @@ func Container(cfg config.Config, _ logr.Logger, opampBridge v1alpha1.OpAMPBridg EnvFrom: opampBridge.Spec.EnvFrom, Resources: opampBridge.Spec.Resources, SecurityContext: opampBridge.Spec.SecurityContext, + Ports: []corev1.ContainerPort{ + { + Name: "opamp", + ContainerPort: 8080, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "healthz", + ContainerPort: 8081, + Protocol: corev1.ProtocolTCP, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString("healthz"), + }, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString("healthz"), + }, + }, + }, } } diff --git a/internal/manifests/opampbridge/container_test.go b/internal/manifests/opampbridge/container_test.go index f327023497..9f21a596d5 100644 --- a/internal/manifests/opampbridge/container_test.go +++ b/internal/manifests/opampbridge/container_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" logf "sigs.k8s.io/controller-runtime/pkg/log" "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" @@ -29,6 +30,34 @@ func TestContainerNewDefault(t *testing.T) { // verify assert.Equal(t, "default-image", c.Image) + assert.Equal(t, []corev1.ContainerPort{ + { + Name: "opamp", + ContainerPort: 8080, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "healthz", + ContainerPort: 8081, + Protocol: corev1.ProtocolTCP, + }, + }, c.Ports) + assert.Equal(t, &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString("healthz"), + }, + }, + }, c.LivenessProbe) + assert.Equal(t, &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString("healthz"), + }, + }, + }, c.ReadinessProbe) } func TestContainerWithImageOverridden(t *testing.T) { diff --git a/tests/e2e-opampbridge/standalone/00-assert.yaml b/tests/e2e-opampbridge/standalone/00-assert.yaml new file mode 100644 index 0000000000..da9755c5a4 --- /dev/null +++ b/tests/e2e-opampbridge/standalone/00-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: e2e-test-app-bridge-server +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/e2e-opampbridge/standalone/00-install.yaml b/tests/e2e-opampbridge/standalone/00-install.yaml new file mode 100644 index 0000000000..7ad32726e7 --- /dev/null +++ b/tests/e2e-opampbridge/standalone/00-install.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: e2e-test-app-bridge-server +spec: + replicas: 1 + selector: + matchLabels: + app: e2e-test-app-bridge-server + template: + metadata: + labels: + app: e2e-test-app-bridge-server + spec: + containers: + - name: e2e-test-app-bridge-server + image: ghcr.io/open-telemetry/opentelemetry-operator/e2e-test-app-bridge-server:ve2e + ports: + - containerPort: 4320 + - containerPort: 4321 + resources: + limits: + memory: "128Mi" + cpu: "250m" + requests: + memory: "64Mi" + cpu: "100m" +--- +apiVersion: v1 +kind: Service +metadata: + name: e2e-test-app-bridge-server +spec: + selector: + app: e2e-test-app-bridge-server + ports: + - protocol: TCP + port: 4320 + targetPort: 4320 + name: "opamp" + - protocol: TCP + port: 4321 + targetPort: 4321 + name: "http" diff --git a/tests/e2e-opampbridge/standalone/01-assert.yaml b/tests/e2e-opampbridge/standalone/01-assert.yaml new file mode 100644 index 0000000000..c8788df443 --- /dev/null +++ b/tests/e2e-opampbridge/standalone/01-assert.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: otel-opamp-bridge-standalone +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: standalone-collector +data: + collector.yaml: | + receivers: + otlp: + protocols: + grpc: {} + http: {} + exporters: + debug: {} + service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: standalone-collector +spec: + replicas: 0 diff --git a/tests/e2e-opampbridge/standalone/01-install.yaml b/tests/e2e-opampbridge/standalone/01-install.yaml new file mode 100644 index 0000000000..42299c5d3a --- /dev/null +++ b/tests/e2e-opampbridge/standalone/01-install.yaml @@ -0,0 +1,174 @@ +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: true +metadata: + name: otel-opamp-bridge-standalone +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: otel-opamp-bridge-standalone +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - update + - apiGroups: + - apps + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - get + - list + - watch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: otel-opamp-bridge-standalone +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: otel-opamp-bridge-standalone +subjects: + - kind: ServiceAccount + name: otel-opamp-bridge-standalone + namespace: ($namespace) +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-opamp-bridge-standalone-config +data: + config.yaml: | + mode: standalone + endpoint: ws://e2e-test-app-bridge-server:4320/v1/opamp + healthListenAddr: ":8081" + heartbeatInterval: 1s + name: opamp-bridge-standalone + capabilities: + AcceptsRemoteConfig: true + ReportsEffectiveConfig: true + ReportsRemoteConfig: true + ReportsHealth: true + standalone: + agents: + - namespace: ${NAMESPACE} + type: otel-collector + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: standalone-collector + config: + collector: + kind: configmap + name: standalone-collector + key: collector.yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: otel-opamp-bridge-standalone + labels: + app.kubernetes.io/name: otel-opamp-bridge-standalone + app.kubernetes.io/component: opamp-bridge +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: otel-opamp-bridge-standalone + template: + metadata: + labels: + app.kubernetes.io/name: otel-opamp-bridge-standalone + app.kubernetes.io/component: opamp-bridge + spec: + serviceAccountName: otel-opamp-bridge-standalone + containers: + - name: bridge + image: ($operatorOpampBridgeImage) + args: + - --config-file=/conf/config.yaml + - --mode=standalone + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: healthz + containerPort: 8081 + protocol: TCP + volumeMounts: + - name: config + mountPath: /conf + readOnly: true + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 5 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + volumes: + - name: config + configMap: + name: otel-opamp-bridge-standalone-config +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: standalone-collector +data: + collector.yaml: | + receivers: + otlp: + protocols: + grpc: {} + http: {} + exporters: + debug: {} + service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: standalone-collector +spec: + replicas: 0 + selector: + matchLabels: + app: standalone-collector + template: + metadata: + labels: + app: standalone-collector + spec: + containers: + - name: collector + image: busybox:1.36 + command: ["sleep", "3600"] diff --git a/tests/e2e-opampbridge/standalone/02-assert.yaml b/tests/e2e-opampbridge/standalone/02-assert.yaml new file mode 100644 index 0000000000..3cc0fe4f3c --- /dev/null +++ b/tests/e2e-opampbridge/standalone/02-assert.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: otel-opamp-bridge-standalone +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: standalone-agent +spec: + replicas: 0 diff --git a/tests/e2e-opampbridge/standalone/02-install.yaml b/tests/e2e-opampbridge/standalone/02-install.yaml new file mode 100644 index 0000000000..4e23a8fe35 --- /dev/null +++ b/tests/e2e-opampbridge/standalone/02-install.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: standalone-agent +data: + agent.yaml: | + receivers: + otlp: + protocols: + http: {} + exporters: + debug: {} + service: + pipelines: + metrics: + receivers: [otlp] + exporters: [debug] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: standalone-agent +spec: + replicas: 0 + selector: + matchLabels: + app: standalone-agent + template: + metadata: + labels: + app: standalone-agent + spec: + containers: + - name: collector + image: busybox:1.36 + command: ["sleep", "3600"] +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-opamp-bridge-standalone-config +data: + config.yaml: | + mode: standalone + endpoint: ws://e2e-test-app-bridge-server:4320/v1/opamp + healthListenAddr: ":8081" + heartbeatInterval: 1s + name: opamp-bridge-standalone + capabilities: + AcceptsRemoteConfig: true + ReportsEffectiveConfig: true + ReportsRemoteConfig: true + ReportsHealth: true + standalone: + agents: + - namespace: ${NAMESPACE} + type: otel-collector + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: standalone-collector + config: + collector: + kind: configmap + name: standalone-collector + key: collector.yaml + - namespace: ${NAMESPACE} + type: otel-agent + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: standalone-agent + config: + agent: + kind: configmap + name: standalone-agent + key: agent.yaml diff --git a/tests/e2e-opampbridge/standalone/chainsaw-test.yaml b/tests/e2e-opampbridge/standalone/chainsaw-test.yaml new file mode 100644 index 0000000000..aa398a20d2 --- /dev/null +++ b/tests/e2e-opampbridge/standalone/chainsaw-test.yaml @@ -0,0 +1,325 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: opampbridge-standalone +spec: + steps: + - catch: + - podLogs: + selector: app=e2e-test-app-bridge-server + name: install fake OpAMP server + try: + - apply: + file: 00-install.yaml + - assert: + file: 00-assert.yaml + + - catch: + - podLogs: + selector: app=e2e-test-app-bridge-server + - podLogs: + selector: app.kubernetes.io/name=otel-opamp-bridge-standalone + name: install standalone bridge + try: + - apply: + bindings: + - name: operatorOpampBridgeImage + value: (env('OPERATOROPAMPBRIDGE_IMG')) + file: 01-install.yaml + template: true + - assert: + file: 01-assert.yaml + + - catch: + - podLogs: + selector: app=e2e-test-app-bridge-server + - podLogs: + selector: app.kubernetes.io/name=otel-opamp-bridge-standalone + name: verify initial standalone report + try: + - script: + content: | + #!/bin/bash + set -eu + + last="" + for _ in $(seq 1 60); do + if last=$(kubectl get --raw /api/v1/namespaces/$NAMESPACE/services/e2e-test-app-bridge-server:4321/proxy/agents 2>/dev/null); then + if printf '%s' "$last" | grep -q '"effective_config":{"collector":' && + printf '%s' "$last" | grep -q '"health":' && + printf '%s' "$last" | grep -q '"status":"0/0"'; then + printf '%s' "$last" + exit 0 + fi + fi + sleep 1 + done + + printf '%s\n' "$last" >&2 + exit 1 + outputs: + - name: result + value: (json_parse($stdout)) + - assert: + resource: + (length(values($result))): 1 + (values($result)[0].status.health.status): 0/0 + (values($result)[0].status.health.healthy || `false`): false + (contains(keys(values($result)[0].status.health), 'component_health_map')): false + (contains(keys(values($result)[0].effective_config), 'collector')): true + (parse_yaml(values(values($result)[0].effective_config)[0])): + receivers: + otlp: + protocols: + grpc: (`{}`) + http: (`{}`) + exporters: + debug: (`{}`) + service: + pipelines: + traces: + receivers: + - otlp + exporters: + - debug + + - catch: + - podLogs: + selector: app=e2e-test-app-bridge-server + - podLogs: + selector: app.kubernetes.io/name=otel-opamp-bridge-standalone + name: send valid remote config + try: + - script: + content: | + #!/bin/bash + set -eu + + remote_config=$(cat < `0`): true + + - catch: + - podLogs: + selector: app=e2e-test-app-bridge-server + - podLogs: + selector: app.kubernetes.io/name=otel-opamp-bridge-standalone + name: verify valid remote config status + try: + - script: + content: | + #!/bin/bash + set -eu + + last="" + for _ in $(seq 1 60); do + if last=$(kubectl get --raw /api/v1/namespaces/$NAMESPACE/services/e2e-test-app-bridge-server:4321/proxy/agents 2>/dev/null); then + if printf '%s' "$last" | grep -q '"remote_config_status":' && + printf '%s' "$last" | grep -q '"status":1' && + printf '%s' "$last" | grep -q 'verbosity: detailed'; then + printf '%s' "$last" + exit 0 + fi + fi + sleep 1 + done + + printf '%s\n' "$last" >&2 + exit 1 + outputs: + - name: result + value: (json_parse($stdout)) + - assert: + resource: + (length(values($result))): 1 + (values($result)[0].status.remote_config_status.status): 1 + (length(values($result)[0].status.remote_config_status.last_remote_config_hash) > `0`): true + (contains(keys(values($result)[0].effective_config), 'collector')): true + (parse_yaml(values(values($result)[0].effective_config)[0])): + receivers: + otlp: + protocols: + http: (`{}`) + exporters: + debug: + verbosity: detailed + service: + pipelines: + traces: + receivers: + - otlp + exporters: + - debug + + - catch: + - podLogs: + selector: app=e2e-test-app-bridge-server + - podLogs: + selector: app.kubernetes.io/name=otel-opamp-bridge-standalone + name: reject invalid remote config + try: + - script: + content: | + #!/bin/bash + set -eu + + remote_config=$(cat </dev/null); then + if printf '%s' "$last" | grep -q '"remote_config_status":' && + printf '%s' "$last" | grep -q '"status":3'; then + printf '%s' "$last" + exit 0 + fi + fi + sleep 1 + done + + printf '%s\n' "$last" >&2 + exit 1 + outputs: + - name: result + value: (json_parse($stdout)) + - assert: + resource: + (length(values($result))): 1 + (values($result)[0].status.remote_config_status.status): 3 + (contains(values($result)[0].status.remote_config_status.error_message, 'invalid collector config')): true + - assert: + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + name: standalone-collector + data: + collector.yaml: | + receivers: + otlp: + protocols: + http: {} + exporters: + debug: + verbosity: detailed + service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] + + - catch: + - podLogs: + selector: app=e2e-test-app-bridge-server + - podLogs: + selector: app.kubernetes.io/name=otel-opamp-bridge-standalone + name: verify multiple standalone agents + try: + - apply: + file: 02-install.yaml + - script: + content: | + #!/bin/bash + set -eu + + kubectl delete pod -n "$NAMESPACE" -l app.kubernetes.io/name=otel-opamp-bridge-standalone + - assert: + file: 02-assert.yaml + - script: + content: | + #!/bin/bash + set -eu + + last="" + for _ in $(seq 1 60); do + if last=$(kubectl get --raw /api/v1/namespaces/$NAMESPACE/services/e2e-test-app-bridge-server:4321/proxy/agents 2>/dev/null); then + if printf '%s' "$last" | grep -q '"effective_config":{"collector":' && + printf '%s' "$last" | grep -q '"effective_config":{"agent":'; then + printf '%s' "$last" + exit 0 + fi + fi + sleep 1 + done + + printf '%s\n' "$last" >&2 + exit 1 + outputs: + - name: result + value: (json_parse($stdout)) + - assert: + resource: + (length(values($result))): 2 + (length(values($result)[?contains(keys(effective_config), 'collector')])): 1 + (length(values($result)[?contains(keys(effective_config), 'agent')])): 1 diff --git a/tests/test-e2e-apps/bridge-server/data/agent.go b/tests/test-e2e-apps/bridge-server/data/agent.go index 40f090fd8d..b5adae76e1 100644 --- a/tests/test-e2e-apps/bridge-server/data/agent.go +++ b/tests/test-e2e-apps/bridge-server/data/agent.go @@ -293,6 +293,13 @@ func (agent *Agent) SetCustomConfig( ) { agent.mux.Lock() + if agent.CustomInstanceConfig == nil { + agent.CustomInstanceConfig = map[string]string{} + } + if agent.EffectiveConfig == nil { + agent.EffectiveConfig = map[string]string{} + } + for key, file := range config.GetConfigMap() { agent.CustomInstanceConfig[key] = string(file.Body) agent.EffectiveConfig[key] = string(file.Body) diff --git a/tests/test-e2e-apps/bridge-server/opampsrv/opampsrv.go b/tests/test-e2e-apps/bridge-server/opampsrv/opampsrv.go index 5577238ea3..31d7f8f30b 100644 --- a/tests/test-e2e-apps/bridge-server/opampsrv/opampsrv.go +++ b/tests/test-e2e-apps/bridge-server/opampsrv/opampsrv.go @@ -6,10 +6,12 @@ package opampsrv import ( "context" "encoding/json" + "errors" "log" "net/http" "os" "regexp" + "time" "github.com/google/uuid" "github.com/oklog/ulid/v2" @@ -28,6 +30,11 @@ type Server struct { httpServer *http.Server } +type remoteConfigRequest struct { + Config map[string]string `json:"config"` + ContentType string `json:"content_type,omitempty"` +} + func NewServer(agents *data.Agents) *Server { logger := &Logger{ log.New( @@ -68,6 +75,7 @@ func (srv *Server) Start() { mux := http.NewServeMux() mux.HandleFunc("/agents", srv.getAgents) + mux.HandleFunc("/agents/push-config-to-agent", srv.pushConfigToAgent) mux.HandleFunc("/agents/", srv.getAgentById) srv.httpServer = &http.Server{ Addr: "0.0.0.0:4321", @@ -169,3 +177,66 @@ func (srv *Server) getAgentById(writer http.ResponseWriter, request *http.Reques } writer.Write(marshaled) } + +func (srv *Server) pushConfigToAgent(writer http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost { + http.Error(writer, "method not allowed", http.StatusMethodNotAllowed) + return + } + + config, err := parseRemoteConfigRequest(request) + if err != nil { + http.Error(writer, err.Error(), http.StatusBadRequest) + return + } + + allAgents := srv.agents.GetAllAgentsReadonlyClone() + if len(allAgents) != 1 { + http.Error(writer, "expected exactly one connected agent", http.StatusConflict) + return + } + + var agentId data.InstanceId + for id := range allAgents { + agentId = id + } + + statusUpdated := make(chan struct{}, 1) + srv.agents.SetCustomConfigForAgent(agentId, config, statusUpdated) + + select { + case <-statusUpdated: + case <-request.Context().Done(): + return + case <-time.After(30 * time.Second): + http.Error(writer, "timed out waiting for agent status update", http.StatusGatewayTimeout) + return + } + + writer.WriteHeader(http.StatusAccepted) +} + +func parseRemoteConfigRequest(request *http.Request) (*protobufs.AgentConfigMap, error) { + var req remoteConfigRequest + if err := json.NewDecoder(request.Body).Decode(&req); err != nil { + return nil, err + } + if len(req.Config) == 0 { + return nil, errors.New("config must contain at least one entry") + } + + configMap := make(map[string]*protobufs.AgentConfigFile, len(req.Config)) + for key, body := range req.Config { + if key == "" { + return nil, errors.New("config keys must be non-empty") + } + configMap[key] = &protobufs.AgentConfigFile{ + Body: []byte(body), + ContentType: req.ContentType, + } + } + + return &protobufs.AgentConfigMap{ + ConfigMap: configMap, + }, nil +}