diff --git a/Makefile b/Makefile index 3b2d443f..652d4c3f 100644 --- a/Makefile +++ b/Makefile @@ -190,6 +190,7 @@ local-deploy: docker-build-local kind-load-operator manifests ## Build, load, an --set image.pullPolicy=Never \ --set metrics.secure=false \ --set leaderElection.enabled=false \ + --set telemetry.enabled=false \ --set logging.development=true \ --set-string podAnnotations.deploy-timestamp="$(shell date +%s)" $(KUBECTL) rollout status deployment/firebolt-operator -n default --timeout=30s diff --git a/README.md b/README.md index d9b00c9b..76d9dd50 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # Firebolt Kubernetes Operator +[![Downloads](https://scarf.sh/installs-badge/firebolt-db/firebolt-operator?package-type=docker)](https://scarf.sh/) +[![Companies](https://scarf.sh/company-badge/firebolt-db/firebolt-operator?package-type=docker)](https://scarf.sh/) + A Kubernetes operator that manages Firebolt infrastructure: metadata services, an Envoy query-routing proxy, and compute engines with zero-downtime scaling via blue-green deployments. ## Overview @@ -53,6 +56,26 @@ This is the fast path if you want the agent to drive the install for you. If you For a step-by-step walkthrough, follow the quickstart guide in our [official documentation](https://docs.firebolt.io/self-managed/firebolt-operator/quickstart) or using the documentation source file at [`docs/quickstart.mdx`](docs/quickstart.mdx). +## Telemetry + +The operator sends one anonymous, aggregate usage event to [Scarf](https://scarf.sh) at most once per day. This helps Firebolt understand how the community runs the operator and prioritize improvements. + +The event contains the operator and engine versions, Kubernetes minor version, OS and architecture, and bucketed instance, engine, and replica counts. It does not contain names, stable identifiers, query data, schemas, connection parameters, secrets, or configuration. Counts are bucketed rather than exact. As with any network request, the source IP address is visible to Scarf; Scarf may use it to infer the company and does not store it. + +This README also contains a cookie-free Scarf pixel for aggregate page-view statistics. Viewing it in a client that loads remote images can make a request to Scarf independently of the operator's runtime settings; blocking remote images prevents that request. + +Image and chart pulls through `oci.firebolt.io` additionally record the requested version and platform before redirecting to GitHub Container Registry. + +You can opt out in any of these ways: + +- Set `telemetry.enabled=false` in the Helm values. This disables runtime events + and switches the default operator and engine repositories to GHCR; custom + image repositories remain unchanged. +- Install the chart from `oci://ghcr.io/firebolt-db/helm-charts` to bypass Scarf + for the chart download as well. Helm selects the chart repository before it + reads chart values. +- `DO_NOT_TRACK=1` and `SCARF_NO_ANALYTICS=1` disable runtime events only. + ## Firebolt Operator flags The Firebolt Operator supports these runtime flags. The binary default is what @@ -78,6 +101,8 @@ the manager uses when you run it directly. The Helm chart default is what the | `--engine-max-cpu` | `""` | Not set | Maximum allowed CPU request and limit on the engine container (`spec.template.spec.containers[name=engine].resources`). Empty disables the bound. | | `--engine-max-memory` | `""` | Not set | Maximum allowed memory request and limit on the engine container. Empty disables the bound. | | `--engine-max-ephemeral-storage` | `""` | Not set | Maximum allowed ephemeral-storage request and limit on the engine container. Empty disables the bound. | +| `--telemetry` | `true` | `true` | Send a once-daily anonymous aggregate usage event. | +| `--telemetry-endpoint` | `https://telemetry.firebolt.io/firebolt-operator` | Same as binary | Scarf Event Collection endpoint for anonymous aggregate usage events. | | `--zap-devel` | `false` | Not set | Enable controller-runtime development logging defaults. | | `--zap-encoder` | `json` | `json` | Log encoding. Valid values are `json` and `console`. | | `--zap-log-level` | `info` | `info` | Minimum log level. Valid values include `debug`, `info`, `error`, and `panic`. | @@ -95,4 +120,6 @@ make test-e2e # E2E tests (requires Kind cluster) ## Where to go next - For **contributor** detail, conventions, and rules for making changes to this repo, see [`AGENTS.md`](AGENTS.md). - The Helm chart for the operator lives in [helm/firebolt-operator](helm/firebolt-operator/README.md). -- The pure CRD chart for the operator lives in [helm/firebolt-operator-crds](helm/firebolt-operator-crds/README.md). \ No newline at end of file +- The pure CRD chart for the operator lives in [helm/firebolt-operator-crds](helm/firebolt-operator-crds/README.md). + + \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go index a701e25a..4c5ff0d8 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -45,6 +45,7 @@ import ( computev1alpha1 "github.com/firebolt-db/firebolt-kubernetes-operator/api/v1alpha1" "github.com/firebolt-db/firebolt-kubernetes-operator/internal/controller" fireboltmetrics "github.com/firebolt-db/firebolt-kubernetes-operator/internal/metrics" + "github.com/firebolt-db/firebolt-kubernetes-operator/internal/telemetry" // +kubebuilder:scaffold:imports ) @@ -75,6 +76,8 @@ func main() { var watchNamespacesArg string var engineMaxCPUStr, engineMaxMemoryStr, engineMaxEphemeralStorageStr string var gatewayWakeClusterRole string + var telemetryEnabled bool + var telemetryEndpoint string var tlsOpts []func(*tls.Config) flag.BoolVar(&showVersion, "version", false, "Print the version and exit.") flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ @@ -115,6 +118,10 @@ func main() { "a per-instance RoleBinding (so the gateway can stamp the wake annotation). "+ "Empty fails any FireboltInstance reconcile that requires operator-managed gateway RBAC; "+ "users supplying their own gateway ServiceAccount via spec.gateway.template.spec.serviceAccountName are unaffected.") + flag.BoolVar(&telemetryEnabled, "telemetry", true, + "Send a once-daily anonymous aggregate usage event. Set false to disable.") + flag.StringVar(&telemetryEndpoint, "telemetry-endpoint", telemetry.DefaultEndpoint, + "Scarf Event Collection endpoint for anonymous aggregate usage events.") zapOpts := zap.Options{Development: false} zapOpts.BindFlags(flag.CommandLine) flag.Parse() @@ -123,6 +130,9 @@ func main() { _, _ = fmt.Println(version) os.Exit(0) } + if !telemetryEnabled { + controller.UseDirectEngineRepository() + } ctrl.SetLogger(zap.New(zapLoggerOpts(zapOpts)...)) @@ -283,6 +293,17 @@ func main() { os.Exit(1) } } + + if err := mgr.Add(&telemetry.Reporter{ + Client: mgr.GetClient(), + Config: mgr.GetConfig(), + OperatorVersion: version, + Endpoint: telemetryEndpoint, + Enabled: telemetryEnabled, + }); err != nil { + setupLog.Error(err, "unable to register telemetry reporter") + os.Exit(1) + } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/config/images/defaults.latest.env b/config/images/defaults.latest.env index 0bb94dd2..b8bbc72f 100644 --- a/config/images/defaults.latest.env +++ b/config/images/defaults.latest.env @@ -20,7 +20,7 @@ # image inside each kind node's containerd — see # test/e2e/e2e_suite_test.go's SynchronizedBeforeSuite. -ENGINE_IMAGE=ghcr.io/firebolt-db/engine +ENGINE_IMAGE=oci.firebolt.io/firebolt-db/engine ENGINE_TAG=release-5.0.1-0.20260713060957.513515666721 METADATA_IMAGE=ghcr.io/firebolt-db/metadata METADATA_TAG=release-5.0.1-0.20260713060957.513515666721 diff --git a/config/images/images.go b/config/images/images.go index 4635620f..9ab1a889 100644 --- a/config/images/images.go +++ b/config/images/images.go @@ -30,6 +30,9 @@ import ( "strings" ) +// DirectEngineRepository is the engine's upstream GHCR repository. +const DirectEngineRepository = "ghcr.io/firebolt-db/engine" + var defaults = parse(raw) func parse(data string) map[string]string { diff --git a/docs/installation.mdx b/docs/installation.mdx index 84ca4d28..5f338224 100644 --- a/docs/installation.mdx +++ b/docs/installation.mdx @@ -125,7 +125,7 @@ Install the CRD chart before installing the Firebolt Operator chart. This is the allows you full control over the CRDs lifecycle. ```bash -helm upgrade --install firebolt-crds oci://ghcr.io/firebolt-db/helm-charts/firebolt-operator-crds +helm upgrade --install firebolt-crds oci://oci.firebolt.io/firebolt-db/helm-charts/firebolt-operator-crds ``` ### Option 2: Firebolt Operator chart `crds/` directory @@ -145,11 +145,17 @@ Prefer Option 1 for any deployment that will need to upgrade the operator over t Install the Firebolt Operator controller: ```bash -helm upgrade --skip-crds --install firebolt-operator oci://ghcr.io/firebolt-db/helm-charts/firebolt-operator +helm upgrade --skip-crds --install firebolt-operator oci://oci.firebolt.io/firebolt-db/helm-charts/firebolt-operator ``` Omit `--skip-crds` if you chose the bundled `crds/` directory option above. +The `oci.firebolt.io` host is Firebolt's [Scarf](https://scarf.sh) gateway. It records the requested chart version and platform and redirects to GitHub Container Registry. To bypass the gateway, replace `oci://oci.firebolt.io` with `oci://ghcr.io`. + +By default, the operator also sends one anonymous aggregate usage event to Scarf at most once per day. The event contains operator and engine versions, Kubernetes minor version, OS and architecture, and bucketed instance, engine, and replica counts. It contains no names, stable identifiers, query data, schemas, connection parameters, secrets, or configuration. As with any network request, the source IP is visible to Scarf; Scarf may use it to infer the company and does not store it. + +Set `--set telemetry.enabled=false` to disable runtime telemetry and switch the default operator and engine image repositories to GHCR. Images explicitly configured in Helm values, `FireboltEngine`, or `FireboltEngineClass` resources are preserved. To bypass Scarf for the chart download too, install from `oci://ghcr.io/firebolt-db/helm-charts`; Helm selects the chart repository before reading chart values. `DO_NOT_TRACK=1` and `SCARF_NO_ANALYTICS=1` disable runtime events only. + ## Multi-tenant install: scope the Firebolt Operator to specific namespaces By default the Firebolt Operator watches every namespace and the chart @@ -160,7 +166,7 @@ them under `watchNamespaces`: ```bash helm upgrade --skip-crds --install firebolt-operator \ - oci://ghcr.io/firebolt-db/helm-charts/firebolt-operator \ + oci://oci.firebolt.io/firebolt-db/helm-charts/firebolt-operator \ --set 'watchNamespaces={tenant-a,tenant-b}' ``` diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index e48e16fe..49b60e42 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -41,10 +41,12 @@ firebolt-control-plane Ready control-plane 30s v1.35.0 Install the CRDs and operator with Helm: ```bash -helm upgrade --install firebolt-crds oci://ghcr.io/firebolt-db/helm-charts/firebolt-operator-crds -helm upgrade --skip-crds --install firebolt-operator oci://ghcr.io/firebolt-db/helm-charts/firebolt-operator +helm upgrade --install firebolt-crds oci://oci.firebolt.io/firebolt-db/helm-charts/firebolt-operator-crds +helm upgrade --skip-crds --install firebolt-operator oci://oci.firebolt.io/firebolt-db/helm-charts/firebolt-operator ``` +The `oci.firebolt.io` host is Firebolt's [Scarf](https://scarf.sh) gateway. It records anonymous download statistics and redirects to GitHub Container Registry. To bypass the gateway, use `oci://ghcr.io/firebolt-db/helm-charts` instead. + For more installation options, see [Installation](./installation). ## Create a namespace diff --git a/go.mod b/go.mod index a672cca1..78aebd43 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,13 @@ go 1.26.0 require ( github.com/distribution/reference v0.6.0 + github.com/go-logr/logr v1.4.3 github.com/oklog/ulid/v2 v2.1.1 github.com/onsi/ginkgo/v2 v2.32.0 github.com/onsi/gomega v1.42.1 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 + github.com/scarf-sh/scarf-go v0.1.0 github.com/spf13/cobra v1.10.2 go.uber.org/zap v1.28.0 k8s.io/api v0.36.2 @@ -36,7 +38,6 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect diff --git a/go.sum b/go.sum index 9d34cc22..4e82f4ea 100644 --- a/go.sum +++ b/go.sum @@ -130,6 +130,8 @@ github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05Zp github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/scarf-sh/scarf-go v0.1.0 h1:A/o5+Q1T8ExmKNCa4D0c1iw5iB3EpS10jyycG9isWr0= +github.com/scarf-sh/scarf-go v0.1.0/go.mod h1:5Ll7/48YAVqo+GKDb+ptdZhwGat6TqPSbkGaZLVWtLI= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= diff --git a/helm/firebolt-operator/README.md b/helm/firebolt-operator/README.md index b8575110..c1546ec0 100644 --- a/helm/firebolt-operator/README.md +++ b/helm/firebolt-operator/README.md @@ -50,7 +50,7 @@ kubectl delete crd fireboltengines.compute.firebolt.io fireboltinstances.compute | gatewayWakeClusterRole.name | string | `""` | ClusterRole name. Empty defaults to `-gateway-wake` (see the `firebolt-operator.gatewayWakeClusterRoleName` helper). The same value is passed to the operator via `--gateway-wake-cluster-role`, so override here when using an externally managed ClusterRole. | | healthProbeBindAddress | string | `":8081"` | Address the health probe endpoint binds to. | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. | -| image.repository | string | `"ghcr.io/firebolt-db/firebolt-operator"` | Container image repository. | +| image.repository | string | `"oci.firebolt.io/firebolt-db/firebolt-operator"` | Container image repository. | | image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | | imagePullSecrets | list | `[]` | Secrets for pulling images from private registries. | | leaderElection.enabled | bool | `true` | Enable leader election for the controller manager. | @@ -79,6 +79,7 @@ kubectl delete crd fireboltengines.compute.firebolt.io fireboltinstances.compute | serviceAccount.name | string | `""` | The name of the ServiceAccount to use. If empty, a name is generated from the fullname template. | | serviceMonitor.operator | object | `{"bearerTokenSecret":{"key":"token","name":""},"enabled":false,"interval":"15s"}` | Create a ServiceMonitor that scrapes the operator's controller-manager metrics Service (`-metrics`). A ServiceMonitor is used (instead of a PodMonitor) because the secure path (`metrics.secure: true`) needs to authenticate to controller-runtime's built-in authn/authz, and only `ServiceMonitor.endpoint.authorization` exposes a credential reference. | | serviceMonitor.operator.bearerTokenSecret | object | {} | Secret holding the bearer token Prometheus presents to the operator's metrics endpoint. Required when `metrics.secure: true`. Bring your own Secret (e.g. an ESO/Vault-projected one, a `kubernetes.io/service-account-token` Secret bound to your scrape SA, or whatever your Prometheus install already uses); leave `name` empty to render no authorization (the secure endpoint will reject scrapes). | +| telemetry.enabled | bool | `true` | Send a once-daily anonymous aggregate usage event to Scarf. The event contains operator/engine versions, Kubernetes minor, OS/architecture, and bucketed instance/engine/replica counts. It contains no names, stable identifiers, query data, schemas, or configuration. The source IP is visible to Scarf for company inference but is not stored. Set false to disable; when the default Scarf image repositories are unchanged, this also switches operator and engine pulls to GHCR. User-supplied image repositories are preserved. DO_NOT_TRACK and SCARF_NO_ANALYTICS disable runtime events only. | | tolerations | list | `[]` | Tolerations for the operator pod. | | topologySpreadConstraints | list | `[]` | Topology spread constraints for the operator pod. | | watchNamespaces | list | `[]` | Namespaces the operator watches and the chart's manager RBAC is scoped to. Empty (the default) renders a cluster-wide install: one `ClusterRole` + `ClusterRoleBinding` for the manager, and the operator starts with `--namespaces=""` (controller-runtime watches every namespace). A non-empty list renders a `Role` + `RoleBinding` pair per listed namespace with the same rule set, and the operator starts with `--namespaces=` so its cache spans only those namespaces. Switching modes is a `helm upgrade` plus an operator restart. | diff --git a/helm/firebolt-operator/templates/deployment.yaml b/helm/firebolt-operator/templates/deployment.yaml index f6f0ab95..4fe87709 100644 --- a/helm/firebolt-operator/templates/deployment.yaml +++ b/helm/firebolt-operator/templates/deployment.yaml @@ -44,7 +44,11 @@ spec: {{- end }} containers: - name: manager - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- $imageRepository := .Values.image.repository }} + {{- if and (eq .Values.telemetry.enabled false) (eq $imageRepository "oci.firebolt.io/firebolt-db/firebolt-operator") }} + {{- $imageRepository = "ghcr.io/firebolt-db/firebolt-operator" }} + {{- end }} + image: "{{ $imageRepository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} command: - /manager @@ -82,6 +86,9 @@ spec: - --engine-max-ephemeral-storage={{ . }} {{- end }} - --gateway-wake-cluster-role={{ include "firebolt-operator.gatewayWakeClusterRoleName" . }} + {{- if eq .Values.telemetry.enabled false }} + - --telemetry=false + {{- end }} {{- if (.Values.logging.development | default false) }} - --zap-devel=true {{- else }} diff --git a/helm/firebolt-operator/values.schema.json b/helm/firebolt-operator/values.schema.json index 04f753fd..925fa6b1 100644 --- a/helm/firebolt-operator/values.schema.json +++ b/helm/firebolt-operator/values.schema.json @@ -220,6 +220,15 @@ "priorityClassName": { "type": "string" }, + "telemetry": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": ["enabled"] + }, "additionalArgs": { "type": "array", "items": { "type": "string" } diff --git a/helm/firebolt-operator/values.yaml b/helm/firebolt-operator/values.yaml index ef3ffba3..589030b0 100644 --- a/helm/firebolt-operator/values.yaml +++ b/helm/firebolt-operator/values.yaml @@ -6,7 +6,7 @@ replicaCount: 1 image: # -- Container image repository. - repository: ghcr.io/firebolt-db/firebolt-operator + repository: oci.firebolt.io/firebolt-db/firebolt-operator # -- Image pull policy. pullPolicy: IfNotPresent # -- Overrides the image tag whose default is the chart appVersion. @@ -209,6 +209,17 @@ logging: # -- Priority class name for the operator pod. priorityClassName: "" +telemetry: + # -- Send a once-daily anonymous aggregate usage event to Scarf. The event + # contains operator/engine versions, Kubernetes minor, OS/architecture, and + # bucketed instance/engine/replica counts. It contains no names, stable + # identifiers, query data, schemas, or configuration. The source IP is visible + # to Scarf for company inference but is not stored. Set false to disable; + # when the default Scarf image repositories are unchanged, this also switches + # operator and engine pulls to GHCR. User-supplied image repositories are + # preserved. DO_NOT_TRACK and SCARF_NO_ANALYTICS disable runtime events only. + enabled: true + # -- Additional CLI arguments passed to the operator binary. additionalArgs: [] diff --git a/internal/controller/constants.go b/internal/controller/constants.go index c7f4973a..1f61b588 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -228,6 +228,14 @@ var ( DefaultEngineWebImage = images.DefaultEngineWeb() ) +// UseDirectEngineRepository changes only the operator's fallback engine +// repository. Images set on a FireboltEngine or FireboltEngineClass continue to +// take precedence. +func UseDirectEngineRepository() { + DefaultEngineRepository = images.DirectEngineRepository + DefaultEngineImage = resolveImageRef(nil, DefaultEngineRepository, DefaultEngineTag) +} + // resolveImageRef returns "repository:tag" for a component, using fields from // the user-supplied ImageSpec when present and falling back to the supplied // component defaults otherwise. A nil spec, an empty repository, or an empty diff --git a/internal/controller/constants_test.go b/internal/controller/constants_test.go index 0a6645d4..713afef6 100644 --- a/internal/controller/constants_test.go +++ b/internal/controller/constants_test.go @@ -22,8 +22,31 @@ import ( corev1 "k8s.io/api/core/v1" computev1alpha1 "github.com/firebolt-db/firebolt-kubernetes-operator/api/v1alpha1" + "github.com/firebolt-db/firebolt-kubernetes-operator/config/images" ) +func TestUseDirectEngineRepositoryPreservesUserOverride(t *testing.T) { + originalRepository := DefaultEngineRepository + originalImage := DefaultEngineImage + t.Cleanup(func() { + DefaultEngineRepository = originalRepository + DefaultEngineImage = originalImage + }) + + UseDirectEngineRepository() + + if DefaultEngineRepository != images.DirectEngineRepository { + t.Errorf("DefaultEngineRepository = %q, want %q", DefaultEngineRepository, images.DirectEngineRepository) + } + if got, want := DefaultEngineImage, images.DirectEngineRepository+":"+DefaultEngineTag; got != want { + t.Errorf("DefaultEngineImage = %q, want %q", got, want) + } + override := &computev1alpha1.ImageSpec{Repository: "mirror.example.com/engine"} + if got, want := resolveImageRef(override, DefaultEngineRepository, DefaultEngineTag), "mirror.example.com/engine:"+DefaultEngineTag; got != want { + t.Errorf("user repository override = %q, want %q", got, want) + } +} + // TestResolveImageRef pins the partial-override semantics that make // ImageSpec.Repository and ImageSpec.Tag independently optional. Each // dimension must fall back to its component default on its own so users diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go new file mode 100644 index 00000000..7495714f --- /dev/null +++ b/internal/telemetry/telemetry.go @@ -0,0 +1,278 @@ +/* +Copyright 2026 Firebolt Analytics. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package telemetry sends one anonymous, aggregate usage event to Scarf per +// day. The payload contains coarse deployment information and no names, stable +// identifiers, query data, schemas, or configuration. As with any network +// request, the source IP is visible to Scarf; Scarf may use it to infer the +// company and does not store it. +package telemetry + +import ( + "context" + "math/rand/v2" + "runtime" + "sort" + "strings" + "time" + + dockerref "github.com/distribution/reference" + "github.com/go-logr/logr" + "github.com/scarf-sh/scarf-go/scarf" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + + computev1alpha1 "github.com/firebolt-db/firebolt-kubernetes-operator/api/v1alpha1" + "github.com/firebolt-db/firebolt-kubernetes-operator/config/images" +) + +const ( + // DefaultEndpoint is the Firebolt-owned Scarf Event Collection endpoint. + DefaultEndpoint = "https://telemetry.firebolt.io/firebolt-operator" + + reportInterval = 24 * time.Hour + sendTimeout = 3 * time.Second + maxStartupJitter = 5 * time.Minute +) + +type eventLogger interface { + Enabled() bool + LogEvent(map[string]any) error +} + +// Reporter is a leader-only controller-runtime runnable. Running on the +// elected leader ensures an HA operator reports once per cluster rather than +// once per operator replica. +type Reporter struct { + Client client.Client + Config *rest.Config + OperatorVersion string + Endpoint string + Enabled bool + + // Interval overrides the daily interval in tests. + Interval time.Duration + logger eventLogger +} + +var _ manager.LeaderElectionRunnable = (*Reporter)(nil) + +// NeedLeaderElection makes the reporter run only on the elected manager. +func (*Reporter) NeedLeaderElection() bool { return true } + +// Start implements manager.Runnable and blocks until ctx is canceled. +func (r *Reporter) Start(ctx context.Context) error { + log := logf.FromContext(ctx).WithName("telemetry") + + if !r.Enabled { + log.Info("anonymous usage telemetry is disabled; not reporting") + return nil + } + + if r.logger == nil { + endpoint := r.Endpoint + if endpoint == "" { + endpoint = DefaultEndpoint + } + r.logger = scarf.NewScarfEventLogger(endpoint, sendTimeout) + } + if !r.logger.Enabled() { + log.Info("anonymous usage telemetry is disabled by environment; not reporting") + return nil + } + + log.Info("anonymous usage telemetry is enabled; disable with --telemetry=false, DO_NOT_TRACK=1, or SCARF_NO_ANALYTICS=1") + + interval := r.Interval + if interval <= 0 { + interval = reportInterval + } + + // Wait a full reporting interval before the first event. This preserves the + // at-most-once-per-day guarantee across pod restarts and leader failovers + // without storing an identifier or last-send marker in the cluster. Jitter + // spreads otherwise simultaneous reports and does not need cryptographic + // randomness. + initialDelay := interval + time.Duration(rand.Int64N(int64(maxStartupJitter))) //nolint:gosec // non-cryptographic jitter + timer := time.NewTimer(initialDelay) + defer timer.Stop() + select { + case <-ctx.Done(): + return nil + case <-timer.C: + } + r.report(ctx, log) + + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + r.report(ctx, log) + } + } +} + +func (r *Reporter) report(ctx context.Context, log logr.Logger) { + payload, err := r.collect(ctx) + if err != nil { + log.V(1).Info("telemetry collection failed; skipping report", "error", err) + return + } + if err := r.logger.LogEvent(payload); err != nil { + log.V(1).Info("telemetry send failed; ignoring", "error", err) + } +} + +func (r *Reporter) collect(ctx context.Context) (map[string]any, error) { + var instances computev1alpha1.FireboltInstanceList + if err := r.Client.List(ctx, &instances); err != nil { + return nil, err + } + + var engines computev1alpha1.FireboltEngineList + if err := r.Client.List(ctx, &engines); err != nil { + return nil, err + } + + var classList computev1alpha1.FireboltEngineClassList + if err := r.Client.List(ctx, &classList); err != nil { + return nil, err + } + classes := make(map[string]*computev1alpha1.FireboltEngineClass, len(classList.Items)) + for i := range classList.Items { + class := &classList.Items[i] + classes[class.Namespace+"/"+class.Name] = class + } + + payload := map[string]any{ + "event": "operator_daily_summary", + "operator_version": r.OperatorVersion, + "os": runtime.GOOS, + "arch": runtime.GOARCH, + "instance_count_bucket": countBucket(len(instances.Items)), + "engine_count_bucket": countBucket(len(engines.Items)), + "engine_versions": engineVersions(engines.Items, classes), + "replica_size_buckets": replicaSizeBuckets(engines.Items), + } + if version := k8sMinor(r.Config); version != "" { + payload["k8s_minor"] = version + } + return payload, nil +} + +func countBucket(count int) string { + switch { + case count <= 0: + return "0" + case count == 1: + return "1" + case count <= 4: + return "2-4" + case count <= 16: + return "5-16" + default: + return "17+" + } +} + +var bucketOrder = []string{"0", "1", "2-4", "5-16", "17+"} + +func engineVersions(engines []computev1alpha1.FireboltEngine, classes map[string]*computev1alpha1.FireboltEngineClass) string { + versions := make(map[string]struct{}) + for i := range engines { + if tag := engineTag(&engines[i], classes); tag != "" { + versions[tag] = struct{}{} + } + } + + result := make([]string, 0, len(versions)) + for version := range versions { + result = append(result, version) + } + sort.Strings(result) + return strings.Join(result, ",") +} + +// engineTag mirrors the controller's effective engine image precedence: +// operator default, then class template, then engine template. +func engineTag(engine *computev1alpha1.FireboltEngine, classes map[string]*computev1alpha1.FireboltEngineClass) string { + tag := images.EngineTag + if ref := engine.Spec.EngineClassRef; ref != nil && *ref != "" { + if class := classes[engine.Namespace+"/"+*ref]; class != nil { + if container := computev1alpha1.EngineContainerInTemplate(&class.Spec.Template); container != nil && container.Image != "" { + tag = tagOf(container.Image) + } + } + } + if container := computev1alpha1.EngineContainerInTemplate(engine.Spec.Template); container != nil && container.Image != "" { + tag = tagOf(container.Image) + } + return tag +} + +func tagOf(image string) string { + named, err := dockerref.ParseNormalizedNamed(image) + if err != nil { + return "" + } + if tagged, ok := named.(dockerref.Tagged); ok { + return tagged.Tag() + } + return "" +} + +func replicaSizeBuckets(engines []computev1alpha1.FireboltEngine) string { + buckets := make(map[string]struct{}) + for i := range engines { + buckets[countBucket(int(engines[i].Spec.Replicas))] = struct{}{} + } + + result := make([]string, 0, len(buckets)) + for _, bucket := range bucketOrder { + if _, ok := buckets[bucket]; ok { + result = append(result, bucket) + } + } + return strings.Join(result, ",") +} + +func k8sMinor(config *rest.Config) string { + if config == nil { + return "" + } + discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) + if err != nil { + return "" + } + version, err := discoveryClient.ServerVersion() + if err != nil { + return "" + } + major := strings.TrimFunc(version.Major, notDigit) + minor := strings.TrimFunc(version.Minor, notDigit) + if major == "" || minor == "" { + return "" + } + return major + "." + minor +} + +func notDigit(r rune) bool { return r < '0' || r > '9' } diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go new file mode 100644 index 00000000..1b2ce4c5 --- /dev/null +++ b/internal/telemetry/telemetry_test.go @@ -0,0 +1,285 @@ +/* +Copyright 2026 Firebolt Analytics. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package telemetry + +import ( + "context" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + computev1alpha1 "github.com/firebolt-db/firebolt-kubernetes-operator/api/v1alpha1" + "github.com/firebolt-db/firebolt-kubernetes-operator/config/images" +) + +type fakeLogger struct { + enabled bool + events []map[string]any +} + +func (f *fakeLogger) Enabled() bool { return f.enabled } + +func (f *fakeLogger) LogEvent(payload map[string]any) error { + f.events = append(f.events, payload) + return nil +} + +func TestTagOf(t *testing.T) { + cases := map[string]string{ + "ghcr.io/firebolt-db/engine:release-4.32.0": "release-4.32.0", + "ghcr.io/firebolt-db/engine": "", + "engine:dev": "dev", + "localhost:5000/firebolt-db/engine:tag": "tag", + "localhost:5000/firebolt-db/engine": "", + "ghcr.io/firebolt-db/engine@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": "", + "ghcr.io/firebolt-db/engine:1.2.3@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": "1.2.3", + } + for input, want := range cases { + if got := tagOf(input); got != want { + t.Errorf("tagOf(%q) = %q, want %q", input, got, want) + } + } +} + +func TestCountBucket(t *testing.T) { + cases := []struct { + count int + want string + }{ + {-1, "0"}, {0, "0"}, {1, "1"}, {2, "2-4"}, {4, "2-4"}, + {5, "5-16"}, {16, "5-16"}, {17, "17+"}, {1000, "17+"}, + } + for _, test := range cases { + if got := countBucket(test.count); got != test.want { + t.Errorf("countBucket(%d) = %q, want %q", test.count, got, test.want) + } + } +} + +func engineWithImage(name, image string, replicas int32) computev1alpha1.FireboltEngine { + engine := computev1alpha1.FireboltEngine{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "firebolt"}, + Spec: computev1alpha1.FireboltEngineSpec{Replicas: replicas}, + } + if image != "" { + engine.Spec.Template = templateWithImage(image) + } + return engine +} + +func templateWithImage(image string) *corev1.PodTemplateSpec { + return &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: computev1alpha1.EngineContainerName, + Image: image, + }}, + }, + } +} + +func classWithImage(name, image string) *computev1alpha1.FireboltEngineClass { + return &computev1alpha1.FireboltEngineClass{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "firebolt"}, + Spec: computev1alpha1.FireboltEngineClassSpec{ + Template: *templateWithImage(image), + }, + } +} + +func engineWithClassRef(name, ref, image string) computev1alpha1.FireboltEngine { + engine := engineWithImage(name, image, 1) + engine.Spec.EngineClassRef = &ref + return engine +} + +func TestEngineTagPrecedence(t *testing.T) { + classes := map[string]*computev1alpha1.FireboltEngineClass{ + "firebolt/pinned": classWithImage("pinned", "ghcr.io/firebolt-db/engine:release-class"), + } + + tests := []struct { + name string + engine computev1alpha1.FireboltEngine + want string + }{ + { + name: "operator default", + engine: engineWithImage("default", "", 1), + want: images.EngineTag, + }, + { + name: "class image", + engine: engineWithClassRef("class", "pinned", ""), + want: "release-class", + }, + { + name: "engine override wins", + engine: engineWithClassRef("override", "pinned", "ghcr.io/firebolt-db/engine:release-engine"), + want: "release-engine", + }, + { + name: "missing class uses default", + engine: engineWithClassRef("missing", "does-not-exist", ""), + want: images.EngineTag, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := engineTag(&test.engine, classes); got != test.want { + t.Errorf("engineTag() = %q, want %q", got, test.want) + } + }) + } +} + +func TestEngineVersionsDeduplicateAndSort(t *testing.T) { + engines := []computev1alpha1.FireboltEngine{ + engineWithImage("a", "ghcr.io/firebolt-db/engine:release-4.32.0", 1), + engineWithImage("b", "ghcr.io/firebolt-db/engine:release-4.31.0", 3), + engineWithImage("c", "ghcr.io/firebolt-db/engine:release-4.32.0", 5), + } + if got, want := engineVersions(engines, nil), "release-4.31.0,release-4.32.0"; got != want { + t.Errorf("engineVersions() = %q, want %q", got, want) + } +} + +func TestReplicaSizeBuckets(t *testing.T) { + engines := []computev1alpha1.FireboltEngine{ + engineWithImage("a", "engine:1", 1), + engineWithImage("b", "engine:1", 8), + engineWithImage("c", "engine:1", 3), + engineWithImage("d", "engine:1", 12), + } + if got, want := replicaSizeBuckets(engines), "1,2-4,5-16"; got != want { + t.Errorf("replicaSizeBuckets() = %q, want %q", got, want) + } +} + +func newFakeClient(t *testing.T, objects ...client.Object) client.Client { + t.Helper() + scheme := runtime.NewScheme() + if err := computev1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add scheme: %v", err) + } + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() +} + +func TestCollectPayload(t *testing.T) { + instance := &computev1alpha1.FireboltInstance{ + ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: "firebolt"}, + } + class := classWithImage("shared", "ghcr.io/firebolt-db/engine:release-4.40.0") + engine1 := engineWithClassRef("e1", "shared", "") + engine2 := engineWithImage("e2", "ghcr.io/firebolt-db/engine:release-4.40.0", 6) + reporter := &Reporter{ + Client: newFakeClient(t, instance, class, &engine1, &engine2), + OperatorVersion: "v9.9.9", + } + + payload, err := reporter.collect(context.Background()) + if err != nil { + t.Fatalf("collect: %v", err) + } + + expected := map[string]string{ + "event": "operator_daily_summary", + "operator_version": "v9.9.9", + "instance_count_bucket": "1", + "engine_count_bucket": "2-4", + "engine_versions": "release-4.40.0", + "replica_size_buckets": "1,5-16", + } + for key, want := range expected { + if got, _ := payload[key].(string); got != want { + t.Errorf("payload[%q] = %v, want %q", key, payload[key], want) + } + } + + for _, forbidden := range []string{"instance_id", "name", "names", "ip", "namespace"} { + if _, exists := payload[forbidden]; exists { + t.Errorf("payload contains forbidden identifying field %q", forbidden) + } + } +} + +func TestReporterRequiresLeaderElection(t *testing.T) { + if !(&Reporter{}).NeedLeaderElection() { + t.Error("Reporter must require leader election") + } +} + +func TestStartDisabled(t *testing.T) { + tests := []struct { + name string + reporter *Reporter + }{ + { + name: "flag", + reporter: &Reporter{ + Client: newFakeClient(t), + Enabled: false, + logger: &fakeLogger{enabled: true}, + }, + }, + { + name: "environment", + reporter: &Reporter{ + Client: newFakeClient(t), + Enabled: true, + logger: &fakeLogger{enabled: false}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := test.reporter.Start(context.Background()); err != nil { + t.Fatalf("Start() returned error: %v", err) + } + logger := test.reporter.logger.(*fakeLogger) + if len(logger.events) != 0 { + t.Errorf("disabled reporter sent %d events, want 0", len(logger.events)) + } + }) + } +} + +func TestStartWaitsFullIntervalBeforeFirstReport(t *testing.T) { + logger := &fakeLogger{enabled: true} + reporter := &Reporter{ + Client: newFakeClient(t), + Enabled: true, + Interval: time.Hour, + logger: logger, + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + + if err := reporter.Start(ctx); err != nil { + t.Fatalf("Start() returned error: %v", err) + } + if len(logger.events) != 0 { + t.Errorf("reporter sent %d events before the first interval, want 0", len(logger.events)) + } +} diff --git a/scripts/kind-config.yaml b/scripts/kind-config.yaml index 2ef461f8..de6719e8 100644 --- a/scripts/kind-config.yaml +++ b/scripts/kind-config.yaml @@ -6,7 +6,7 @@ apiVersion: kind.x-k8s.io/v1alpha4 # `containerdConfigPatches` switches every node's containerd to look up # per-host pull configuration under /etc/containerd/certs.d//hosts.toml. # scripts/setup-kind-cluster.sh writes those files post-create to alias -# ghcr.io / docker.io to the local OCI registry started by +# ghcr.io, docker.io, and oci.firebolt.io to the local OCI registry started by # scripts/setup-local-registry.sh, so we never duplicate the engine image # (~5 GB) into every kind node's containerd snapshotter. # diff --git a/scripts/load-e2e-images.sh b/scripts/load-e2e-images.sh index a68228a0..d0f9484f 100755 --- a/scripts/load-e2e-images.sh +++ b/scripts/load-e2e-images.sh @@ -105,11 +105,12 @@ declare -a IMAGES=( # mirror expects. The transformation matches Docker's implicit image # normalisation: # ghcr.io/firebolt-db/engine:dev -> firebolt-db/engine:dev (strip explicit host) +# oci.firebolt.io/firebolt-db/engine:latest -> firebolt-db/engine:latest # envoyproxy/envoy:v1.37.2 -> envoyproxy/envoy:v1.37.2 (org/name; keep) # postgres:16-alpine -> library/postgres:16-alpine (official Docker Hub) -# The kind nodes' containerd hosts.toml maps both ghcr.io and docker.io to -# the local registry, so a single push under each upstream's path makes the -# image resolvable without changing the operator-baked image references. +# The kind nodes' containerd hosts.toml maps each embedded image registry to the +# local registry, so a single push under each upstream's path makes the image +# resolvable without changing the operator-baked image references. to_registry_path() { local image="$1" local first_seg="${image%%/*}" diff --git a/scripts/setup-kind-cluster.sh b/scripts/setup-kind-cluster.sh index dbbcddec..4f40b0cd 100755 --- a/scripts/setup-kind-cluster.sh +++ b/scripts/setup-kind-cluster.sh @@ -25,8 +25,9 @@ CLUSTER_NAME="${1:-operator-test-e2e}" CONTROL_PLANE="${CLUSTER_NAME}-control-plane" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Local OCI registry that kind nodes use as a transparent mirror for -# ghcr.io / docker.io. Defaults match scripts/setup-local-registry.sh. +# Local OCI registry that kind nodes use as a transparent mirror for the +# registries referenced by the embedded image defaults. Defaults match +# scripts/setup-local-registry.sh. REGISTRY_NAME="${REGISTRY_NAME:-kind-registry}" REGISTRY_PORT="${REGISTRY_PORT:-5001}" # In-cluster endpoint that kind nodes use to reach the registry. Resolved @@ -37,7 +38,7 @@ REGISTRY_ENDPOINT="http://${REGISTRY_NAME}:5000" # under the path the upstream uses (firebolt-db/engine, library/postgres, # envoyproxy/envoy, ...), so a single hosts.toml per upstream is enough to # make pulls transparent. -MIRRORED_HOSTS=("ghcr.io" "docker.io") +MIRRORED_HOSTS=("ghcr.io" "docker.io" "oci.firebolt.io") # Write /etc/containerd/certs.d//hosts.toml on every kind node, aliasing # each mirrored upstream to the local registry. containerd hot-reloads this @@ -172,7 +173,8 @@ echo "Waiting for node to be Ready..." docker exec "${CONTROL_PLANE}" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ wait --for=condition=Ready node --all --timeout=120s -# Wire containerd to use the local registry as a mirror for ghcr.io / docker.io. +# Wire containerd to use the local registry as a mirror for every embedded +# image registry. # Done after nodes are Ready so we know the node containers are running and # `kind get nodes` returns the full set. configure_mirrors diff --git a/scripts/setup-local-registry.sh b/scripts/setup-local-registry.sh index de4f7cc5..1880f018 100755 --- a/scripts/setup-local-registry.sh +++ b/scripts/setup-local-registry.sh @@ -11,7 +11,8 @@ set -euo pipefail # A local OCI registry sidesteps the duplication: images live ONCE in the # registry, and each kind node pulls only the layers it actually runs. The # kind nodes are wired to use this registry as a transparent mirror for -# ghcr.io / docker.io via /etc/containerd/certs.d hosts.toml files written +# ghcr.io, docker.io, and oci.firebolt.io via /etc/containerd/certs.d +# hosts.toml files written # in scripts/setup-kind-cluster.sh after the cluster comes up. # # Layout: