|
| 1 | +// Copyright 2026 The Cockroach Authors. |
| 2 | +// |
| 3 | +// Use of this software is governed by the CockroachDB Software License |
| 4 | +// included in the /LICENSE file. |
| 5 | + |
| 6 | +// This file implements vCPU consumption tracking for self-hosted clusters. |
| 7 | +// |
| 8 | +// Background: CockroachDB's usage-based licensing model requires tracking |
| 9 | +// vCPU-hours consumed by each node. This package writes hourly audit records |
| 10 | +// to system.vcpu_hours_audit for later export via the `cockroach license audit` CLI. |
| 11 | +// |
| 12 | +// The audit system operates independently of license enforcement - it records |
| 13 | +// consumption data even when no license is installed or when licenses expire. |
| 14 | + |
| 15 | +package license |
| 16 | + |
| 17 | +import ( |
| 18 | + "bytes" |
| 19 | + "context" |
| 20 | + "time" |
| 21 | + |
| 22 | + "github.com/cockroachdb/cockroach/pkg/roachpb" |
| 23 | + "github.com/cockroachdb/cockroach/pkg/server/status" |
| 24 | + "github.com/cockroachdb/cockroach/pkg/util/log" |
| 25 | + "github.com/cockroachdb/cockroach/pkg/util/stop" |
| 26 | + "github.com/cockroachdb/cockroach/pkg/util/timeutil" |
| 27 | + "github.com/cockroachdb/errors" |
| 28 | +) |
| 29 | + |
| 30 | +const ( |
| 31 | + // defaultVCPUAuditInterval is how often we write vCPU consumption records. |
| 32 | + defaultVCPUAuditInterval = 1 * time.Hour |
| 33 | + |
| 34 | + // vcpuAuditShutdownTimeout is the maximum time we wait to write the final |
| 35 | + // audit record during shutdown. |
| 36 | + vcpuAuditShutdownTimeout = 5 * time.Second |
| 37 | + |
| 38 | + // noLicenseID is the sentinel value used when no license is installed. |
| 39 | + // This matches the CLI behavior matrix in the design doc. |
| 40 | + noLicenseID = "no-license" |
| 41 | +) |
| 42 | + |
| 43 | +// vcpuAuditData contains the data needed to write a vCPU audit record. |
| 44 | +type vcpuAuditData struct { |
| 45 | + // licenseID is the unique identifier for the license. |
| 46 | + licenseID []byte |
| 47 | + |
| 48 | + // vcpuCount is the number of vCPUs allocated to this node. |
| 49 | + vcpuCount float64 |
| 50 | +} |
| 51 | + |
| 52 | +// getVCPUAuditData collects the current vCPU count and license ID. |
| 53 | +// This is the data needed to write one audit record. |
| 54 | +func (e *Enforcer) getVCPUAuditData(ctx context.Context) vcpuAuditData { |
| 55 | + data := vcpuAuditData{} |
| 56 | + |
| 57 | + // Get vCPU count from status package (cgroup-aware). |
| 58 | + data.vcpuCount = status.GetVCPUs(ctx) |
| 59 | + // Allow test override for deterministic vCPU counts. |
| 60 | + if tk := e.GetTestingKnobs(); tk != nil && tk.OverrideVCPUCount != nil { |
| 61 | + data.vcpuCount = *tk.OverrideVCPUCount |
| 62 | + } |
| 63 | + |
| 64 | + licIDRaw := e.currentLicenseID.Load() |
| 65 | + if licIDRaw != nil { |
| 66 | + data.licenseID = licIDRaw.([]byte) |
| 67 | + } else { |
| 68 | + data.licenseID = []byte(noLicenseID) |
| 69 | + } |
| 70 | + |
| 71 | + return data |
| 72 | +} |
| 73 | + |
| 74 | +// getVCPUAuditInterval returns the interval for writing vCPU audit records. |
| 75 | +// Can be overridden for testing. |
| 76 | +func (e *Enforcer) getVCPUAuditInterval() time.Duration { |
| 77 | + if tk := e.GetTestingKnobs(); tk != nil && tk.OverrideVCPUAuditInterval != nil { |
| 78 | + return *tk.OverrideVCPUAuditInterval |
| 79 | + } |
| 80 | + return defaultVCPUAuditInterval |
| 81 | +} |
| 82 | + |
| 83 | +// TestingGetCurrentLicenseID returns the currently cached license ID. |
| 84 | +// Used for testing license rotation detection. |
| 85 | +func (e *Enforcer) TestingGetCurrentLicenseID() []byte { |
| 86 | + val := e.currentLicenseID.Load() |
| 87 | + if val == nil { |
| 88 | + return nil |
| 89 | + } |
| 90 | + return val.([]byte) |
| 91 | +} |
| 92 | + |
| 93 | +// VCPUAuditDataForTest exposes vcpuAuditData for testing. |
| 94 | +type VCPUAuditDataForTest struct { |
| 95 | + LicenseID []byte |
| 96 | + VCPUCount float64 |
| 97 | +} |
| 98 | + |
| 99 | +// GetVCPUAuditDataForTest returns the current vCPU audit data for testing. |
| 100 | +func (e *Enforcer) GetVCPUAuditDataForTest(ctx context.Context) VCPUAuditDataForTest { |
| 101 | + data := e.getVCPUAuditData(ctx) |
| 102 | + return VCPUAuditDataForTest{ |
| 103 | + LicenseID: data.licenseID, |
| 104 | + VCPUCount: data.vcpuCount, |
| 105 | + } |
| 106 | +} |
| 107 | + |
| 108 | +// writeVCPUAuditRecord writes a single vCPU consumption record for the given hour. |
| 109 | +// TODO(sadaf-crl): Replace log.Infof with actual SQL INSERT when |
| 110 | +// system.vcpu_hours_audit table is available. |
| 111 | +func (e *Enforcer) writeVCPUAuditRecord(ctx context.Context, hourTimestamp time.Time) { |
| 112 | + nodeID := roachpb.NodeID(e.nodeID.Load()) |
| 113 | + data := e.getVCPUAuditData(ctx) |
| 114 | + |
| 115 | + // Round timestamp to hour boundary. |
| 116 | + hourTimestamp = hourTimestamp.Truncate(time.Hour) |
| 117 | + |
| 118 | + // TODO(sadaf-crl): Replace this log statement with SQL INSERT: |
| 119 | + // INSERT INTO system.vcpu_hours_audit (node_id, license_id, hour_timestamp, num_vcpu) |
| 120 | + // VALUES ($1, $2, $3, $4) |
| 121 | + log.Dev.Infof(ctx, |
| 122 | + "TODO: write vcpu audit record: node_id=%d license_id=%x hour=%s vcpu_count=%.2f", |
| 123 | + nodeID, data.licenseID, hourTimestamp.UTC().Format(time.RFC3339), data.vcpuCount) |
| 124 | + |
| 125 | + if tk := e.GetTestingKnobs(); tk != nil && tk.OnAuditRecordWritten != nil { |
| 126 | + tk.OnAuditRecordWritten() |
| 127 | + } |
| 128 | +} |
| 129 | + |
| 130 | +// updateLicenseIDAndMaybeWrite checks if the license ID changed and triggers |
| 131 | +// an immediate vCPU audit write to prevent gaps in consumption tracking. |
| 132 | +// This is called from RefreshForLicenseChange when the license is updated. |
| 133 | +func (e *Enforcer) updateLicenseIDAndMaybeWrite(ctx context.Context, newLicenseID []byte) { |
| 134 | + var licenseIDCopy []byte |
| 135 | + if newLicenseID != nil { |
| 136 | + licenseIDCopy = append([]byte(nil), newLicenseID...) |
| 137 | + } else { |
| 138 | + licenseIDCopy = []byte(noLicenseID) |
| 139 | + } |
| 140 | + |
| 141 | + // Serialize the read-update-compare cycle so that each license change |
| 142 | + // triggers at most one immediate write. |
| 143 | + var shouldWrite bool |
| 144 | + func() { |
| 145 | + e.auditMu.Lock() |
| 146 | + defer e.auditMu.Unlock() |
| 147 | + |
| 148 | + prevLicenseIDRaw := e.currentLicenseID.Load() |
| 149 | + var prevLicenseID []byte |
| 150 | + if prevLicenseIDRaw != nil { |
| 151 | + prevLicenseID = prevLicenseIDRaw.([]byte) |
| 152 | + } |
| 153 | + |
| 154 | + e.currentLicenseID.Store(licenseIDCopy) |
| 155 | + |
| 156 | + shouldWrite = !bytes.Equal(prevLicenseID, licenseIDCopy) && e.nodeID.Load() != 0 |
| 157 | + }() |
| 158 | + |
| 159 | + if shouldWrite { |
| 160 | + log.Dev.Infof(ctx, "license rotation detected, writing immediate vCPU audit record") |
| 161 | + e.writeVCPUAuditRecord(ctx, timeutil.Now()) |
| 162 | + } |
| 163 | +} |
| 164 | + |
| 165 | +// StartVCPUAuditWriter starts a background goroutine that writes vCPU audit |
| 166 | +// records at regular intervals (default: hourly). The writer will: |
| 167 | +// - Write one record per hour to system.vcpu_hours_audit |
| 168 | +// - Attempt a final write on graceful shutdown (with timeout) |
| 169 | +// |
| 170 | +// This should be called once during enforcer initialization, after Start(). |
| 171 | +// Returns an error if called multiple times or with invalid nodeID. |
| 172 | +func (e *Enforcer) StartVCPUAuditWriter( |
| 173 | + ctx context.Context, stopper *stop.Stopper, nodeID roachpb.NodeID, |
| 174 | +) error { |
| 175 | + if nodeID == 0 { |
| 176 | + return errors.AssertionFailedf("invalid nodeID: %d", nodeID) |
| 177 | + } |
| 178 | + |
| 179 | + // CAS 0 → nodeID atomically marks the writer as started and publishes the |
| 180 | + // node ID for concurrent license rotation handlers. |
| 181 | + if !e.nodeID.CompareAndSwap(0, int32(nodeID)) { |
| 182 | + return errors.New("vCPU audit writer already started") |
| 183 | + } |
| 184 | + |
| 185 | + return stopper.RunAsyncTask(ctx, "vcpu-audit-writer", func(ctx context.Context) { |
| 186 | + // Write an initial record immediately so the first partial hour |
| 187 | + // is not lost (e.g. on node restart during a rolling upgrade). |
| 188 | + e.writeVCPUAuditRecord(ctx, timeutil.Now()) |
| 189 | + ticker := time.NewTicker(e.getVCPUAuditInterval()) |
| 190 | + defer ticker.Stop() |
| 191 | + |
| 192 | + for { |
| 193 | + select { |
| 194 | + case <-ticker.C: |
| 195 | + e.writeVCPUAuditRecord(ctx, timeutil.Now()) |
| 196 | + |
| 197 | + case <-stopper.ShouldQuiesce(): |
| 198 | + // Attempt final write with timeout on shutdown. |
| 199 | + _ = timeutil.RunWithTimeout( |
| 200 | + context.Background(), "vcpu-audit-shutdown", vcpuAuditShutdownTimeout, |
| 201 | + func(ctx context.Context) error { |
| 202 | + e.writeVCPUAuditRecord(ctx, timeutil.Now()) |
| 203 | + return nil |
| 204 | + }) |
| 205 | + log.Dev.Infof(ctx, "vcpu audit writer stopped") |
| 206 | + return |
| 207 | + } |
| 208 | + } |
| 209 | + }) |
| 210 | +} |
0 commit comments