admission: collapse from two WorkQueues to one

The CPU Time Token system previously used two work queues, one for
system tenant traffic and one for application tenant traffic. While
this does allow us to set different top-level utilization targets for
the system tenant vs the application tenant, we believe we can get
most of the benefit by mapping all serverless-mode system-tenant
requests into a group that is almost surely going to be prioritized
over any other tenant.

This collapse allows us to delete resourceTier,
cpuTimeTokenChildGranter, and the per-tier arrays that organized the
system into two WorkQueues, two requesters, and four token
buckets. The granter now holds two buckets
(canBurst, noBurst): and implements the granter interface directly.
The coordinator owns a single WorkQueue, and GetKVWorkQueue always
returns it when CTT is enabled.

Settings are consolidated: KVCPUTimeUtilGoal becomes the canonical
target under admission.cpu_time_tokens.target_util, with the old
system_tenant key retired.

Also delete the activeMode/lastMode indirection that was previously
used to track mode transitions across the filler. GetKVWorkQueue
and resetInterval read the cpuTimeTokenACMode cluster setting
directly.

The per-tier CPU time token metrics (suffixed .system_tenant and
.app_tenant) are replaced by single, untiered metrics.

Epic: none
Release note: none

Co-Authored-By: roachdev-claude <roachdev-claude-bot@cockroachlabs.com>

Author: stevendanna

docs/generated/metrics/metrics.yaml

@@ -13589,36 +13589,18 @@ layers:
       aggregation: AVG
       derivative: NON_NEGATIVE_DERIVATIVE
       owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.exhausted_duration_nanos.app_tenant.can_burst
-      exported_name: admission_cpu_time_tokens_exhausted_duration_nanos_app_tenant_can_burst
-      description: Cumulative nanoseconds the app_tenant/can_burst CPU time token bucket has spent exhausted (tokens <= 0); rate() gives the fraction of wall-clock time the bucket was exhausted
+    - name: admission.cpu_time_tokens.exhausted_duration_nanos.can_burst
+      exported_name: admission_cpu_time_tokens_exhausted_duration_nanos_can_burst
+      description: Cumulative nanoseconds the can_burst CPU time token bucket has spent exhausted (tokens <= 0); rate() gives the fraction of wall-clock time the bucket was exhausted
       y_axis_label: Nanoseconds
       type: COUNTER
       unit: NANOSECONDS
       aggregation: AVG
       derivative: NON_NEGATIVE_DERIVATIVE
       owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.exhausted_duration_nanos.app_tenant.no_burst
-      exported_name: admission_cpu_time_tokens_exhausted_duration_nanos_app_tenant_no_burst
-      description: Cumulative nanoseconds the app_tenant/no_burst CPU time token bucket has spent exhausted (tokens <= 0); rate() gives the fraction of wall-clock time the bucket was exhausted
-      y_axis_label: Nanoseconds
-      type: COUNTER
-      unit: NANOSECONDS
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-      owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.exhausted_duration_nanos.system_tenant.can_burst
-      exported_name: admission_cpu_time_tokens_exhausted_duration_nanos_system_tenant_can_burst
-      description: Cumulative nanoseconds the system_tenant/can_burst CPU time token bucket has spent exhausted (tokens <= 0); rate() gives the fraction of wall-clock time the bucket was exhausted
-      y_axis_label: Nanoseconds
-      type: COUNTER
-      unit: NANOSECONDS
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-      owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.exhausted_duration_nanos.system_tenant.no_burst
-      exported_name: admission_cpu_time_tokens_exhausted_duration_nanos_system_tenant_no_burst
-      description: Cumulative nanoseconds the system_tenant/no_burst CPU time token bucket has spent exhausted (tokens <= 0); rate() gives the fraction of wall-clock time the bucket was exhausted
+    - name: admission.cpu_time_tokens.exhausted_duration_nanos.no_burst
+      exported_name: admission_cpu_time_tokens_exhausted_duration_nanos_no_burst
+      description: Cumulative nanoseconds the no_burst CPU time token bucket has spent exhausted (tokens <= 0); rate() gives the fraction of wall-clock time the bucket was exhausted
       y_axis_label: Nanoseconds
       type: COUNTER
       unit: NANOSECONDS
@@ -13634,144 +13616,72 @@ layers:
       aggregation: AVG
       derivative: NONE
       owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.per_tenant.admitted_count.app_tenant
-      exported_name: admission_cpu_time_tokens_per_tenant_admitted_count_app_tenant
-      description: Cumulative number of requests admitted per tenant by CPU time token admission control; use with wait_time_nanos to compute mean wait time via rate(wait_time) / rate(admitted_count)
-      y_axis_label: Requests
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-      owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.per_tenant.admitted_count.system_tenant
-      exported_name: admission_cpu_time_tokens_per_tenant_admitted_count_system_tenant
+    - name: admission.cpu_time_tokens.per_tenant.admitted_count
+      exported_name: admission_cpu_time_tokens_per_tenant_admitted_count
       description: Cumulative number of requests admitted per tenant by CPU time token admission control; use with wait_time_nanos to compute mean wait time via rate(wait_time) / rate(admitted_count)
       y_axis_label: Requests
       type: COUNTER
       unit: COUNT
       aggregation: AVG
       derivative: NON_NEGATIVE_DERIVATIVE
       owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.per_tenant.tokens_returned.app_tenant
-      exported_name: admission_cpu_time_tokens_per_tenant_tokens_returned_app_tenant
-      description: Cumulative CPU time tokens returned per tenant, for example when actual CPU usage was lower than the initial estimate; rate() gives the per-tenant token return rate
-      y_axis_label: Tokens
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-      owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.per_tenant.tokens_returned.system_tenant
-      exported_name: admission_cpu_time_tokens_per_tenant_tokens_returned_system_tenant
+    - name: admission.cpu_time_tokens.per_tenant.tokens_returned
+      exported_name: admission_cpu_time_tokens_per_tenant_tokens_returned
       description: Cumulative CPU time tokens returned per tenant, for example when actual CPU usage was lower than the initial estimate; rate() gives the per-tenant token return rate
       y_axis_label: Tokens
       type: COUNTER
       unit: COUNT
       aggregation: AVG
       derivative: NON_NEGATIVE_DERIVATIVE
       owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.per_tenant.tokens_used.app_tenant
-      exported_name: admission_cpu_time_tokens_per_tenant_tokens_used_app_tenant
-      description: Cumulative CPU time tokens consumed per tenant by admitted work; rate() gives the per-tenant token consumption rate
-      y_axis_label: Tokens
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-      owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.per_tenant.tokens_used.system_tenant
-      exported_name: admission_cpu_time_tokens_per_tenant_tokens_used_system_tenant
+    - name: admission.cpu_time_tokens.per_tenant.tokens_used
+      exported_name: admission_cpu_time_tokens_per_tenant_tokens_used
       description: Cumulative CPU time tokens consumed per tenant by admitted work; rate() gives the per-tenant token consumption rate
       y_axis_label: Tokens
       type: COUNTER
       unit: COUNT
       aggregation: AVG
       derivative: NON_NEGATIVE_DERIVATIVE
       owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.per_tenant.wait_time_nanos.app_tenant
-      exported_name: admission_cpu_time_tokens_per_tenant_wait_time_nanos_app_tenant
-      description: Cumulative nanoseconds of admission queue wait time per tenant in CPU time token admission control; use with admitted_count to compute mean wait time via rate(wait_time) / rate(admitted_count)
-      y_axis_label: Nanoseconds
-      type: COUNTER
-      unit: NANOSECONDS
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-      owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.per_tenant.wait_time_nanos.system_tenant
-      exported_name: admission_cpu_time_tokens_per_tenant_wait_time_nanos_system_tenant
+    - name: admission.cpu_time_tokens.per_tenant.wait_time_nanos
+      exported_name: admission_cpu_time_tokens_per_tenant_wait_time_nanos
       description: Cumulative nanoseconds of admission queue wait time per tenant in CPU time token admission control; use with admitted_count to compute mean wait time via rate(wait_time) / rate(admitted_count)
       y_axis_label: Nanoseconds
       type: COUNTER
       unit: NANOSECONDS
       aggregation: AVG
       derivative: NON_NEGATIVE_DERIVATIVE
       owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.refill.added.app_tenant.can_burst
-      exported_name: admission_cpu_time_tokens_refill_added_app_tenant_can_burst
-      description: Cumulative tokens added to the app_tenant/can_burst CPU time token bucket via the refill process; rate() gives the effective refill rate
-      y_axis_label: Tokens
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-      owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.refill.added.app_tenant.no_burst
-      exported_name: admission_cpu_time_tokens_refill_added_app_tenant_no_burst
-      description: Cumulative tokens added to the app_tenant/no_burst CPU time token bucket via the refill process; rate() gives the effective refill rate
-      y_axis_label: Tokens
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-      owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.refill.added.system_tenant.can_burst
-      exported_name: admission_cpu_time_tokens_refill_added_system_tenant_can_burst
-      description: Cumulative tokens added to the system_tenant/can_burst CPU time token bucket via the refill process; rate() gives the effective refill rate
-      y_axis_label: Tokens
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-      owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.refill.added.system_tenant.no_burst
-      exported_name: admission_cpu_time_tokens_refill_added_system_tenant_no_burst
-      description: Cumulative tokens added to the system_tenant/no_burst CPU time token bucket via the refill process; rate() gives the effective refill rate
-      y_axis_label: Tokens
-      type: COUNTER
-      unit: COUNT
-      aggregation: AVG
-      derivative: NON_NEGATIVE_DERIVATIVE
-      owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.refill.removed.app_tenant.can_burst
-      exported_name: admission_cpu_time_tokens_refill_removed_app_tenant_can_burst
-      description: Cumulative tokens removed from the app_tenant/can_burst CPU time token bucket when refill rates decrease between intervals
+    - name: admission.cpu_time_tokens.refill.added.can_burst
+      exported_name: admission_cpu_time_tokens_refill_added_can_burst
+      description: Cumulative tokens added to the can_burst CPU time token bucket via the refill process; rate() gives the effective refill rate
       y_axis_label: Tokens
       type: COUNTER
       unit: COUNT
       aggregation: AVG
       derivative: NON_NEGATIVE_DERIVATIVE
       owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.refill.removed.app_tenant.no_burst
-      exported_name: admission_cpu_time_tokens_refill_removed_app_tenant_no_burst
-      description: Cumulative tokens removed from the app_tenant/no_burst CPU time token bucket when refill rates decrease between intervals
+    - name: admission.cpu_time_tokens.refill.added.no_burst
+      exported_name: admission_cpu_time_tokens_refill_added_no_burst
+      description: Cumulative tokens added to the no_burst CPU time token bucket via the refill process; rate() gives the effective refill rate
       y_axis_label: Tokens
       type: COUNTER
       unit: COUNT
       aggregation: AVG
       derivative: NON_NEGATIVE_DERIVATIVE
       owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.refill.removed.system_tenant.can_burst
-      exported_name: admission_cpu_time_tokens_refill_removed_system_tenant_can_burst
-      description: Cumulative tokens removed from the system_tenant/can_burst CPU time token bucket when refill rates decrease between intervals
+    - name: admission.cpu_time_tokens.refill.removed.can_burst
+      exported_name: admission_cpu_time_tokens_refill_removed_can_burst
+      description: Cumulative tokens removed from the can_burst CPU time token bucket when refill rates decrease between intervals
       y_axis_label: Tokens
       type: COUNTER
       unit: COUNT
       aggregation: AVG
       derivative: NON_NEGATIVE_DERIVATIVE
       owner: cockroachdb/admission-control
-    - name: admission.cpu_time_tokens.refill.removed.system_tenant.no_burst
-      exported_name: admission_cpu_time_tokens_refill_removed_system_tenant_no_burst
-      description: Cumulative tokens removed from the system_tenant/no_burst CPU time token bucket when refill rates decrease between intervals
+    - name: admission.cpu_time_tokens.refill.removed.no_burst
+      exported_name: admission_cpu_time_tokens_refill_removed_no_burst
+      description: Cumulative tokens removed from the no_burst CPU time token bucket when refill rates decrease between intervals
       y_axis_label: Tokens
       type: COUNTER
       unit: COUNT

pkg/internal/metricscan/metric_owners.yaml

@@ -22,16 +22,16 @@ owners:
   addsstable_copies: cockroachdb/kv
   addsstable_delay_total: cockroachdb/kv
   addsstable_proposals: cockroachdb/kv
-  admission.cpu_time_tokens.exhausted_duration_nanos.%s.%s: cockroachdb/admission-control
-  admission.cpu_time_tokens.per_tenant.admitted_count.%s: cockroachdb/admission-control
-  admission.cpu_time_tokens.per_tenant.tokens_returned.%s: cockroachdb/admission-control
-  admission.cpu_time_tokens.per_tenant.tokens_used.%s: cockroachdb/admission-control
-  admission.cpu_time_tokens.per_tenant.wait_time_nanos.%s: cockroachdb/admission-control
-  admission.cpu_time_tokens.refill.added.%s.%s: cockroachdb/admission-control
-  admission.cpu_time_tokens.refill.removed.%s.%s: cockroachdb/admission-control
+  admission.cpu_time_tokens.exhausted_duration_nanos.%s: cockroachdb/admission-control
+  admission.cpu_time_tokens.refill.added.%s: cockroachdb/admission-control
+  admission.cpu_time_tokens.refill.removed.%s: cockroachdb/admission-control
   admission_admitted_snapshot_bytes: cockroachdb/admission-control
   admission_cpu_time_tokens_dampening_deficit_nanos: cockroachdb/admission-control
   admission_cpu_time_tokens_multiplier: cockroachdb/admission-control
+  admission_cpu_time_tokens_per_tenant_admitted_count: cockroachdb/admission-control
+  admission_cpu_time_tokens_per_tenant_tokens_returned: cockroachdb/admission-control
+  admission_cpu_time_tokens_per_tenant_tokens_used: cockroachdb/admission-control
+  admission_cpu_time_tokens_per_tenant_wait_time_nanos: cockroachdb/admission-control
   admission_cpu_time_tokens_usage_consumed: cockroachdb/admission-control
   admission_cpu_time_tokens_usage_returned: cockroachdb/admission-control
   admission_elastic_cpu_acquired_nanos: cockroachdb/admission-control

pkg/kv/kvserver/kvadmission/kvadmission.go

@@ -353,7 +353,7 @@ func (n *controllerImpl) AdmitKVWork(
 			}
 		}
 	}
-	cpuAdmissionQ := n.cpuGrantCoords.GetKVWorkQueue(admissionInfo.TenantID.IsSystem())
+	cpuAdmissionQ := n.cpuGrantCoords.GetKVWorkQueue()
 	if admissionEnabled {
 		// Bulk jobs such as backups or row-level TTL issue KV requests with a
 		// priority of admissionpb.BulkNormalPri or lower; these are eligible for

pkg/roachprod/agents/opentelemetry/files/cockroachdb_metrics.yaml

@@ -111,27 +111,17 @@ admission_admitted_sql_sql_response_normal_pri: admission.admitted.sql_sql_respo
 admission_admitted_sql_sql_response_user_high_pri: admission.admitted.sql_sql_response.user_high_pri
 admission_admitted_sql_sql_response_user_low_pri: admission.admitted.sql_sql_response.user_low_pri
 admission_cpu_time_tokens_dampening_deficit_nanos: admission.cpu_time_tokens.dampening_deficit_nanos
-admission_cpu_time_tokens_exhausted_duration_nanos_app_tenant_can_burst: admission.cpu_time_tokens.exhausted_duration_nanos.app_tenant.can_burst
-admission_cpu_time_tokens_exhausted_duration_nanos_app_tenant_no_burst: admission.cpu_time_tokens.exhausted_duration_nanos.app_tenant.no_burst
-admission_cpu_time_tokens_exhausted_duration_nanos_system_tenant_can_burst: admission.cpu_time_tokens.exhausted_duration_nanos.system_tenant.can_burst
-admission_cpu_time_tokens_exhausted_duration_nanos_system_tenant_no_burst: admission.cpu_time_tokens.exhausted_duration_nanos.system_tenant.no_burst
+admission_cpu_time_tokens_exhausted_duration_nanos_can_burst: admission.cpu_time_tokens.exhausted_duration_nanos.can_burst
+admission_cpu_time_tokens_exhausted_duration_nanos_no_burst: admission.cpu_time_tokens.exhausted_duration_nanos.no_burst
 admission_cpu_time_tokens_multiplier: admission.cpu_time_tokens.multiplier
-admission_cpu_time_tokens_per_tenant_admitted_count_app_tenant: admission.cpu_time_tokens.per_tenant.admitted_count.app_tenant
-admission_cpu_time_tokens_per_tenant_admitted_count_system_tenant: admission.cpu_time_tokens.per_tenant.admitted_count.system_tenant
-admission_cpu_time_tokens_per_tenant_tokens_returned_app_tenant: admission.cpu_time_tokens.per_tenant.tokens_returned.app_tenant
-admission_cpu_time_tokens_per_tenant_tokens_returned_system_tenant: admission.cpu_time_tokens.per_tenant.tokens_returned.system_tenant
-admission_cpu_time_tokens_per_tenant_tokens_used_app_tenant: admission.cpu_time_tokens.per_tenant.tokens_used.app_tenant
-admission_cpu_time_tokens_per_tenant_tokens_used_system_tenant: admission.cpu_time_tokens.per_tenant.tokens_used.system_tenant
-admission_cpu_time_tokens_per_tenant_wait_time_nanos_app_tenant: admission.cpu_time_tokens.per_tenant.wait_time_nanos.app_tenant
-admission_cpu_time_tokens_per_tenant_wait_time_nanos_system_tenant: admission.cpu_time_tokens.per_tenant.wait_time_nanos.system_tenant
-admission_cpu_time_tokens_refill_added_app_tenant_can_burst: admission.cpu_time_tokens.refill.added.app_tenant.can_burst
-admission_cpu_time_tokens_refill_added_app_tenant_no_burst: admission.cpu_time_tokens.refill.added.app_tenant.no_burst
-admission_cpu_time_tokens_refill_added_system_tenant_can_burst: admission.cpu_time_tokens.refill.added.system_tenant.can_burst
-admission_cpu_time_tokens_refill_added_system_tenant_no_burst: admission.cpu_time_tokens.refill.added.system_tenant.no_burst
-admission_cpu_time_tokens_refill_removed_app_tenant_can_burst: admission.cpu_time_tokens.refill.removed.app_tenant.can_burst
-admission_cpu_time_tokens_refill_removed_app_tenant_no_burst: admission.cpu_time_tokens.refill.removed.app_tenant.no_burst
-admission_cpu_time_tokens_refill_removed_system_tenant_can_burst: admission.cpu_time_tokens.refill.removed.system_tenant.can_burst
-admission_cpu_time_tokens_refill_removed_system_tenant_no_burst: admission.cpu_time_tokens.refill.removed.system_tenant.no_burst
+admission_cpu_time_tokens_per_tenant_admitted_count: admission.cpu_time_tokens.per_tenant.admitted_count
+admission_cpu_time_tokens_per_tenant_tokens_returned: admission.cpu_time_tokens.per_tenant.tokens_returned
+admission_cpu_time_tokens_per_tenant_tokens_used: admission.cpu_time_tokens.per_tenant.tokens_used
+admission_cpu_time_tokens_per_tenant_wait_time_nanos: admission.cpu_time_tokens.per_tenant.wait_time_nanos
+admission_cpu_time_tokens_refill_added_can_burst: admission.cpu_time_tokens.refill.added.can_burst
+admission_cpu_time_tokens_refill_added_no_burst: admission.cpu_time_tokens.refill.added.no_burst
+admission_cpu_time_tokens_refill_removed_can_burst: admission.cpu_time_tokens.refill.removed.can_burst
+admission_cpu_time_tokens_refill_removed_no_burst: admission.cpu_time_tokens.refill.removed.no_burst
 admission_cpu_time_tokens_usage_consumed: admission.cpu_time_tokens.usage.consumed
 admission_cpu_time_tokens_usage_returned: admission.cpu_time_tokens.usage.returned
 admission_elastic_cpu_acquired_nanos: admission.elastic_cpu.acquired_nanos

pkg/server/server.go

@@ -611,8 +611,8 @@ func NewServer(cfg Config, stopper *stop.Stopper) (serverctl.ServerStartupInterf
 	db.AdmissionPacerFactory = gcoords.ElasticCPU
 	sqlCPUProvider := admission.NewSQLCPUProvider(
 		&st.SV,
-		func(tenantID roachpb.TenantID) *admission.WorkQueue {
-			return gcoords.RegularCPU.GetCTTWorkQueue(tenantID.IsSystem())
+		func() *admission.WorkQueue {
+			return gcoords.RegularCPU.GetCTTWorkQueue()
 		},
 	)
 	db.SQLCPUProvider = sqlCPUProvider

pkg/util/admission/BUILD.bazel

@@ -69,7 +69,6 @@ go_library(
         "@com_github_cockroachdb_pebble//:pebble",
         "@com_github_cockroachdb_redact//:redact",
         "@com_github_cockroachdb_tokenbucket//:tokenbucket",
-        "@com_github_olekukonko_tablewriter//:tablewriter",
         "@com_github_petermattis_goid//:goid",
     ],
 )