Skip to content

Commit a4f4b3e

Browse files
kadupoornimakadupoornima
authored
[Telemetry] Add support dynamic min/max node count metrics (#5961)
1 parent 3cb7d63 commit a4f4b3e

4 files changed

Lines changed: 494 additions & 44 deletions

File tree

‎pkg/telemetry/collector.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ func (c *Collector) CollectMetrics(errorCode int, err error) {
6969
c.metadata[ZONE] = getZone(c.blueprint)
7070
c.metadata[MODULES] = getModules(bpModulesList)
7171
c.metadata[STATIC_NODE_COUNTS] = getStaticNodeCounts(c.blueprint)
72+
c.metadata[DYNAMIC_MIN_NODE_COUNTS] = getDynamicNodeCounts(c.blueprint, "min")
73+
c.metadata[DYNAMIC_MAX_NODE_COUNTS] = getDynamicNodeCounts(c.blueprint, "max")
7274
c.metadata[OS_NAME] = getOSName()
7375
c.metadata[OS_VERSION] = getOSVersion()
7476
c.metadata[TERRAFORM_VERSION] = getTerraformVersion()
@@ -302,6 +304,28 @@ func getStaticNodeCounts(bp config.Blueprint) string {
302304

303305
}
304306

307+
func getDynamicNodeCounts(bp config.Blueprint, kind string) string {
308+
counts := make(map[string]int)
309+
targetKeys := dynamicMinNodeCountSettings
310+
if kind == "max" {
311+
targetKeys = dynamicMaxNodeCountSettings
312+
}
313+
314+
for _, m := range config.GetAllBpModules(&bp) {
315+
moduleCounts := getModuleDynamicNodeCounts(m, bp, targetKeys)
316+
for mt, cnt := range moduleCounts {
317+
if cnt > 0 {
318+
counts[mt] += cnt
319+
}
320+
}
321+
}
322+
countsJSON, err := json.Marshal(counts)
323+
if err != nil || len(counts) == 0 {
324+
return ""
325+
}
326+
return strings.ReplaceAll(strings.Trim(string(countsJSON), "{}"), "\"", "")
327+
}
328+
305329
func getOSName() string {
306330
return runtime.GOOS
307331
}

‎pkg/telemetry/collector_test.go‎

Lines changed: 346 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2434,3 +2434,349 @@ func TestGetStaticNodeCounts(t *testing.T) {
24342434
})
24352435
}
24362436
}
2437+
2438+
func TestGetDynamicNodeCounts(t *testing.T) {
2439+
tests := []struct {
2440+
name string
2441+
bp config.Blueprint
2442+
kind string
2443+
want string
2444+
}{
2445+
{
2446+
name: "Extracts global dynamic max nodes natively without applying zonal multiplication on GKE",
2447+
kind: "max",
2448+
bp: config.Blueprint{
2449+
Groups: []config.Group{
2450+
{
2451+
Modules: []config.Module{
2452+
{
2453+
Source: "modules/compute/gke-node-pool",
2454+
Settings: config.NewDict(map[string]cty.Value{
2455+
"machine_type": cty.StringVal("e2-standard-4"),
2456+
"autoscaling_total_max_nodes": cty.NumberIntVal(15),
2457+
"zones": cty.TupleVal([]cty.Value{cty.StringVal("z1"), cty.StringVal("z2")}),
2458+
}),
2459+
},
2460+
},
2461+
},
2462+
},
2463+
},
2464+
want: "e2-standard-4:15",
2465+
},
2466+
{
2467+
name: "Skips dynamic max nodes for GKE if static_node_count is set",
2468+
kind: "max",
2469+
bp: config.Blueprint{
2470+
Groups: []config.Group{
2471+
{
2472+
Modules: []config.Module{
2473+
{
2474+
Source: "modules/compute/gke-node-pool",
2475+
Settings: config.NewDict(map[string]cty.Value{
2476+
"machine_type": cty.StringVal("e2-standard-4"),
2477+
"static_node_count": cty.NumberIntVal(5),
2478+
"autoscaling_total_max_nodes": cty.NumberIntVal(15),
2479+
}),
2480+
},
2481+
},
2482+
},
2483+
},
2484+
},
2485+
want: "",
2486+
},
2487+
{
2488+
name: "Extracts global dynamic min nodes natively without applying zonal multiplication on GKE",
2489+
kind: "min",
2490+
bp: config.Blueprint{
2491+
Groups: []config.Group{
2492+
{
2493+
Modules: []config.Module{
2494+
{
2495+
Source: "modules/compute/gke-node-pool",
2496+
Settings: config.NewDict(map[string]cty.Value{
2497+
"machine_type": cty.StringVal("e2-standard-4"),
2498+
"autoscaling_total_min_nodes": cty.NumberIntVal(2),
2499+
"zones": cty.TupleVal([]cty.Value{cty.StringVal("z1"), cty.StringVal("z2")}),
2500+
}),
2501+
},
2502+
},
2503+
},
2504+
},
2505+
},
2506+
want: "e2-standard-4:2",
2507+
},
2508+
{
2509+
name: "Extracts dynamic max nodes for Slurm partition",
2510+
kind: "max",
2511+
bp: config.Blueprint{
2512+
Groups: []config.Group{
2513+
{
2514+
Modules: []config.Module{
2515+
{
2516+
Source: "community/modules/compute/schedmd-slurm-gcp-v6-partition",
2517+
Settings: config.NewDict(map[string]cty.Value{
2518+
"machine_type": cty.StringVal("c2-standard-30"),
2519+
"node_count_dynamic_max": cty.NumberIntVal(100),
2520+
}),
2521+
},
2522+
},
2523+
},
2524+
},
2525+
},
2526+
want: "c2-standard-30:100",
2527+
},
2528+
{
2529+
name: "Skips dynamic extraction for GKE if static_node_count is explicitly set to 0",
2530+
kind: "max",
2531+
bp: config.Blueprint{
2532+
Groups: []config.Group{
2533+
{
2534+
Modules: []config.Module{
2535+
{
2536+
Source: "modules/compute/gke-node-pool",
2537+
Settings: config.NewDict(map[string]cty.Value{
2538+
"machine_type": cty.StringVal("e2-standard-4"),
2539+
"static_node_count": cty.NumberIntVal(0),
2540+
}),
2541+
},
2542+
},
2543+
},
2544+
},
2545+
},
2546+
want: "",
2547+
},
2548+
{
2549+
name: "Aggregates dynamic min nodes across multiple GKE modules for the same machine type",
2550+
kind: "min",
2551+
bp: config.Blueprint{
2552+
Groups: []config.Group{
2553+
{
2554+
Modules: []config.Module{
2555+
{
2556+
Source: "modules/compute/gke-node-pool",
2557+
Settings: config.NewDict(map[string]cty.Value{
2558+
"machine_type": cty.StringVal("n1-standard-8"),
2559+
"autoscaling_total_min_nodes": cty.NumberIntVal(5),
2560+
}),
2561+
},
2562+
{
2563+
Source: "modules/compute/gke-node-pool",
2564+
Settings: config.NewDict(map[string]cty.Value{
2565+
"machine_type": cty.StringVal("n1-standard-8"),
2566+
"autoscaling_total_min_nodes": cty.NumberIntVal(10),
2567+
}),
2568+
},
2569+
},
2570+
},
2571+
},
2572+
},
2573+
want: "n1-standard-8:15",
2574+
},
2575+
{
2576+
name: "Extracts dynamic max nodes from inline partition configuration in Slurm",
2577+
kind: "max",
2578+
bp: config.Blueprint{
2579+
Groups: []config.Group{
2580+
{
2581+
Modules: []config.Module{
2582+
{
2583+
Source: "community/modules/scheduler/schedmd-slurm-gcp-v6-controller",
2584+
Settings: config.NewDict(map[string]cty.Value{
2585+
"machine_type": cty.StringVal("e2-standard-2"),
2586+
"partition": cty.TupleVal([]cty.Value{
2587+
cty.ObjectVal(map[string]cty.Value{
2588+
"machine_type": cty.StringVal("c2d-standard-112"),
2589+
"node_count_dynamic_max": cty.NumberIntVal(200),
2590+
}),
2591+
cty.ObjectVal(map[string]cty.Value{
2592+
"node_count_dynamic_max": cty.NumberIntVal(50),
2593+
}),
2594+
}),
2595+
}),
2596+
},
2597+
},
2598+
},
2599+
},
2600+
},
2601+
want: "c2d-standard-112:200,e2-standard-2:50",
2602+
},
2603+
{
2604+
name: "Extracts max_size for dynamic bounded modules like HTCondor",
2605+
kind: "max",
2606+
bp: config.Blueprint{
2607+
Groups: []config.Group{
2608+
{
2609+
Modules: []config.Module{
2610+
{
2611+
Source: "modules/compute/htcondor-execute-point",
2612+
Settings: config.NewDict(map[string]cty.Value{
2613+
"machine_type": cty.StringVal("n2-standard-4"),
2614+
"max_size": cty.NumberIntVal(50),
2615+
}),
2616+
},
2617+
},
2618+
},
2619+
},
2620+
},
2621+
want: "n2-standard-4:50",
2622+
},
2623+
{
2624+
name: "Skips dynamic max nodes for VM instance since there are none",
2625+
kind: "max",
2626+
bp: config.Blueprint{
2627+
Groups: []config.Group{
2628+
{
2629+
Modules: []config.Module{
2630+
{
2631+
Source: "modules/compute/vm-instance",
2632+
Settings: config.NewDict(map[string]cty.Value{
2633+
"machine_type": cty.StringVal("n1-standard-1"),
2634+
"instance_count": cty.NumberIntVal(10),
2635+
}),
2636+
},
2637+
},
2638+
},
2639+
},
2640+
},
2641+
want: "",
2642+
},
2643+
{
2644+
name: "Extracts dot notation nested keys like system_node_pool_node_count.total_max_nodes",
2645+
kind: "max",
2646+
bp: config.Blueprint{
2647+
Groups: []config.Group{
2648+
{
2649+
Modules: []config.Module{
2650+
{
2651+
Source: "modules/compute/gke-cluster",
2652+
Settings: config.NewDict(map[string]cty.Value{
2653+
"machine_type": cty.StringVal("c3-standard-4"),
2654+
"system_node_pool_node_count": cty.ObjectVal(map[string]cty.Value{
2655+
"total_min_nodes": cty.NumberIntVal(2),
2656+
"total_max_nodes": cty.NumberIntVal(10),
2657+
}),
2658+
}),
2659+
},
2660+
},
2661+
},
2662+
},
2663+
},
2664+
want: "c3-standard-4:10",
2665+
},
2666+
{
2667+
name: "Zonal multiplier multiplies dynamic zonal bounds by number of elements in zones list",
2668+
kind: "max",
2669+
bp: config.Blueprint{
2670+
Groups: []config.Group{
2671+
{
2672+
Modules: []config.Module{
2673+
{
2674+
Source: "modules/compute/gke-node-pool",
2675+
Settings: config.NewDict(map[string]cty.Value{
2676+
"machine_type": cty.StringVal("n2d-standard-32"),
2677+
"autoscaling_max_node_count": cty.NumberIntVal(5),
2678+
"zones": cty.TupleVal([]cty.Value{cty.StringVal("z1"), cty.StringVal("z2")}),
2679+
}),
2680+
},
2681+
},
2682+
},
2683+
},
2684+
},
2685+
want: "n2d-standard-32:10", // 5 bound max * 2 zones
2686+
},
2687+
{
2688+
name: "Skips zonal multiplier for global default auto-scaling limits",
2689+
kind: "max",
2690+
bp: config.Blueprint{
2691+
Groups: []config.Group{
2692+
{
2693+
Modules: []config.Module{
2694+
{
2695+
Source: "../../modules/compute/gke-node-pool",
2696+
Settings: config.NewDict(map[string]cty.Value{
2697+
"machine_type": cty.StringVal("e2-standard-4"),
2698+
"zones": cty.TupleVal([]cty.Value{cty.StringVal("z1"), cty.StringVal("z2")}),
2699+
}),
2700+
},
2701+
},
2702+
},
2703+
},
2704+
},
2705+
want: "e2-standard-4:1000",
2706+
},
2707+
2708+
{
2709+
name: "Skips tracking missing machine types completely when collecting properties organically",
2710+
kind: "max",
2711+
bp: config.Blueprint{
2712+
Groups: []config.Group{
2713+
{
2714+
Modules: []config.Module{
2715+
{
2716+
Source: "modules/compute/gke-node-pool",
2717+
Settings: config.NewDict(map[string]cty.Value{
2718+
"node_count_dynamic_max": cty.NumberIntVal(5),
2719+
}),
2720+
},
2721+
},
2722+
},
2723+
},
2724+
},
2725+
want: "",
2726+
},
2727+
{
2728+
name: "Blueprint variable evaluation correctly proxies node count limits as safely omitted unknown integers",
2729+
kind: "min",
2730+
bp: config.Blueprint{
2731+
Vars: config.NewDict(map[string]cty.Value{
2732+
"min_cluster_nodes": cty.NumberIntVal(7),
2733+
}),
2734+
Groups: []config.Group{
2735+
{
2736+
Modules: []config.Module{
2737+
{
2738+
Source: "modules/compute/gke-node-pool",
2739+
Settings: config.NewDict(map[string]cty.Value{
2740+
"machine_type": cty.StringVal("a2-highgpu-1g"),
2741+
"autoscaling_total_min_nodes": cty.StringVal("$(vars.min_cluster_nodes)"),
2742+
}),
2743+
},
2744+
},
2745+
},
2746+
},
2747+
},
2748+
want: "",
2749+
},
2750+
{
2751+
name: "Prioritizes first configured bounds correctly when conflicting definitions coexist",
2752+
kind: "max",
2753+
bp: config.Blueprint{
2754+
Groups: []config.Group{
2755+
{
2756+
Modules: []config.Module{
2757+
{
2758+
Source: "modules/compute/gke-node-pool",
2759+
Settings: config.NewDict(map[string]cty.Value{
2760+
"machine_type": cty.StringVal("n2d-standard-8"),
2761+
"autoscaling_total_max_nodes": cty.NumberIntVal(100),
2762+
"autoscaling_max_node_count": cty.NumberIntVal(5),
2763+
"zones": cty.TupleVal([]cty.Value{cty.StringVal("z1"), cty.StringVal("z2")}),
2764+
}),
2765+
},
2766+
},
2767+
},
2768+
},
2769+
},
2770+
want: "n2d-standard-8:10",
2771+
},
2772+
}
2773+
2774+
for _, tc := range tests {
2775+
t.Run(tc.name, func(t *testing.T) {
2776+
got := getDynamicNodeCounts(tc.bp, tc.kind)
2777+
if got != tc.want {
2778+
t.Errorf("getDynamicNodeCounts() = %q; want %q", got, tc.want)
2779+
}
2780+
})
2781+
}
2782+
}

0 commit comments

Comments
 (0)