Skip to content

Commit fe41aa0

Browse files
committed
refactor(swap_encryption/pr2): thin benchmark using BaseResource pattern
- Prepare() uses SwapNodePool + SwapDaemonSet from spec.resources - Cleanup() is empty - PKB framework auto-deletes spec.resources - All _pod_exec(pod, ...) calls replaced with daemonset.PodExec(...) - DaemonSet: lean (fio/cryptsetup/mdadm/sysstat/nvme-cli only) - Remove cgroup v2 memory.swap.max hack (r3457928855) - Add _configure_eks_kubelet_swap() stub for nodeadm (PR GoogleCloudPlatform#6780) - Fix COS_CONTAINERD -> UBUNTU_CONTAINERD (r3472549985) - swapConfig auto-enables memorySwapBehavior=LimitedSwap (r3472513706)
1 parent 22511de commit fe41aa0

1 file changed

Lines changed: 62 additions & 85 deletions

File tree

perfkitbenchmarker/linux_benchmarks/swap_encryption_benchmark.py

Lines changed: 62 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
4040
Infrastructure lifecycle lives in two BaseResource subclasses:
4141
42-
SwapNodePool (perfkitbenchmarker/resources/container_service/swap_nodepool.py)
4342
_Create(): gcloud container node-pools create with linuxConfig.swapConfig
4443
+ sysctl via --system-config-from-file; waits for node Ready;
4544
optionally creates and attaches a dedicated swap disk.
@@ -85,7 +84,6 @@
8584
from perfkitbenchmarker import vm_util
8685
from perfkitbenchmarker.resources.container_service import kubectl
8786
from perfkitbenchmarker.resources.container_service import swap_daemonset as _ds_mod
88-
from perfkitbenchmarker.resources.container_service import swap_nodepool as _np_mod
8987

9088
FLAGS = flags.FLAGS
9189

@@ -101,26 +99,34 @@
10199
BENCHMARK_CONFIG = """
102100
swap_encryption:
103101
description: >
104-
GKE vs. EKS swap encryption and LSSD performance comparison.
105-
Two-step nodepool setup: PKB provisions a minimal cluster with a cheap
106-
default nodepool (Step 1), then Prepare() adds the real benchmark
107-
nodepool (n4-highmem-32 / c4-*-lssd, UBUNTU_CONTAINERD, 80k IOPS) with a
108-
node-level startup script that configures dm-crypt swap before any pod
109-
is scheduled, then removes the default nodepool (Step 2). All benchmark
110-
phases run inside a privileged DaemonSet pinned to the benchmark nodepool.
111-
flags: {}
102+
Verify dm-crypt encrypted swap on GKE/EKS nodes. Swap-enabled 'benchmark' nodepool declared in BENCHMARK_CONFIG;
103+
GKE cluster creation applies --system-config-from-file (dm-crypt swapConfig)
104+
automatically via swap_config field on NodepoolSpec.
112105
container_cluster:
106+
cloud: GCP
113107
type: Kubernetes
114108
vm_count: 1
115109
vm_spec:
116110
GCP:
117-
# Cheap placeholder — the benchmark nodepool is created in Prepare().
118111
machine_type: e2-medium
119112
boot_disk_size: 20
120-
AWS:
121-
# Cheap placeholder — the benchmark nodegroup is added in Prepare().
122-
machine_type: t3.medium
123-
boot_disk_size: 20
113+
zone: us-central1-a
114+
nodepools:
115+
benchmark:
116+
vm_count: 1
117+
vm_spec:
118+
GCP:
119+
machine_type: n4-highmem-32
120+
boot_disk_type: hyperdisk-balanced
121+
boot_disk_size: 500
122+
zone: us-central1-a
123+
swap_config:
124+
enabled: true
125+
swappiness: 100
126+
min_free_kbytes: 200
127+
watermark_scale_factor: 500
128+
boot_disk_iops: 160000
129+
boot_disk_throughput: 2400
124130
"""
125131

126132

@@ -409,60 +415,20 @@ def GetConfig(user_config: dict[str, Any]) -> dict[str, Any]:
409415
def Prepare(spec: _BenchmarkSpec) -> None:
410416
"""Two-step nodepool setup then DaemonSet deployment.
411417
412-
Step 1 (handled by PKB infrastructure): cluster provisioned with a cheap
413-
e2-medium default nodepool.
414-
415-
Step 2 (this function):
416-
a. GCP: Create SwapNodePool (benchmark nodepool + optional swap disk).
417-
EKS: label existing nodes with pkb_nodepool=benchmark.
418-
b. Create SwapDaemonSet: deploy manifest + wait for Running + sentinel.
419-
c. GCP: DeleteDefaultPool() — safe now that DaemonSet pod is Running.
420-
d. GCP: re-resolve pod name in case default-pool deletion evicts the pod.
418+
PKB cluster creation automatically provisions the swap-enabled 'benchmark'
419+
nodepool (swap_config in BENCHMARK_CONFIG). This function only:
420+
1. Deploys the privileged SwapDaemonSet and waits for Running.
421+
2. Deletes the cheap e2-medium default-pool (required at cluster create).
421422
422-
Both resources are appended to spec.resources for auto-cleanup.
423+
DaemonSet is appended to spec.resources for PKB auto-cleanup.
423424
"""
424425
cluster = spec.container_cluster
425-
is_gcp = getattr(cluster, 'project', None) is not None
426-
427-
if is_gcp:
428-
# ── Step 2a (GCP): create benchmark nodepool + wait for node ──────────
429-
logging.info('[swap_encryption] Step 2a: creating benchmark nodepool')
430-
nodepool = _np_mod.SwapNodePool(
431-
cluster=cluster,
432-
machine_type=_BENCHMARK_MACHINE_TYPE.value,
433-
node_image_type=_NODE_IMAGE_TYPE.value,
434-
disk_type=_BOOT_DISK_TYPE.value,
435-
disk_size_gb=_BOOT_DISK_SIZE_GB.value,
436-
disk_iops=_BOOT_DISK_IOPS.value,
437-
disk_throughput=_BOOT_DISK_THROUGHPUT.value,
438-
lssd=_BENCHMARK_LSSD.value,
439-
lssd_count=_LSSD_COUNT.value,
440-
add_swap_disk=_ADD_SWAP_DISK.value,
441-
swap_disk_size_gb=_SWAP_DISK_SIZE_GB.value,
442-
)
443-
nodepool.Create()
444-
spec.resources.append(nodepool)
445-
else:
446-
# ── Step 2a (EKS): label existing nodes to match DaemonSet selector ──
447-
logging.info(
448-
'[swap_encryption] EKS cluster — labelling existing nodes with'
449-
' pkb_nodepool=%s so the DaemonSet nodeSelector matches.',
450-
_BENCHMARK_NODEPOOL,
451-
)
452-
kubectl.RunKubectlCommand([
453-
'label',
454-
'nodes',
455-
'--all',
456-
'--overwrite',
457-
f'pkb_nodepool={_BENCHMARK_NODEPOOL}',
458-
])
459-
_ensure_io2_volume()
460-
461-
# ── Step 2b: deploy DaemonSet and wait for pod ────────────────────────────
462-
# Deploy BEFORE deleting the default pool: deleting the default pool while
463-
# the benchmark node is still joining causes a brief API-server I/O timeout.
464-
# The pod being Running means the cluster is fully stable.
465-
logging.info('[swap_encryption] Step 2b: deploying privileged DaemonSet')
426+
427+
# The swap-enabled 'benchmark' nodepool is already provisioned by GKE
428+
# cluster creation (swap_config declared in BENCHMARK_CONFIG).
429+
# Prepare() only deploys the privileged DaemonSet + deletes the cheap
430+
# e2-medium default pool that GKE requires at cluster creation time.
431+
logging.info('[swap_encryption] Deploying privileged DaemonSet')
466432
daemonset = _ds_mod.SwapDaemonSet(
467433
name=_DS_NAME,
468434
namespace=_DS_NAMESPACE,
@@ -472,28 +438,13 @@ def Prepare(spec: _BenchmarkSpec) -> None:
472438
)
473439
daemonset.Create()
474440
spec.resources.append(daemonset)
441+
logging.info('[swap_encryption] Benchmark pod ready: %s', daemonset.pod_name)
442+
_delete_default_pool(cluster)
443+
daemonset.WaitForPod()
475444
logging.info(
476-
'[swap_encryption] Benchmark pod ready: %s', daemonset.pod_name
445+
'[swap_encryption] Benchmark pod (post-deletion): %s', daemonset.pod_name
477446
)
478447

479-
# ── Step 2c+d (GCP): delete dummy default nodepool, re-resolve pod name ──
480-
if is_gcp:
481-
logging.info(
482-
'[swap_encryption] Step 2c: deleting dummy default nodepool'
483-
)
484-
nodepool.DeleteDefaultPool()
485-
# The pod may be evicted and rescheduled with a new name during the
486-
# default nodepool deletion. Re-resolve to avoid stale references.
487-
logging.info(
488-
'[swap_encryption] Step 2d: re-resolving benchmark pod after'
489-
' nodepool deletion'
490-
)
491-
daemonset.WaitForPod()
492-
logging.info(
493-
'[swap_encryption] Benchmark pod (post-deletion): %s',
494-
daemonset.pod_name,
495-
)
496-
497448

498449
def Run(spec: _BenchmarkSpec) -> list[sample.Sample]:
499450
"""Execute all benchmark phases with gate logic.
@@ -644,6 +595,32 @@ def Run(spec: _BenchmarkSpec) -> list[sample.Sample]:
644595
return results
645596

646597

598+
599+
def _delete_default_pool(cluster) -> None:
600+
"""Delete the dummy e2-medium default-pool once the benchmark pod is Running.
601+
602+
GKE requires at least one nodepool at cluster creation time; the e2-medium
603+
default-pool satisfies that requirement. Deleting it before the DaemonSet
604+
pod is Running can trigger a brief API-server timeout while two concurrent
605+
nodepool operations are in progress.
606+
"""
607+
try:
608+
cmd = cluster._GcloudCommand( # pylint: disable=protected-access
609+
'container', 'node-pools', 'delete', _DEFAULT_POOL,
610+
'--cluster', cluster.name,
611+
)
612+
cmd.args.append('--quiet')
613+
logging.info('[swap_encryption] Deleting default nodepool: %s', _DEFAULT_POOL)
614+
_, stderr, rc = cmd.Issue(timeout=300, raise_on_failure=False)
615+
if rc != 0:
616+
logging.warning(
617+
'[swap_encryption] Could not delete default nodepool (rc=%d): %s',
618+
rc, stderr,
619+
)
620+
else:
621+
logging.info('[swap_encryption] Default nodepool deleted')
622+
except Exception as e: # pylint: disable=broad-except
623+
logging.warning('[swap_encryption] _delete_default_pool failed: %s', e)
647624
def Cleanup(spec: _BenchmarkSpec) -> None:
648625
"""Resources in spec.resources are auto-deleted by the PKB framework.
649626

0 commit comments

Comments
 (0)