Skip to content

Commit 7780d15

Browse files
committed
chore: add metadata to resume metrics
also ensures that resumed sessions use the resource requests matching current resource class configuration.
1 parent bd0bf2e commit 7780d15

2 files changed

Lines changed: 44 additions & 25 deletions

File tree

components/renku_data_services/k8s/watcher/core.py

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -196,29 +196,27 @@ async def collect_metrics(
196196
await metrics.session_stopped(user=user, metadata={"session_id": new_obj.meta.name})
197197
return
198198
previous_state = previous_obj.manifest.get("status", {}).get("state", None) if previous_obj else None
199+
200+
resource_class_id = int(new_obj.obj.metadata.annotations.get("renku.io/resource_class_id"))
201+
resource_pool = await rp_repo.get_resource_pool_from_class(user, resource_class_id)
202+
resource_class = await rp_repo.get_resource_class(user, resource_class_id)
203+
metadata = {
204+
"cpu": int(resource_class.cpu * 1000),
205+
"memory": resource_class.memory,
206+
"gpu": resource_class.gpu,
207+
"storage": new_obj.obj.spec.session.storage.size,
208+
"resource_class_id": resource_class_id,
209+
"resource_pool_id": resource_pool.id or "",
210+
"resource_class_name": f"{resource_pool.name}.{resource_class.name}",
211+
"session_id": new_obj.meta.name,
212+
}
199213
match new_obj.obj.raw.get("status", {}).get("state"):
200214
case State.Running.value if previous_state is None or previous_state == State.NotReady.value:
201215
# session starting
202-
resource_class_id = int(new_obj.obj.metadata.annotations.get("renku.io/resource_class_id"))
203-
resource_pool = await rp_repo.get_resource_pool_from_class(k8s_watcher_admin_user, resource_class_id)
204-
resource_class = await rp_repo.get_resource_class(k8s_watcher_admin_user, resource_class_id)
205-
206-
await metrics.session_started(
207-
user=user,
208-
metadata={
209-
"cpu": int(resource_class.cpu * 1000),
210-
"memory": resource_class.memory,
211-
"gpu": resource_class.gpu,
212-
"storage": new_obj.obj.spec.session.storage.size,
213-
"resource_class_id": resource_class_id,
214-
"resource_pool_id": resource_pool.id or "",
215-
"resource_class_name": f"{resource_pool.name}.{resource_class.name}",
216-
"session_id": new_obj.meta.name,
217-
},
218-
)
216+
await metrics.session_started(user=user, metadata=metadata)
219217
case State.Running.value | State.NotReady.value if previous_state == State.Hibernated.value:
220218
# session resumed
221-
await metrics.session_resumed(user, metadata={"session_id": new_obj.meta.name})
219+
await metrics.session_resumed(user=user, metadata=metadata)
222220
case State.Hibernated.value if previous_state != State.Hibernated.value:
223221
# session hibernated
224222
await metrics.session_hibernated(user=user, metadata={"session_id": new_obj.meta.name})

components/renku_data_services/notebooks/core_sessions.py

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1097,10 +1097,9 @@ async def patch_session(
10971097
):
10981098
# Session is being resumed
10991099
patch.spec.hibernated = False
1100-
await metrics.user_requested_session_resume(user, metadata={"session_id": session_id})
11011100

11021101
rp: ResourcePool | None = None
1103-
# Resource class
1102+
# Resource class is being changed
11041103
if body.resource_class_id is not None:
11051104
new_cluster = await nb_config.k8s_v2_client.cluster_by_class_id(body.resource_class_id, user)
11061105
if new_cluster.id != cluster.id:
@@ -1116,6 +1115,21 @@ async def patch_session(
11161115
raise errors.MissingResourceError(
11171116
message=f"The resource class you requested with ID {body.resource_class_id} does not exist"
11181117
)
1118+
# Resource class is not being changed but we still need to get the resource pool and class for patching
1119+
# in case they changed since the session was created
1120+
else:
1121+
rp = await rp_repo.get_resource_pool_from_class(user, session.resource_class_id())
1122+
rc = rp.get_resource_class(session.resource_class_id())
1123+
1124+
# If the session is being hibernated we do not need to patch anything else that is
1125+
# not specifically called for in the request body, we can refresh things when the user resumes.
1126+
if is_getting_hibernated:
1127+
return await nb_config.k8s_v2_client.patch_session(session_id, user.id, patch.to_rfc7386())
1128+
1129+
# If the session is being resumed, we need to patch the resource requests/limits to match the current
1130+
# values of the resource class since they might have changed since the session was created.
1131+
# We also patch the annotations for the resource pool and class to make sure they are up to date.
1132+
else:
11191133
if not patch.metadata:
11201134
patch.metadata = AmaltheaSessionV1Alpha1MetadataPatch()
11211135
# Patch the resource pool and class ID in the annotations
@@ -1137,14 +1151,21 @@ async def patch_session(
11371151
patch.spec.service_account_name = (
11381152
rp.cluster.service_account_name if rp.cluster.service_account_name is not None else RESET
11391153
)
1154+
await metrics.user_requested_session_resume(
1155+
user,
1156+
metadata={
1157+
"cpu": int(rc.cpu * 1000),
1158+
"memory": rc.memory,
1159+
"gpu": rc.gpu,
1160+
"resource_class_id": str(rc.id),
1161+
"resource_pool_id": str(rp.id) or "",
1162+
"resource_class_name": f"{rp.name}.{rc.name}",
1163+
"session_id": session_id,
1164+
}
1165+
)
11401166

11411167
patch.spec.culling = get_culling_patch(user, rp, nb_config, body.lastInteraction)
11421168

1143-
# If the session is being hibernated we do not need to patch anything else that is
1144-
# not specifically called for in the request body, we can refresh things when the user resumes.
1145-
if is_getting_hibernated:
1146-
return await nb_config.k8s_v2_client.patch_session(session_id, user.id, patch.to_rfc7386())
1147-
11481169
server_name = session.metadata.name
11491170
launcher = await session_repo.get_launcher(user, session.launcher_id)
11501171
project = await project_repo.get_project(user=user, project_id=session.project_id)

0 commit comments

Comments
 (0)