diff --git a/controller/culling.py b/controller/culling.py index d577b4ed..bf383f2f 100644 --- a/controller/culling.py +++ b/controller/culling.py @@ -4,6 +4,7 @@ import requests from requests.exceptions import RequestException +from controller.k8s_resources import get_urls from controller.utils import get_pod_metrics, parse_pod_metrics @@ -43,19 +44,22 @@ def get_js_server_status(js_body): try: server_url = js_body["status"]["create_fn"]["fullServerURL"] except KeyError: - return None + server_url = get_urls(js_body["spec"])[1] + if server_url is None or len(server_url) == 0: + return None token = js_body["spec"]["auth"].get("token") payload = {} if not token else {"token": token} + url = f"{server_url.rstrip('/')}/api/status" try: - res = requests.get(f"{server_url.rstrip('/')}/api/status", params=payload) + res = requests.get(url, params=payload) except RequestException as err: - logging.warning(f"Could not get js server status for {server_url}, because: {err}") + logging.warning(f"Could not get js server status for {url}, because: {err}") return None if res.status_code != 200: - logging.warning(f"Could not get js server status for {server_url}, response status code is {res.status_code}") + logging.warning(f"Could not get js server status for {url}, response status code is {res.status_code}") return None try: @@ -67,7 +71,7 @@ def get_js_server_status(js_body): # } res = res.json() except JSONDecodeError as err: - logging.warning(f"Could not parse js server status for {server_url}, because: {err}") + logging.warning(f"Could not parse js server status {res.text[:10]} for {url}, because: {err}") return None if isinstance(res, dict): diff --git a/controller/main.py b/controller/main.py index d7d4499d..afe8e610 100644 --- a/controller/main.py +++ b/controller/main.py @@ -34,6 +34,12 @@ def register_jupyter_server_handlers( ) -> kopf.OperatorRegistry: logging.info("Populating regsitry") kopf.on.startup(registry=registry)(configure) + kopf.on.delete( + config.api_group, + config.api_version, + config.custom_resource_name, + registry=registry, + )(delete_fn) kopf.on.create( config.api_group, config.api_version, @@ -43,12 +49,6 @@ def register_jupyter_server_handlers( backoff=config.KOPF_CREATE_BACKOFF, registry=registry, )(create_fn) - kopf.on.delete( - config.api_group, - config.api_version, - config.custom_resource_name, - registry=registry, - )(delete_fn) kopf.on.event( version=config.api_version, kind=config.custom_resource_name, diff --git a/controller/server_controller.py b/controller/server_controller.py index 04841d79..32738fe6 100644 --- a/controller/server_controller.py +++ b/controller/server_controller.py @@ -67,8 +67,10 @@ def create_namespaced_resource(namespace, body): Create a k8s resource given the namespace and the full resource object. """ api = get_api(body["apiVersion"], body["kind"]) - return api.create(namespace=namespace, body=body) - + logging.info(f"Got the api for {body['kind']} for {body['metadata']['name']}") + res = api.create(namespace=namespace, body=body) + logging.info(f"Created resource {body['kind']} for {body['metadata']['name']}") + return res def configure(logger, settings, **_): """ @@ -90,8 +92,10 @@ def create_fn(labels, logger, name, namespace, spec, uid, body, **_): Watch the creation of jupyter server objects and create all the necessary k8s child resources which make the actual jupyter server. """ + logging.info(f"Starting create_fn for resource {name}") api = get_api(config.api_version, config.custom_resource_name, config.api_group) now = datetime.now(UTC).isoformat(timespec="seconds") + logging.info(f"create_fn: got api for resource {name}") try: api.patch( namespace=namespace, @@ -111,6 +115,7 @@ def create_fn(labels, logger, name, namespace, spec, uid, body, **_): ) except NotFoundError: pass + logging.info(f"create_fn: attempted patch for {name}") children_specs = get_children_specs(name, spec, logger) @@ -121,6 +126,7 @@ def create_fn(labels, logger, name, namespace, spec, uid, body, **_): children_specs["statefulset"]["spec"]["template"], labels=get_labels(name, uid, labels, is_main_pod=True), ) + logging.info(f"create_fn: got child specs for {name}") # Add the labels to all child resources and create them in the cluster children_uids = {} @@ -132,13 +138,14 @@ def create_fn(labels, logger, name, namespace, spec, uid, body, **_): labels=get_labels(name, uid, labels, child_key=child_key), ) kopf.adopt(child_spec) - + logging.info(f"create_fn: created namespaced resource for {child_key} for {name}") children_uids[child_key] = create_namespaced_resource(namespace=namespace, body=child_spec).metadata.uid - - return {"createdResources": children_uids, "fullServerURL": get_urls(spec)[1]} + + output = {"createdResources": children_uids, "fullServerURL": get_urls(spec)[1]} + logging.info(f"create_fn: completed for {name}") + return output -@kopf.on.delete(config.api_group, config.api_version, config.custom_resource_name) def delete_fn(labels, body, namespace, name, **_): """ The jupyter server has been deleted. diff --git a/controller/server_status.py b/controller/server_status.py index 1de0582d..62eeaa57 100644 --- a/controller/server_status.py +++ b/controller/server_status.py @@ -30,6 +30,7 @@ import requests from controller import config +from controller.k8s_resources import get_urls from controller.server_status_enum import ServerStatusEnum @@ -238,6 +239,9 @@ def from_server_spec( reverse=True, ) hibernated = server.get("spec", {}).get("jupyterServer", {}).get("hibernated", False) + server_url=server.get("status", {}).get("create_fn", {}).get("fullServerURL") + if server_url is None or len(server_url) == 0: + server_url = get_urls(server["spec"])[1] return cls( init_statuses=init_container_statuses, statuses=container_statuses, @@ -245,7 +249,7 @@ def from_server_spec( pod_conditions=pod_conditions, deletion_timestamp=deletion_timestamp, events=server.get("status", {}).get("events", {}), - server_url=server.get("status", {}).get("create_fn", {}).get("fullServerURL"), + server_url=server_url, hibernated=hibernated, )