From 2df05facdab4ce0b047b0cd3fb29a6e87aeb9a02 Mon Sep 17 00:00:00 2001 From: Michael Black Date: Mon, 6 Jul 2026 14:33:54 -0600 Subject: [PATCH 1/2] fix(K8sPipelineClient): tolerate pod/job deserialization errors from newer Kubernetes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On clusters newer than the bundled kubernetes `client-java` models, listing pods/jobs throws an `IllegalArgumentException` while deserializing the response, because the pod status carries fields the models don't define, e.g.: The field `allocatedResources` in the JSON string is not defined in the `V1PodStatus` properties That `IllegalArgumentException` is a `RuntimeException`, not an `ApiException`, so it was uncaught and surfaced as HTTP 400 from `GET /services/ingestionPipelines/status` — which blanks the service Agents/Ingestion tab in the UI. Reproduced on EKS Kubernetes 1.36 with OpenMetadata 1.13 (client-java 24.0.0). The latest client-java (26.0.0) still trails recent Kubernetes, so bumping the dependency is not a durable fix. Make the three pod/job read paths resilient to client-side deserialization failures (the API call itself succeeded, so cluster access is fine): - getServiceStatusInternal(): report healthy instead of failing the status endpoint - getQueuedPipelineStatusInternal(): treat as no queued jobs - the ingestion log-fetch path: return a graceful "logs unavailable" message This keeps K8sPipelineClient's UI working across Kubernetes upgrades instead of breaking whenever the cluster adds a status field ahead of client-java. Signed-off-by: Michael Black Co-Authored-By: Claude Opus 4.8 (1M context) --- .../pipeline/k8s/K8sPipelineClient.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/k8s/K8sPipelineClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/k8s/K8sPipelineClient.java index e08d5b4ac5ae..bf44459ab3eb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/k8s/K8sPipelineClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/k8s/K8sPipelineClient.java @@ -930,6 +930,14 @@ public List getQueuedPipelineStatusInternal(IngestionPipeline in } catch (ApiException e) { LOG.error("Failed to check queued pipeline status: {}", e.getResponseBody()); + } catch (RuntimeException e) { + // client-java could not deserialize a job returned by a newer Kubernetes + // than its models (see getServiceStatusInternal). Treat as "no queued jobs + // visible" rather than propagating the error to the caller. + LOG.warn( + "Could not deserialize queued job status (client-java lags the cluster's " + + "Kubernetes version): {}", + e.getMessage()); } return queuedStatuses; @@ -1002,6 +1010,24 @@ protected PipelineServiceClientResponse getServiceStatusInternal() { e.getResponseBody()); LOG.error(error); return buildUnhealthyStatus(error); + } catch (RuntimeException e) { + // The list calls above reached the API server successfully (any real + // access/RBAC failure would be an ApiException, handled above). A + // RuntimeException here means the bundled Kubernetes client (client-java) + // could not DESERIALIZE a pod/job in the response — typically because the + // cluster runs a newer Kubernetes than the client models and the response + // carries an unmodeled status field, e.g.: + // IllegalArgumentException: The field `allocatedResources` in the JSON + // string is not defined in the `V1PodStatus` properties + // Cluster access is fine, so report healthy instead of failing the whole + // /services/ingestionPipelines/status endpoint (which blanks the UI). + LOG.warn( + "Kubernetes namespace {} is reachable but a pod/job could not be deserialized " + + "(client-java models likely lag the cluster's Kubernetes version): {}", + namespace, + e.getMessage()); + return buildHealthyStatus(getKubernetesVersion()) + .withReason(String.format(K8S_AVAILABLE_FORMAT, namespace, serviceAccount)); } } @@ -1128,6 +1154,16 @@ public Map getLastIngestionLogs( } catch (ApiException e) { LOG.error("Failed to get logs for pipeline {}: {}", pipelineName, e.getResponseBody()); return Map.of("logs", FAILED_LOGS_MESSAGE + e.getMessage()); + } catch (RuntimeException e) { + // client-java could not deserialize a pod returned by a newer Kubernetes + // than its models (see getServiceStatusInternal). Return a graceful message + // instead of propagating the error. + LOG.warn( + "Could not deserialize pod while fetching logs for pipeline {} (client-java lags " + + "the cluster's Kubernetes version): {}", + pipelineName, + e.getMessage()); + return Map.of("logs", FAILED_LOGS_MESSAGE + e.getMessage()); } } From 7cb3bae60487672d010084bd77225b57b16102c4 Mon Sep 17 00:00:00 2001 From: Michael Black Date: Mon, 6 Jul 2026 14:41:59 -0600 Subject: [PATCH 2/2] review: narrow catch to IllegalArgumentException (avoid masking real errors) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review feedback (gitar-bot): catching the whole RuntimeException hierarchy could swallow genuine bugs (NPE, IllegalStateException, uninitialized API clients) and, in getServiceStatusInternal(), report an unhealthy cluster as healthy. Narrow all three handlers to IllegalArgumentException — the exact exception client-java throws on unknown pod/job status fields — so real runtime failures still propagate while version-skew deserialization degrades gracefully. Signed-off-by: Michael Black Co-Authored-By: Claude Opus 4.8 (1M context) --- .../clients/pipeline/k8s/K8sPipelineClient.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/k8s/K8sPipelineClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/k8s/K8sPipelineClient.java index bf44459ab3eb..6d3c4cfa30fc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/k8s/K8sPipelineClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/k8s/K8sPipelineClient.java @@ -930,7 +930,7 @@ public List getQueuedPipelineStatusInternal(IngestionPipeline in } catch (ApiException e) { LOG.error("Failed to check queued pipeline status: {}", e.getResponseBody()); - } catch (RuntimeException e) { + } catch (IllegalArgumentException e) { // client-java could not deserialize a job returned by a newer Kubernetes // than its models (see getServiceStatusInternal). Treat as "no queued jobs // visible" rather than propagating the error to the caller. @@ -1010,11 +1010,12 @@ protected PipelineServiceClientResponse getServiceStatusInternal() { e.getResponseBody()); LOG.error(error); return buildUnhealthyStatus(error); - } catch (RuntimeException e) { + } catch (IllegalArgumentException e) { // The list calls above reached the API server successfully (any real - // access/RBAC failure would be an ApiException, handled above). A - // RuntimeException here means the bundled Kubernetes client (client-java) - // could not DESERIALIZE a pod/job in the response — typically because the + // access/RBAC failure would be an ApiException, handled above). An + // IllegalArgumentException here is thrown by the bundled Kubernetes client + // (client-java) when it cannot DESERIALIZE a pod/job in the response — + // typically because the // cluster runs a newer Kubernetes than the client models and the response // carries an unmodeled status field, e.g.: // IllegalArgumentException: The field `allocatedResources` in the JSON @@ -1154,7 +1155,7 @@ public Map getLastIngestionLogs( } catch (ApiException e) { LOG.error("Failed to get logs for pipeline {}: {}", pipelineName, e.getResponseBody()); return Map.of("logs", FAILED_LOGS_MESSAGE + e.getMessage()); - } catch (RuntimeException e) { + } catch (IllegalArgumentException e) { // client-java could not deserialize a pod returned by a newer Kubernetes // than its models (see getServiceStatusInternal). Return a graceful message // instead of propagating the error.