refactor(BA-5978): swap except ValidationError -> BackendAIModelValidationFailed

Across the codebase, audit every ``except ValidationError`` site and:

* For specific ``Foo.model_validate(...)`` call sites where ``Foo`` is a
  ``BackendAIModel`` subclass, swap to ``except BackendAIModelValidationFailed``.
  The auto-conversion override means raw ``ValidationError`` no longer
  reaches these sites.
* For generalized middleware / decorator entry points (``pydantic_params_api_handler``,
  ``extract_param_value``, ``convert_validation_error``,
  ``PydanticJWTValidator.validate``, the appproxy coordinator's
  ``exception_middleware``), keep BOTH ``BackendAIModelValidationFailed``
  AND ``ValidationError`` branches with a comment explaining the
  ``ValidationError`` arm is a defensive fallback for plain
  ``BaseModel`` subclasses that bypass the auto-conversion override.
* For constructor calls (``Model(**data)``) and ``TypeAdapter.validate_python(...)``,
  leave ``ValidationError`` untouched — the override is opt-in on the
  ``model_validate`` classmethods only.

No public response shapes change; the swap is purely about which type
the right branch catches now that all project Pydantic models inherit
``BackendAIModel``.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: jopemachine

src/ai/backend/account_manager/api/utils.py

@@ -13,6 +13,7 @@
 from pydantic import BaseModel, ConfigDict, ValidationError
 
 from ai.backend.account_manager.exceptions import AuthorizationFailed, InvalidAPIParameters
+from ai.backend.common.exception import BackendAIModelValidationFailed
 from ai.backend.common.types import BackendAIModel
 from ai.backend.logging import BraceStyleAdapter
 
@@ -174,7 +175,12 @@ async def wrapped(
                     kwargs["query"] = query_params
             except (json.decoder.JSONDecodeError, yaml.YAMLError, yaml.MarkedYAMLError) as e:
                 raise InvalidAPIParameters("Malformed body") from e
+            except BackendAIModelValidationFailed as e:
+                raise InvalidAPIParameters("Input validation error", extra_data=e.extra_data) from e
             except ValidationError as e:
+                # Defensive fallback: plain ``BaseModel`` subclasses (no
+                # BackendAIModel inheritance) skip the auto-conversion
+                # override and raise raw ``ValidationError``.
                 raise InvalidAPIParameters("Input validation error", extra_data=e.errors()) from e
             result = await handler(request, checked_params, *args, **kwargs)
             return ensure_stream_response_type(result)

src/ai/backend/agent/server.py

@@ -44,7 +44,6 @@
 from callosum.ordering import ExitOrderedAsyncScheduler
 from callosum.rpc import Peer, RPCMessage
 from etcd_client import WatchEventType
-from pydantic import ValidationError
 from setproctitle import setproctitle
 from zmq.auth.certs import load_certificate
 
@@ -87,7 +86,7 @@
     KernelTerminatedBroadcastEvent,
 )
 from ai.backend.common.events.event_types.kernel.types import KernelLifecycleEventReason
-from ai.backend.common.exception import ConfigurationError
+from ai.backend.common.exception import BackendAIModelValidationFailed, ConfigurationError
 from ai.backend.common.health_checker.checkers.etcd import EtcdHealthChecker
 from ai.backend.common.health_checker.checkers.valkey import ValkeyHealthChecker
 from ai.backend.common.health_checker.probe import HealthProbe, HealthProbeOptions
@@ -1740,7 +1739,7 @@ def main(
         if server_config.debug.enabled:
             print("== Agent configuration ==")
             pprint(server_config.model_dump(by_alias=True))
-    except ValidationError as e:
+    except BackendAIModelValidationFailed as e:
         print(
             "ConfigurationError: Agent local config failed validation checks:",
             file=sys.stderr,

src/ai/backend/appproxy/common/utils.py

@@ -28,6 +28,7 @@
 
 from ai.backend.appproxy.common.types import PydanticResponse
 from ai.backend.common import redis_helper
+from ai.backend.common.exception import BackendAIModelValidationFailed
 from ai.backend.common.types import RedisConnectionInfo
 from ai.backend.logging import BraceStyleAdapter
 
@@ -232,7 +233,11 @@ async def wrapped(
                     kwargs["query"] = query_params
             except (json.decoder.JSONDecodeError, yaml.YAMLError, yaml.MarkedYAMLError) as e:
                 raise InvalidAPIParameters("Malformed body") from e
+            except BackendAIModelValidationFailed as e:
+                raise InvalidAPIParameters("Input validation error", extra_data=e.extra_data) from e
             except ValidationError as e:
+                # Defensive fallback for plain ``BaseModel`` subclasses that
+                # bypass the BackendAIModel auto-conversion override.
                 raise InvalidAPIParameters("Input validation error", extra_data=e.errors()) from e
             result = await handler(request, checked_params, *args, **kwargs)
             return ensure_stream_response_type(result)

src/ai/backend/appproxy/coordinator/server.py

@@ -80,7 +80,7 @@
 from ai.backend.common.defs import REDIS_LIVE_DB, REDIS_STREAM_DB, REDIS_STREAM_LOCK, RedisRole
 from ai.backend.common.etcd import ConfigScopes
 from ai.backend.common.events.dispatcher import EventDispatcher, EventProducer
-from ai.backend.common.exception import BackendAIError
+from ai.backend.common.exception import BackendAIError, BackendAIModelValidationFailed
 from ai.backend.common.health_checker.checkers.valkey import ValkeyHealthChecker
 from ai.backend.common.health_checker.probe import HealthProbe, HealthProbeOptions
 from ai.backend.common.health_checker.types import ComponentId
@@ -197,7 +197,13 @@ async def exception_middleware(
     root_ctx: RootContext = request.app["_root.context"]
     try:
         resp = await handler(request)
+    except BackendAIModelValidationFailed as ex:
+        log.exception("Failed to create response model: {}", ex.extra_msg)
+        raise InternalServerError() from ex
     except ValidationError as ex:
+        # Defensive fallback: a stray plain ``BaseModel`` subclass that
+        # skipped the BackendAIModel auto-conversion would still surface
+        # its raw ``ValidationError`` here.
         log.exception("Failed to create response model: {}", ex.json(indent=2))
         raise InternalServerError() from ex
     except BackendAIError as ex:

src/ai/backend/appproxy/worker/config.py

@@ -10,7 +10,7 @@
 from typing import Annotated, Self
 
 import click
-from pydantic import AnyUrl, Field, FilePath, ValidationError, model_validator
+from pydantic import AnyUrl, Field, FilePath, model_validator
 
 from ai.backend.appproxy.common.config import (
     BaseSchema,
@@ -38,6 +38,7 @@
     PyroscopeConfig,
     ServiceDiscoveryConfig,
 )
+from ai.backend.common.exception import BackendAIModelValidationFailed
 from ai.backend.common.meta import BackendAIConfigMeta, CompositeType, ConfigExample
 from ai.backend.common.typed_validators import AutoDirectoryPath
 from ai.backend.common.types import ServiceDiscoveryType
@@ -1002,12 +1003,12 @@ def load(config_path: Path | None = None, log_level: LogLevel = LogLevel.NOTSET)
                 raise ConfigValidationError("Pyroscope enabled but config is not populated")
             if server_config.profiling.pyroscope_config.application_name is None:
                 server_config.profiling.pyroscope_config.application_name = f"proxy-worker-{server_config.proxy_worker.authority}-{server_config.proxy_worker.api_bind_addr.port}"
-    except (ValidationError, ConfigValidationError) as e:
+    except (BackendAIModelValidationFailed, ConfigValidationError) as e:
         print(
             "ConfigurationError: Could not read or validate the manager local config:",
             file=sys.stderr,
         )
-        if isinstance(e, ValidationError):
+        if isinstance(e, BackendAIModelValidationFailed):
             detail = str(e)
         else:
             detail = pformat(e)

src/ai/backend/client/cli/v2/deployment/chat/types.py

@@ -7,7 +7,7 @@
 from datetime import datetime, timedelta
 from typing import Annotated, Self
 
-from pydantic import Field, ValidationError
+from pydantic import Field
 
 from ai.backend.client.cli.v2.deployment.chat.utils import (
     CHAT_CACHE_FILE,
@@ -16,6 +16,7 @@
     read_json_file,
     write_json_file,
 )
+from ai.backend.common.exception import BackendAIModelValidationFailed
 from ai.backend.common.identifier.deployment import DeploymentID
 from ai.backend.common.types import BackendAIModel
 
@@ -72,7 +73,7 @@ def load(cls) -> Self:
             return cls()
         try:
             return cls.model_validate(raw)
-        except ValidationError:
+        except BackendAIModelValidationFailed:
             print(
                 f"WARNING: {CHAT_CACHE_FILE} is in an invalid format and was ignored.",
                 file=sys.stderr,
@@ -164,7 +165,7 @@ def load(cls) -> Self:
             return cls()
         try:
             return cls.model_validate(raw)
-        except ValidationError:
+        except BackendAIModelValidationFailed:
             print(
                 f"WARNING: {CHAT_CONFIG_FILE} is in an invalid format and was ignored. "
                 "Re-register tokens with `./bai deployment chat-config set`.",
@@ -242,7 +243,7 @@ def load(cls) -> Self:
             return cls()
         try:
             return cls.model_validate(raw)
-        except ValidationError:
+        except BackendAIModelValidationFailed:
             print(
                 f"WARNING: {CHAT_HISTORY_FILE} is in an invalid format and was ignored.",
                 file=sys.stderr,

src/ai/backend/common/api_handlers.py

@@ -26,6 +26,7 @@
 from ai.backend.logging import BraceStyleAdapter
 
 from .exception import (
+    BackendAIModelValidationFailed,
     InvalidAPIHandlerDefinition,
     InvalidAPIParameters,
     MalformedRequestBody,
@@ -83,7 +84,13 @@ def convert_validation_error[T](func: Callable[..., T]) -> Callable[..., T]:
     def wrapped(*args: Any, **kwargs: Any) -> T:
         try:
             return func(*args, **kwargs)
+        except BackendAIModelValidationFailed as e:
+            raise InvalidAPIParameters(repr(e)) from e
         except ValidationError as e:
+            # Defensive fallback: any Pydantic model that does NOT inherit
+            # BackendAIModel would skip the auto-conversion override and
+            # raise raw ``ValidationError``. Keep the handler so a stray
+            # plain ``BaseModel`` subclass still produces a 400.
             raise InvalidAPIParameters(repr(e)) from e
 
     return wrapped
@@ -268,7 +275,11 @@ async def extract_param_value(request: web.Request, input_param_type: Any) -> _P
             f"Parameter '{input_param_type}' must use one of QueryParam, PathParam, HeaderParam, MiddlewareParam, BodyParam"
         )
 
+    except BackendAIModelValidationFailed as e:
+        raise InvalidAPIParameters(str(e)) from e
     except ValidationError as e:
+        # Defensive fallback for plain ``BaseModel`` subclasses that skipped
+        # the BackendAIModel auto-conversion.
         raise InvalidAPIParameters(str(e)) from e
 
 

src/ai/backend/common/clients/valkey_client/valkey_artifact/client.py

@@ -6,7 +6,6 @@
 from typing import Final, Self, cast
 
 from glide import Batch, ExpirySet, ExpiryType
-from pydantic import ValidationError
 
 from ai.backend.common.clients.valkey_client.client import (
     AbstractValkeyClient,
@@ -17,7 +16,7 @@
     DownloadProgressData,
     FileDownloadProgressData,
 )
-from ai.backend.common.exception import BackendAIError
+from ai.backend.common.exception import BackendAIError, BackendAIModelValidationFailed
 from ai.backend.common.metrics.metric import DomainType, LayerType
 from ai.backend.common.resilience import (
     BackoffStrategy,
@@ -254,7 +253,7 @@ async def update_file_progress(
                         previous_data_bytes
                     )
                     previous_current_bytes = previous_data.current_bytes
-                except (ValidationError, UnicodeDecodeError):
+                except (BackendAIModelValidationFailed, UnicodeDecodeError):
                     log.warning("Failed to parse previous file data for progress update")
 
             # Calculate bytes delta for artifact aggregation
@@ -361,7 +360,7 @@ async def get_file_progress(
 
         try:
             return FileDownloadProgressData.model_validate_json(data_bytes)
-        except (ValidationError, UnicodeDecodeError):
+        except (BackendAIModelValidationFailed, UnicodeDecodeError):
             log.warning("Failed to parse file progress data")
             return None
 
@@ -398,7 +397,7 @@ async def get_all_file_progress(
                                     value_bytes
                                 )
                                 file_progress_list.append(file_progress)
-                            except (ValidationError, UnicodeDecodeError):
+                            except (BackendAIModelValidationFailed, UnicodeDecodeError):
                                 log.warning(
                                     "Failed to parse file progress data for key: {}", key_bytes
                                 )

src/ai/backend/common/typed_validators.py

@@ -28,6 +28,7 @@
 from pydantic_core import core_schema
 
 from ai.backend.common.exception import (
+    BackendAIModelValidationFailed,
     JWTDecodeError,
     JWTExpiredError,
     JWTInvalidSignatureError,
@@ -580,5 +581,10 @@ def validate[TModel: BaseModel](self, token: str, model: type[TModel]) -> TModel
 
         try:
             return model.model_validate(payload)
+        except BackendAIModelValidationFailed as e:
+            raise JWTPayloadValidationError(extra_msg=str(e)) from e
         except ValidationError as e:
+            # Defensive fallback: ``model`` is a generic ``BaseModel`` subclass —
+            # plain pydantic models without the BackendAIModel override still
+            # raise raw ``ValidationError``.
             raise JWTPayloadValidationError(extra_msg=str(e)) from e

src/ai/backend/manager/api/utils.py

@@ -39,7 +39,7 @@
 from pydantic import Field, TypeAdapter, ValidationError
 
 from ai.backend.common.api_handlers import BaseRequestModel, BaseResponseModel
-from ai.backend.common.exception import DeprecatedAPI
+from ai.backend.common.exception import BackendAIModelValidationFailed, DeprecatedAPI
 from ai.backend.common.json import load_json
 from ai.backend.logging import BraceStyleAdapter
 from ai.backend.manager.errors.api import (
@@ -223,8 +223,9 @@ async def wrapped(
                     kwargs["query"] = query_params
             except (json.decoder.JSONDecodeError, yaml.YAMLError, yaml.MarkedYAMLError) as e:
                 raise InvalidAPIParameters("Malformed body") from e
-            except ValidationError as ex:
-                first_error = ex.errors()[0]
+            except BackendAIModelValidationFailed as ex:
+                errors = (ex.extra_data or {}).get("errors") or []
+                first_error = errors[0]
                 # Format the first validation error as the message
                 # The client may refer extra_data to access the full validation errors.
                 metadata = {
@@ -237,7 +238,21 @@ async def wrapped(
                     *(f"{k}={v!r}" for k, v in metadata.items()),
                 ]
                 msg = f"{first_error['msg']} [{', '.join(metadata_formatted_items)}]"
-                # To reuse the json serialization provided by pydantic, we call ex.json() and re-parse it.
+                raise InvalidAPIParameters(msg, extra_data=ex.extra_data) from ex
+            except ValidationError as ex:
+                # Defensive fallback for plain ``BaseModel`` subclasses that
+                # bypass the BackendAIModel auto-conversion override.
+                first_error = ex.errors()[0]
+                metadata = {
+                    "input": first_error["input"],
+                }
+                if loc := first_error["loc"]:
+                    metadata["loc"] = loc[0]
+                metadata_formatted_items = [
+                    f"type={first_error['type']}",
+                    *(f"{k}={v!r}" for k, v in metadata.items()),
+                ]
+                msg = f"{first_error['msg']} [{', '.join(metadata_formatted_items)}]"
                 raise InvalidAPIParameters(msg, extra_data=load_json(ex.json())) from ex
             result = await handler(request, checked_params, *args, **kwargs)
             return ensure_stream_response_type(result)