Return error in case of runner crash

In case the runner crashes it would be nice to return an error to the
user. So, add an error handler on the proxy and use it to try to figure
out if the runner crashed and format the response error accordingly.

Signed-off-by: Piotr Stankiewicz <piotr.stankiewicz@docker.com>

Author: Piotr Stankiewicz

pkg/inference/scheduling/api.go

@@ -44,6 +44,16 @@ type OpenAIInferenceRequest struct {
 	Model string `json:"model"`
 }
 
+// OpenAIErrorResponse is used to format an OpenAI API compatible error response
+// (see https://platform.openai.com/docs/api-reference/responses-streaming/error)
+type OpenAIErrorResponse struct {
+	Type           string  `json:"type"` // always "error"
+	Code           *string `json:"code"`
+	Message        string  `json:"message"`
+	Param          *string `json:"param"`
+	SequenceNumber int     `json:"sequence_number"`
+}
+
 // BackendStatus represents information about a running backend
 type BackendStatus struct {
 	// BackendName is the name of the backend

pkg/inference/scheduling/runner.go

@@ -2,6 +2,7 @@ package scheduling
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -143,11 +144,24 @@ func run(
 			w.WriteHeader(http.StatusInternalServerError)
 			select {
 			case <-r.done:
+				res := OpenAIErrorResponse{
+					Type:           "error",
+					Code:           nil,
+					Message:        r.err.Error(),
+					Param:          nil,
+					SequenceNumber: 1,
+				}
+				errJson, err := json.Marshal(&res)
+				if err == nil {
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
+					w.Write(errJson)
+				}
 				return
 			case <-time.After(30 * time.Second):
 			}
 		} else {
 			w.WriteHeader(http.StatusBadGateway)
+
 		}
 	}