initial feedback feature (#427)

* wip

Signed-off-by: Peter Jausovec <peter.jausovec@solo.io>

* wip

Signed-off-by: Peter Jausovec <peter.jausovec@solo.io>

* first stage of feedback feature (ability to provide feedback and store it)

Signed-off-by: Peter Jausovec <peter.jausovec@solo.io>

* lint

Signed-off-by: Peter Jausovec <peter.jausovec@solo.io>

* pr feedback

Signed-off-by: Peter Jausovec <peter.jausovec@solo.io>

---------

Signed-off-by: Peter Jausovec <peter.jausovec@solo.io>
Co-authored-by: Eitan Yarmush <eitan.yarmush@solo.io>

Author: peterj

go/autogen/client/feedback.go

@@ -0,0 +1,22 @@
+package client
+
+import "fmt"
+
+func (c *Client) CreateFeedback(feedback *FeedbackSubmission) error {
+	err := c.doRequest("POST", "/feedback/", feedback, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Client) ListFeedback(userID string) ([]*FeedbackSubmission, error) {
+	var response []*FeedbackSubmission
+	err := c.doRequest("GET", fmt.Sprintf("/feedback/?user_id=%s", userID), nil, &response)
+	if err != nil {
+		return nil, err
+	}
+
+	return response, nil
+}

go/autogen/client/types.go

@@ -187,3 +187,37 @@ func streamSseResponse(r io.ReadCloser) chan *SseEvent {
 	}()
 	return ch
 }
+
+// FeedbackIssueType represents the category of feedback issue
+type FeedbackIssueType string
+
+const (
+	FeedbackIssueTypeInstructions FeedbackIssueType = "instructions" // Did not follow instructions
+	FeedbackIssueTypeFactual      FeedbackIssueType = "factual"      // Not factually correct
+	FeedbackIssueTypeIncomplete   FeedbackIssueType = "incomplete"   // Incomplete response
+	FeedbackIssueTypeTool         FeedbackIssueType = "tool"         // Should have run the tool
+)
+
+// FeedbackSubmission defines the request payload for submitting feedback
+// and also serves as the response object when listing feedback.
+type FeedbackSubmission struct {
+	// ID is the unique identifier for the feedback, present in responses.
+	ID int `json:"id,omitempty"`
+	// CreatedAt is the timestamp of feedback creation, present in responses.
+	CreatedAt string `json:"created_at,omitempty"`
+	// UpdatedAt is the timestamp of the last update, present in responses.
+	UpdatedAt string `json:"updated_at,omitempty"`
+	// Version of the feedback object, present in responses.
+	Version string `json:"version,omitempty"`
+
+	// UserID is the identifier for the user submitting the feedback.
+	UserID string `json:"user_id,omitempty"`
+	// IsPositive indicates if the feedback is positive. Required for request.
+	IsPositive bool `json:"is_positive"`
+	// FeedbackText is the textual content of the feedback. Required for request.
+	FeedbackText string `json:"feedback_text"`
+	// IssueType categorizes the feedback if it's negative. Optional.
+	IssueType *FeedbackIssueType `json:"issue_type,omitempty"`
+	// MessageID is the ID of the message this feedback pertains to.
+	MessageID int `json:"message_id,omitempty"`
+}

go/controller/internal/httpserver/handlers/feedback.go

@@ -0,0 +1,84 @@
+package handlers
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+
+	"github.com/kagent-dev/kagent/go/autogen/client"
+	"github.com/kagent-dev/kagent/go/controller/internal/httpserver/errors"
+	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+// FeedbackHandler handles user feedback submissions
+type FeedbackHandler struct {
+	*Base
+}
+
+// NewFeedbackHandler creates a new feedback handler
+func NewFeedbackHandler(base *Base) *FeedbackHandler {
+	return &FeedbackHandler{Base: base}
+}
+
+// HandleCreateFeedback handles the submission of user feedback and forwards it to the Python backend
+func (h *FeedbackHandler) HandleCreateFeedback(w ErrorResponseWriter, r *http.Request) {
+	log := ctrllog.FromContext(r.Context()).WithName("feedback-handler").WithValues("operation", "create-feedback")
+
+	log.Info("Received feedback submission")
+
+	// Read request body
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		log.Error(err, "Failed to read request body")
+		w.RespondWithError(errors.NewBadRequestError("Failed to read request body", err))
+		return
+	}
+
+	// Parse the feedback submission request
+	var feedbackReq client.FeedbackSubmission
+	if err := json.Unmarshal(body, &feedbackReq); err != nil {
+		log.Error(err, "Failed to parse feedback data")
+		w.RespondWithError(errors.NewBadRequestError("Invalid feedback data format", err))
+		return
+	}
+
+	// Validate the request
+	if feedbackReq.FeedbackText == "" {
+		log.Error(nil, "Missing required field: feedbackText")
+		w.RespondWithError(errors.NewBadRequestError("Missing required field: feedbackText", nil))
+		return
+	}
+
+	err = h.AutogenClient.CreateFeedback(&feedbackReq)
+	if err != nil {
+		log.Error(err, "Failed to create feedback")
+		w.RespondWithError(errors.NewInternalServerError("Failed to create feedback", err))
+		return
+	}
+
+	log.Info("Feedback successfully submitted")
+	RespondWithJSON(w, http.StatusOK, "Feedback submitted successfully")
+}
+
+func (h *FeedbackHandler) HandleListFeedback(w ErrorResponseWriter, r *http.Request) {
+	log := ctrllog.FromContext(r.Context()).WithName("feedback-handler").WithValues("operation", "list-feedback")
+
+	log.Info("Listing feedback")
+
+	userID, err := GetUserID(r)
+	if err != nil {
+		log.Error(err, "Failed to get user ID")
+		w.RespondWithError(errors.NewBadRequestError("Failed to get user ID", err))
+		return
+	}
+
+	feedback, err := h.AutogenClient.ListFeedback(userID)
+	if err != nil {
+		log.Error(err, "Failed to list feedback")
+		w.RespondWithError(errors.NewInternalServerError("Failed to list feedback", err))
+		return
+	}
+
+	log.Info("Feedback listed successfully")
+	RespondWithJSON(w, http.StatusOK, feedback)
+}

go/controller/internal/httpserver/handlers/handlers.go

@@ -19,6 +19,7 @@ type Handlers struct {
 	ToolServers *ToolServersHandler
 	Invoke      *InvokeHandler
 	Memory      *MemoryHandler
+	Feedback    *FeedbackHandler
 }
 
 // Base holds common dependencies for all handlers
@@ -47,5 +48,6 @@ func NewHandlers(kubeClient client.Client, autogenClient *autogen_client.Client,
 		ToolServers: NewToolServersHandler(base),
 		Invoke:      NewInvokeHandler(base),
 		Memory:      NewMemoryHandler(base),
+		Feedback:    NewFeedbackHandler(base),
 	}
 }

go/controller/internal/httpserver/handlers/sessions.go

@@ -200,11 +200,16 @@ func (h *SessionsHandler) HandleListSessionMessages(w ErrorResponseWriter, r *ht
 	configs := []autogen_client.TaskMessageMap{}
 	for _, run := range runs {
 		for _, message := range run.Messages {
-			configs = append(configs, message.Config)
+			item := make(autogen_client.TaskMessageMap)
+			if message.Config != nil {
+				for k, v := range message.Config {
+					item[k] = v
+				}
+			}
+			item["id"] = message.ID
+			configs = append(configs, item)
 		}
 	}
-
-	log.Info("Successfully listed session runs", "count", len(runs))
 	RespondWithJSON(w, http.StatusOK, configs)
 }
 

go/controller/internal/httpserver/server.go

@@ -29,6 +29,7 @@ const (
 	APIPathModels      = "/api/models"
 	APIPathMemories    = "/api/memories"
 	APIPathA2A         = "/api/a2a"
+	APIPathFeedback    = "/api/feedback"
 )
 
 var defaultModelConfig = types.NamespacedName{
@@ -165,6 +166,10 @@ func (s *HTTPServer) setupRoutes() {
 	s.router.HandleFunc(APIPathMemories+"/{memoryName}", adaptHandler(s.handlers.Memory.HandleGetMemory)).Methods(http.MethodGet)
 	s.router.HandleFunc(APIPathMemories+"/{memoryName}", adaptHandler(s.handlers.Memory.HandleUpdateMemory)).Methods(http.MethodPut)
 
+	// Feedback
+	s.router.HandleFunc(APIPathFeedback, adaptHandler(s.handlers.Feedback.HandleCreateFeedback)).Methods(http.MethodPost)
+	s.router.HandleFunc(APIPathFeedback, adaptHandler(s.handlers.Feedback.HandleListFeedback)).Methods(http.MethodGet)
+
 	// A2A
 	s.router.PathPrefix(APIPathA2A).Handler(s.config.A2AHandler)
 

python/src/autogenstudio/datamodel/__init__.py

@@ -1,4 +1,4 @@
-from .db import BaseDBModel, Gallery, Message, Run, RunStatus, Session, Settings, Team, Tool, ToolServer
+from .db import BaseDBModel, Feedback, Gallery, Message, Run, RunStatus, Session, Settings, Team, Tool, ToolServer
 from .types import (
     EnvironmentVariable,
     GalleryComponents,
@@ -35,4 +35,5 @@
     "EnvironmentVariable",
     "Gallery",
     "ToolServer",
+    "Feedback",
 ]

python/src/autogenstudio/datamodel/db.py

@@ -7,7 +7,7 @@
 from autogen_core import ComponentModel
 from pydantic import ConfigDict, SecretStr, field_validator
 from sqlalchemy import ForeignKey, Integer
-from sqlmodel import JSON, Column, DateTime, Field, SQLModel, func
+from sqlmodel import JSON, Column, DateTime, Field, Relationship, SQLModel, func
 
 from .eval import EvalJudgeCriteria, EvalRunResult, EvalRunStatus, EvalScore, EvalTask
 from .types import (
@@ -65,6 +65,25 @@ class Message(BaseDBModel, table=True):
     run_id: Optional[int] = Field(default=None, sa_column=Column(Integer, ForeignKey("run.id", ondelete="CASCADE")))
 
     message_meta: Optional[Union[MessageMeta, dict]] = Field(default={}, sa_column=Column(JSON))
+    feedback: List["Feedback"] = Relationship(back_populates="message")
+
+
+class Feedback(BaseDBModel, table=True):
+    """
+    Database model for storing user feedback on agent responses.
+    """
+
+    __table_args__ = {"sqlite_autoincrement": True}
+
+    is_positive: bool = Field(default=False, description="Whether the feedback is positive or negative")
+    feedback_text: str = Field(description="The feedback text provided by the user")
+    issue_type: Optional[str] = Field(default=None, description="Category of issue for negative feedback")
+
+    message_id: Optional[int] = Field(
+        default=None, sa_column=Column(Integer, ForeignKey("message.id", ondelete="CASCADE"))
+    )
+
+    message: Optional["Message"] = Relationship(back_populates="feedback")
 
 
 class Session(BaseDBModel, table=True):

python/src/autogenstudio/sessionmanager/sessionmanager.py

@@ -169,7 +169,7 @@ async def start_stream(
 
                 # The messages[0].content isn't properly being serialized, so it
                 # doesn't even get sent back to the client. (That's why we're seeing undefined in the UI)
-                # I am using the stop_reason to send the error message back and specifically checking 
+                # I am using the stop_reason to send the error message back and specifically checking
                 # for the message type (error).
                 error_result = TeamResult(
                     task_result=TaskResult(

python/src/autogenstudio/web/app.py

@@ -15,7 +15,20 @@
 from .config import settings
 from .deps import cleanup_managers, init_auth_manager, init_managers, register_auth_dependencies
 from .initialization import AppInitializer
-from .routes import gallery, invoke, models, runs, sessions, settingsroute, teams, tool_servers, tools, validation, ws
+from .routes import (
+    feedback,
+    gallery,
+    invoke,
+    models,
+    runs,
+    sessions,
+    settingsroute,
+    teams,
+    tool_servers,
+    tools,
+    validation,
+    ws,
+)
 
 # Initialize application
 app_file_path = os.path.dirname(os.path.abspath(__file__))
@@ -25,7 +38,6 @@
 @asynccontextmanager
 async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
     """
-    Lifecycle manager for the FastAPI application.
     Handles initialization and cleanup of application resources.
     """
 
@@ -105,14 +117,6 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
     responses={404: {"description": "Not found"}},
 )
 
-
-api.include_router(
-    ws.router,
-    prefix="/ws",
-    tags=["websocket"],
-    responses={404: {"description": "Not found"}},
-)
-
 api.include_router(
     validation.router,
     prefix="/validate",
@@ -169,6 +173,13 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
     responses={404: {"description": "Not found"}},
 )
 
+api.include_router(
+    feedback.router,
+    prefix="/feedback",
+    tags=["feedback"],
+    responses={404: {"description": "Not found"}},
+)
+
 # Version endpoint