feat: Add cursor pagination to list endpoints; remove offset pagination (#42)

* chore: cursor pagination on feedback_records and webhooks

* chore: refactoring

* chore: removed offset style pagination

* chore: refactoring

* chore: fix webhook list limit

* chore: sorting list

Author: xernobyl

internal/api/handlers/feedback_records_handler.go

@@ -14,6 +14,7 @@ import (
 	"github.com/formbricks/hub/internal/api/validation"
 	"github.com/formbricks/hub/internal/huberrors"
 	"github.com/formbricks/hub/internal/models"
+	"github.com/formbricks/hub/pkg/cursor"
 )
 
 // FeedbackRecordsService defines the interface for feedback records business logic.
@@ -123,6 +124,12 @@ func (h *FeedbackRecordsHandler) List(w http.ResponseWriter, r *http.Request) {
 
 	result, err := h.service.ListFeedbackRecords(r.Context(), filters)
 	if err != nil {
+		if errors.Is(err, cursor.ErrInvalidCursor) {
+			response.RespondBadRequest(w, "Invalid cursor: omit for first page, or use the exact next_cursor value from the previous response")
+
+			return
+		}
+
 		response.RespondInternalServerError(w, "An unexpected error occurred")
 
 		return

internal/api/handlers/search_handler.go

@@ -18,9 +18,9 @@ import (
 
 // SearchService defines the interface for semantic search and similar feedback.
 type SearchService interface {
-	SemanticSearch(ctx context.Context, query, tenantID string, limit, offset int, minScore float64, cursor string) (
+	SemanticSearch(ctx context.Context, query, tenantID string, limit int, minScore float64, cursor string) (
 		service.SearchResult, error)
-	SimilarFeedback(ctx context.Context, feedbackRecordID uuid.UUID, tenantID string, limit, offset int,
+	SimilarFeedback(ctx context.Context, feedbackRecordID uuid.UUID, tenantID string, limit int,
 		minScore float64, cursor string) (service.SearchResult, error)
 }
 
@@ -55,14 +55,7 @@ type SemanticSearchResultItem struct {
 	ValueText        string    `json:"value_text"` // value_text of the feedback record (the text that was embedded)
 }
 
-// maxSearchOffset caps how far paging can go. With OFFSET-based paging the database
-// still computes and discards all rows before the offset, so large offsets (e.g. 5000)
-// make queries slow. Clamping keeps latency predictable and discourages deep paging.
-// To support deeper paging without this limit, switch to cursor-based (keyset) pagination:
-// return a cursor (e.g. last score + last feedback_record_id), and query WHERE (score, id) after cursor
-// instead of OFFSET.
 const (
-	maxSearchOffset    = 1000
 	defaultSearchLimit = 10
 	maxSearchLimit     = 100
 )
@@ -99,17 +92,10 @@ func (h *SearchHandler) SemanticSearch(w http.ResponseWriter, r *http.Request) {
 	}
 
 	limit := parseLimit(r.URL.Query().Get("limit"), defaultSearchLimit, maxSearchLimit)
-
 	cursor := strings.TrimSpace(r.URL.Query().Get("cursor"))
-
-	offset := 0
-	if cursor == "" {
-		offset = min(parseOffset(r.URL.Query().Get("offset")), maxSearchOffset)
-	}
-
 	minScore := parseMinScore(r.URL.Query().Get("min_score"))
 
-	res, err := h.service.SemanticSearch(r.Context(), req.Query, req.TenantID, limit, offset, minScore, cursor)
+	res, err := h.service.SemanticSearch(r.Context(), req.Query, req.TenantID, limit, minScore, cursor)
 	if err != nil {
 		if errors.Is(err, service.ErrMissingTenantID) {
 			response.RespondBadRequest(w, "tenant_id is required")
@@ -177,17 +163,10 @@ func (h *SearchHandler) SimilarFeedback(w http.ResponseWriter, r *http.Request)
 	}
 
 	limit := parseLimit(r.URL.Query().Get("limit"), defaultSearchLimit, maxSearchLimit)
-
 	cursor := strings.TrimSpace(r.URL.Query().Get("cursor"))
-
-	offset := 0
-	if cursor == "" {
-		offset = min(parseOffset(r.URL.Query().Get("offset")), maxSearchOffset)
-	}
-
 	minScore := parseMinScore(r.URL.Query().Get("min_score"))
 
-	res, err := h.service.SimilarFeedback(r.Context(), id, tenantID, limit, offset, minScore, cursor)
+	res, err := h.service.SimilarFeedback(r.Context(), id, tenantID, limit, minScore, cursor)
 	if err != nil {
 		if errors.Is(err, service.ErrEmbeddingNotFound) {
 			response.RespondNotFound(w, "Feedback record has no embedding for the current model")
@@ -233,20 +212,6 @@ func parseLimit(s string, def, upperBound int) int {
 	return min(n, upperBound)
 }
 
-// parseOffset returns the query param "offset" as a non-negative int; default 0.
-func parseOffset(s string) int {
-	if s == "" {
-		return 0
-	}
-
-	n, err := strconv.Atoi(s)
-	if err != nil || n < 0 {
-		return 0
-	}
-
-	return n
-}
-
 // defaultMinScore is the default minimum similarity score when the query param is omitted (reduces noise).
 const defaultMinScore = 0.7
 

internal/api/handlers/search_handler_test.go

@@ -17,27 +17,27 @@ import (
 )
 
 type mockSearchService struct {
-	semanticFunc func(ctx context.Context, query, tenantID string, limit, offset int, minScore float64,
+	semanticFunc func(ctx context.Context, query, tenantID string, limit int, minScore float64,
 		cursor string) (service.SearchResult, error)
-	similarFunc func(ctx context.Context, feedbackRecordID uuid.UUID, tenantID string, limit, offset int,
+	similarFunc func(ctx context.Context, feedbackRecordID uuid.UUID, tenantID string, limit int,
 		minScore float64, cursor string) (service.SearchResult, error)
 }
 
 func (m *mockSearchService) SemanticSearch(
-	ctx context.Context, query, tenantID string, limit, offset int, minScore float64, cursor string,
+	ctx context.Context, query, tenantID string, limit int, minScore float64, cursor string,
 ) (service.SearchResult, error) {
 	if m.semanticFunc != nil {
-		return m.semanticFunc(ctx, query, tenantID, limit, offset, minScore, cursor)
+		return m.semanticFunc(ctx, query, tenantID, limit, minScore, cursor)
 	}
 
 	return service.SearchResult{}, nil
 }
 
 func (m *mockSearchService) SimilarFeedback(
-	ctx context.Context, feedbackRecordID uuid.UUID, tenantID string, limit, offset int, minScore float64, cursor string,
+	ctx context.Context, feedbackRecordID uuid.UUID, tenantID string, limit int, minScore float64, cursor string,
 ) (service.SearchResult, error) {
 	if m.similarFunc != nil {
-		return m.similarFunc(ctx, feedbackRecordID, tenantID, limit, offset, minScore, cursor)
+		return m.similarFunc(ctx, feedbackRecordID, tenantID, limit, minScore, cursor)
 	}
 
 	return service.SearchResult{}, nil
@@ -60,7 +60,7 @@ func TestSearchHandler_SemanticSearch(t *testing.T) {
 	t.Run("empty query returns 400", func(t *testing.T) {
 		called := false
 		mock := &mockSearchService{
-			semanticFunc: func(_ context.Context, _, _ string, _, _ int, _ float64, _ string) (service.SearchResult, error) {
+			semanticFunc: func(_ context.Context, _, _ string, _ int, _ float64, _ string) (service.SearchResult, error) {
 				called = true
 
 				return service.SearchResult{}, service.ErrEmptyQuery
@@ -85,13 +85,12 @@ func TestSearchHandler_SemanticSearch(t *testing.T) {
 		val1 := "Login is very slow."
 		val2 := "Dashboard loads fast."
 		mock := &mockSearchService{
-			semanticFunc: func(_ context.Context, query, tenantID string, limit, offset int, minScore float64,
+			semanticFunc: func(_ context.Context, query, tenantID string, limit int, minScore float64,
 				cursor string,
 			) (service.SearchResult, error) {
 				assert.Equal(t, "login is slow", query)
 				assert.Equal(t, "env-1", tenantID)
 				assert.Equal(t, 10, limit)
-				assert.Equal(t, 0, offset)
 				assert.InDelta(t, 0.7, minScore, 1e-9)
 				assert.Empty(t, cursor)
 
@@ -132,7 +131,7 @@ func TestSearchHandler_SemanticSearch(t *testing.T) {
 
 	t.Run("invalid cursor returns 400", func(t *testing.T) {
 		mock := &mockSearchService{
-			semanticFunc: func(_ context.Context, _, _ string, _, _ int, _ float64, cursor string) (service.SearchResult, error) {
+			semanticFunc: func(_ context.Context, _, _ string, _ int, _ float64, cursor string) (service.SearchResult, error) {
 				if cursor != "" {
 					return service.SearchResult{}, service.ErrInvalidCursor
 				}
@@ -171,7 +170,7 @@ func TestSearchHandler_SimilarFeedback(t *testing.T) {
 
 	t.Run("embedding not found returns 404", func(t *testing.T) {
 		mock := &mockSearchService{
-			similarFunc: func(_ context.Context, _ uuid.UUID, _ string, _, _ int, _ float64, _ string) (service.SearchResult, error) {
+			similarFunc: func(_ context.Context, _ uuid.UUID, _ string, _ int, _ float64, _ string) (service.SearchResult, error) {
 				return service.SearchResult{}, service.ErrEmbeddingNotFound
 			},
 		}
@@ -191,13 +190,12 @@ func TestSearchHandler_SimilarFeedback(t *testing.T) {
 		similarID := uuid.MustParse("018e1234-5678-9abc-def0-aaaaaaaaaaaa")
 		similarVal := "Similar feedback text."
 		mock := &mockSearchService{
-			similarFunc: func(_ context.Context, fid uuid.UUID, tenantID string, limit, offset int, minScore float64,
+			similarFunc: func(_ context.Context, fid uuid.UUID, tenantID string, limit int, minScore float64,
 				cursor string,
 			) (service.SearchResult, error) {
 				assert.Equal(t, id, fid)
 				assert.Equal(t, "env-1", tenantID)
 				assert.Equal(t, 10, limit)
-				assert.Equal(t, 0, offset)
 				assert.InDelta(t, 0.7, minScore, 1e-9)
 				assert.Empty(t, cursor)
 

internal/api/handlers/webhooks_handler.go

@@ -13,6 +13,7 @@ import (
 	"github.com/formbricks/hub/internal/api/validation"
 	"github.com/formbricks/hub/internal/huberrors"
 	"github.com/formbricks/hub/internal/models"
+	"github.com/formbricks/hub/pkg/cursor"
 )
 
 // WebhooksService defines the interface for webhooks business logic.
@@ -122,6 +123,12 @@ func (h *WebhooksHandler) List(w http.ResponseWriter, r *http.Request) {
 
 	result, err := h.service.ListWebhooks(r.Context(), filters)
 	if err != nil {
+		if errors.Is(err, cursor.ErrInvalidCursor) {
+			response.RespondBadRequest(w, "Invalid cursor: omit for first page, or use the exact next_cursor value from the previous response")
+
+			return
+		}
+
 		slog.Error("Failed to list webhooks", "method", r.Method, "path", r.URL.Path, "error", err)
 		response.RespondInternalServerError(w, "An unexpected error occurred")
 

internal/models/feedback_records.go

@@ -186,15 +186,14 @@ type ListFeedbackRecordsFilters struct {
 	Since          *time.Time `form:"since"           validate:"omitempty"`
 	Until          *time.Time `form:"until"           validate:"omitempty"`
 	Limit          int        `form:"limit"           validate:"omitempty,min=1,max=1000"`
-	Offset         int        `form:"offset"          validate:"omitempty,min=0"`
+	Cursor         string     `form:"cursor"          validate:"omitempty"` // keyset; omit for first page, use next_cursor for next
 }
 
 // ListFeedbackRecordsResponse represents the response for listing feedback records.
 type ListFeedbackRecordsResponse struct {
-	Data   []FeedbackRecord `json:"data"`
-	Total  int64            `json:"total"`
-	Limit  int              `json:"limit"`
-	Offset int              `json:"offset"`
+	Data       []FeedbackRecord `json:"data"`
+	Limit      int              `json:"limit"`
+	NextCursor string           `json:"next_cursor,omitempty"` // present when there may be more results
 }
 
 // BulkDeleteFilters represents query parameters for bulk delete operation.

internal/models/webhooks.go

@@ -198,13 +198,12 @@ type ListWebhooksFilters struct {
 	Enabled  *bool   `form:"enabled"`
 	TenantID *string `form:"tenant_id" validate:"omitempty,no_null_bytes"`
 	Limit    int     `form:"limit"     validate:"omitempty,min=1,max=1000"`
-	Offset   int     `form:"offset"    validate:"omitempty,min=0"`
+	Cursor   string  `form:"cursor"    validate:"omitempty"` // keyset cursor; omit for first page, use next_cursor for subsequent pages
 }
 
 // ListWebhooksResponse represents the response for listing webhooks.
 type ListWebhooksResponse struct {
-	Data   []Webhook `json:"data"`
-	Total  int64     `json:"total"`
-	Limit  int       `json:"limit"`
-	Offset int       `json:"offset"`
+	Data       []Webhook `json:"data"`
+	Limit      int       `json:"limit"`
+	NextCursor string    `json:"next_cursor,omitempty"` // present when there may be more results
 }

internal/repository/embeddings_repository.go

@@ -169,22 +169,18 @@ func (r *EmbeddingsRepository) GetEmbeddingByFeedbackRecordAndModelAndTenant(
 // filtered in application code (not in WHERE) so pgvector's iterative index scan can run. Uses
 // full-precision query vector (no quantization); sets hnsw.ef_search for better recall. Over-fetches
 // then trims to limit to account for tenant/minScore filtering. excludeID optionally excludes one
-// feedback record (e.g. for "similar" endpoint). offset is the number of rows to skip (for paging).
+// feedback record (e.g. for "similar" endpoint). First page only; use NearestFeedbackRecordsByEmbeddingAfterCursor for next pages.
 func (r *EmbeddingsRepository) NearestFeedbackRecordsByEmbedding(
-	ctx context.Context, model string, queryEmbedding []float32, tenantID string, limit, offset int, excludeID *uuid.UUID, minScore float64,
+	ctx context.Context, model string, queryEmbedding []float32, tenantID string, limit int, excludeID *uuid.UUID, minScore float64,
 ) ([]models.FeedbackRecordWithScore, bool, error) {
 	if len(queryEmbedding) != models.EmbeddingVectorDimensions {
 		return nil, false, fmt.Errorf("%w: got %d, want %d", ErrEmbeddingDimensionMismatch, len(queryEmbedding), models.EmbeddingVectorDimensions)
 	}
 
-	if offset < 0 {
-		offset = 0
-	}
-
 	// Full-precision query vector (ephemeral); pgvector compares vector vs halfvec natively.
 	queryVec := pgvector.NewVector(queryEmbedding)
 
-	fetchLimit := min(limit*nearestOverFetchFactor+offset, maxNearestFetchLimit)
+	fetchLimit := min(limit*nearestOverFetchFactor, maxNearestFetchLimit)
 
 	dbTx, err := r.db.BeginTx(ctx, pgx.TxOptions{})
 	if err != nil {
@@ -210,15 +206,15 @@ func (r *EmbeddingsRepository) NearestFeedbackRecordsByEmbedding(
 			INNER JOIN feedback_records fr ON fr.id = e.feedback_record_id
 			WHERE e.model = $2 AND fr.tenant_id = $3
 			ORDER BY (e.embedding <=> $1), e.feedback_record_id
-			LIMIT $4 OFFSET $5`, queryVec, model, tenantID, fetchLimit, offset)
+			LIMIT $4`, queryVec, model, tenantID, fetchLimit)
 	} else {
 		rows, err = dbTx.Query(ctx, `
 			SELECT e.feedback_record_id, (1 - (e.embedding <=> $1)) AS score, COALESCE(fr.field_label, ''), fr.value_text
 			FROM embeddings e
 			INNER JOIN feedback_records fr ON fr.id = e.feedback_record_id
 			WHERE e.model = $2 AND fr.tenant_id = $3 AND e.feedback_record_id != $4
 			ORDER BY (e.embedding <=> $1), e.feedback_record_id
-			LIMIT $5 OFFSET $6`, queryVec, model, tenantID, *excludeID, fetchLimit, offset)
+			LIMIT $5`, queryVec, model, tenantID, *excludeID, fetchLimit)
 	}
 
 	if err != nil {