fix: transcripts fetching

Author: xavidop

pkg/transcript/fetch-all.go

@@ -23,7 +23,7 @@ func FetchAll(agentID, startTime, endTime, tag, rang, outputDirectory string) er
 			return fmt.Errorf("failed to fetch transcript %s: %w", transcriptInformation.ID, err)
 		}
 
-		err = SaveTranscript(transcript, outputDirectory)
+		err = SaveTranscript(transcript, transcriptInformation.ID, outputDirectory)
 		if err != nil {
 			return fmt.Errorf("failed to save transcript %s: %w", transcriptInformation.ID, err)
 		}
@@ -32,15 +32,15 @@ func FetchAll(agentID, startTime, endTime, tag, rang, outputDirectory string) er
 	return nil
 }
 
-func SaveTranscript(transcript [][]string, outputDirectory string) error {
+func SaveTranscript(transcript [][]string, transcriptID, outputDirectory string) error {
 	// Create output directory if it doesn't exist
 	if err := os.MkdirAll(outputDirectory, 0755); err != nil {
 		return fmt.Errorf("failed to create output directory: %w", err)
 	}
 
-	// Generate filename with timestamp
+	// Generate filename with transcript ID and timestamp
 	timestamp := time.Now().Format("20060102-150405")
-	filename := fmt.Sprintf("transcript_%s.csv", timestamp)
+	filename := fmt.Sprintf("transcript_%s_%s.csv", transcriptID, timestamp)
 	fullPath := filepath.Join(outputDirectory, filename)
 	buf := new(bytes.Buffer)
 	wr := csv.NewWriter(buf)

pkg/transcript/fetch.go

@@ -13,7 +13,7 @@ func Fetch(agentID, transcriptID, outputDirectory string) error {
 		return fmt.Errorf("failed to fetch transcript %s: %w", transcriptID, err)
 	}
 
-	err = SaveTranscript(transcript, outputDirectory)
+	err = SaveTranscript(transcript, transcriptID, outputDirectory)
 	if err != nil {
 		return fmt.Errorf("failed to save transcript %s: %w", transcriptID, err)
 	}

pkg/voiceflow/transcript.go

@@ -14,8 +14,9 @@ import (
 )
 
 func FetchTranscriptInformations(agentID, startTime, endTime, tag, rang string) ([]transcript.TranscriptInformation, error) {
-	const pageSize = 100
+	const pageSize = 99 // API max take is exclusive < 100
 	var allTranscripts []transcript.TranscriptInformation
+	seen := make(map[string]bool)
 
 	reqBody := transcript.SearchTranscriptsRequest{}
 	if startTime != "" {
@@ -25,8 +26,11 @@ func FetchTranscriptInformations(agentID, startTime, endTime, tag, rang string)
 		reqBody.EndDate = endTime
 	}
 
-	for skip := 0; ; skip += pageSize {
-		url := fmt.Sprintf("%s/v1/transcript/project/%s?take=%d&skip=%d&order=DESC", global.GetAnalyticsBaseURL(), agentID, pageSize, skip)
+	// Use cursor-based pagination with endDate to avoid skip limits.
+	// Results are ordered DESC (newest first), so we move the endDate
+	// cursor backwards after each full page.
+	for {
+		url := fmt.Sprintf("%s/v1/transcript/project/%s?take=%d&skip=0&order=DESC", global.GetAnalyticsBaseURL(), agentID, pageSize)
 
 		bodyBytes, err := json.Marshal(reqBody)
 		if err != nil {
@@ -63,12 +67,25 @@ func FetchTranscriptInformations(agentID, startTime, endTime, tag, rang string)
 			return nil, fmt.Errorf("error unmarshalling response: %v", err)
 		}
 
-		allTranscripts = append(allTranscripts, searchResponse.Transcripts...)
+		if len(searchResponse.Transcripts) == 0 {
+			break
+		}
+
+		for _, t := range searchResponse.Transcripts {
+			if !seen[t.ID] {
+				seen[t.ID] = true
+				allTranscripts = append(allTranscripts, t)
+			}
+		}
 
 		// If we got fewer results than the page size, we've reached the end
 		if len(searchResponse.Transcripts) < pageSize {
 			break
 		}
+
+		// Use the oldest transcript's createdAt as the new endDate cursor
+		oldest := searchResponse.Transcripts[len(searchResponse.Transcripts)-1]
+		reqBody.EndDate = oldest.CreatedAt
 	}
 
 	return allTranscripts, nil
@@ -87,8 +104,15 @@ func FetchTranscriptCSV(agentID, transcriptID string) ([][]string, error) {
 		message := ""
 		switch turn.Payload.Type {
 		case "text":
-			if tp, err := turn.Payload.GetTextPayload(); err == nil {
-				message = tp.Message
+			if turn.Type == "user-text" {
+				// User text payload is a plain string
+				if s, ok := turn.Payload.Payload.(string); ok {
+					message = s
+				}
+			} else {
+				if tp, err := turn.Payload.GetTextPayload(); err == nil {
+					message = tp.Message
+				}
 			}
 		case "intent":
 			if ip, err := turn.Payload.GetIntentPayload(); err == nil {