refactor(go-api): parse python page_tokens (#5538)

The underlying datastore cursor uses the same proto in python's ndb and
go's datastore libraries, so we can actually just give the Python cursor
straight to the Go implementation since they do identical queries. I
have done a small test to verify that this can work. Even if it's not
100% guaranteed, it's probably still better than nothing.
This is only really needed for the Python to Go cutover, to avoid
throwing a bunch of errors to users when we deploy. We can delete this
code once the Go API is deployed and stable.

Author: michaelkedar

go/internal/database/datastore/vulnerability_matching.go

@@ -55,11 +55,16 @@ func (s *VulnerabilityStore) MatchCommits(ctx context.Context, commit []byte, cu
 		if cursor != "" {
 			parsedCursor, err := parseMatchCommitsCursor(cursor)
 			if err != nil {
-				// TODO: attempt to recover from Python cursor
-				yield(models.MatchResult{}, fmt.Errorf("%w: %w", models.ErrInvalidCursor, err))
-				return
+				// Attempt to recover from Python cursor
+				pyCursor, _, pyErr := parsePythonCursor(cursor)
+				if pyErr != nil {
+					yield(models.MatchResult{}, fmt.Errorf("%w: %w", models.ErrInvalidCursor, err))
+					return
+				}
+				q = q.Start(pyCursor)
+			} else {
+				q = q.Start(parsedCursor.cursor)
 			}
-			q = q.Start(parsedCursor.cursor)
 		}
 
 		it := s.client.Run(ctx, q)
@@ -189,12 +194,18 @@ func (s *VulnerabilityStore) MatchPackages(ctx context.Context, ecosystem, name,
 		if cursor != "" {
 			parsed, err := parseMatchPackagesCursor(cursor)
 			if err != nil {
-				// TODO: attempt to recover from Python cursor
-				yield(models.MatchResult{}, fmt.Errorf("%w: %w", models.ErrInvalidCursor, err))
-				return
+				// Attempt to recover from Python cursor
+				pyCursor, lastID, pyErr := parsePythonCursor(cursor)
+				if pyErr != nil {
+					yield(models.MatchResult{}, fmt.Errorf("%w: %w", models.ErrInvalidCursor, err))
+					return
+				}
+				lastReturnedID = lastID
+				q = q.Start(pyCursor)
+			} else {
+				lastReturnedID = parsed.lastID
+				q = q.Start(parsed.cursor)
 			}
-			lastReturnedID = parsed.lastID
-			q = q.Start(parsed.cursor)
 		}
 
 		it := s.client.Run(ctx, q)
@@ -573,3 +584,42 @@ func (s *VulnerabilityStore) MatchCommitsBatch(ctx context.Context, queries []mo
 
 	return results, nil
 }
+
+// parsePythonCursor parses a Python NDB-style page token/cursor.
+// Python cursor format: query_number:ndb_cursor_urlsafe:base64(metadata_1):...
+func parsePythonCursor(token string) (datastore.Cursor, string, error) {
+	parts := strings.Split(token, ":")
+	if len(parts) < 2 {
+		return datastore.Cursor{}, "", errors.New("invalid cursor format")
+	}
+
+	cursorPart := strings.TrimRight(parts[1], "=")
+	var cursor datastore.Cursor
+	if cursorPart != "" && cursorPart != "RklSU1RfUEFHRV9UT0tFTg" {
+		var err error
+		cursor, err = datastore.DecodeCursor(cursorPart)
+		if err != nil {
+			return datastore.Cursor{}, "", fmt.Errorf("failed to decode datastore cursor: %w", err)
+		}
+	}
+
+	lastID := ""
+	// Parse metadata fields
+	for _, encMeta := range parts[2:] {
+		// Use standard base64 URL encoding (with or without padding)
+		metaBytes, err := base64.RawURLEncoding.DecodeString(encMeta)
+		if err != nil {
+			// Try with padding if raw decoding fails
+			metaBytes, err = base64.URLEncoding.DecodeString(encMeta)
+			if err != nil {
+				continue
+			}
+		}
+		k, v, found := strings.Cut(string(metaBytes), "=")
+		if found && k == "last_id" {
+			lastID = v
+		}
+	}
+
+	return cursor, lastID, nil
+}

go/internal/database/datastore/vulnerability_matching_test.go

@@ -675,3 +675,79 @@ func TestVulnerabilityStore_MatchPackages_Pagination_DuplicateIDAcrossCursor(t *
 		t.Errorf("Second page mismatch (-want +got):\n%s", diff)
 	}
 }
+
+func TestParsePythonCursor(t *testing.T) {
+	tests := []struct {
+		name       string
+		token      string
+		wantCursor string // unpadded base64 representation of the cursor
+		wantLastID string
+		wantErr    bool
+	}{
+		{
+			name:       "Package query real-world token with last_id metadata",
+			token:      "1:ClcKIAoHdnVsbl9pZBIVGhNHSFNBLTIzaG0tN3c0Ny14dzcyEi9qDm1-b3NzLXZkYi10ZXN0ch0LEhBBZmZlY3RlZFZlcnNpb25zGICAgKyCqLcJDBgAIAA=:bGFzdF9pZD1HSFNBLTIzaG0tN3c0Ny14dzcy",
+			wantCursor: "ClcKIAoHdnVsbl9pZBIVGhNHSFNBLTIzaG0tN3c0Ny14dzcyEi9qDm1-b3NzLXZkYi10ZXN0ch0LEhBBZmZlY3RlZFZlcnNpb25zGICAgKyCqLcJDBgAIAA", // unpadded
+			wantLastID: "GHSA-23hm-7w47-xw72",
+			wantErr:    false,
+		},
+		{
+			name:       "Commit query real-world token without metadata",
+			token:      "1:CkESO2oObX5vc3MtdmRiLXRlc3RyKQsSD0FmZmVjdGVkQ29tbWl0cyIUQ1VSTC1DVkUtMjAxMC0wNzM0LTAMGAAgAA==",
+			wantCursor: "CkESO2oObX5vc3MtdmRiLXRlc3RyKQsSD0FmZmVjdGVkQ29tbWl0cyIUQ1VSTC1DVkUtMjAxMC0wNzM0LTAMGAAgAA", // correct unpadded
+			wantLastID: "",
+			wantErr:    false,
+		},
+		{
+			name:       "First page token special case",
+			token:      "1:RklSU1RfUEFHRV9UT0tFTg==",
+			wantCursor: "",
+			wantLastID: "",
+			wantErr:    false,
+		},
+		{
+			name:       "First page token special case without padding",
+			token:      "1:RklSU1RfUEFHRV9UT0tFTg",
+			wantCursor: "",
+			wantLastID: "",
+			wantErr:    false,
+		},
+		{
+			name:    "Malformed token (insufficient parts)",
+			token:   "1",
+			wantErr: true,
+		},
+		{
+			name:    "Malformed token (invalid base64 cursor)",
+			token:   "1:not-base64-!!!:bGFzdF9pZD1HSFNBLTIzaG0tN3c0Ny14dzcy",
+			wantErr: true,
+		},
+		{
+			name:       "Malformed metadata (ignored, but cursor parsed successfully)",
+			token:      "1:ClcKIAoHdnVsbl9pZBIVGhNHSFNBLTIzaG0tN3c0Ny14dzcyEi9qDm1-b3NzLXZkYi10ZXN0ch0LEhBBZmZlY3RlZFZlcnNpb25zGICAgKyCqLcJDBgAIAA=:not-base64-!!!",
+			wantCursor: "ClcKIAoHdnVsbl9pZBIVGhNHSFNBLTIzaG0tN3c0Ny14dzcyEi9qDm1-b3NzLXZkYi10ZXN0ch0LEhBBZmZlY3RlZFZlcnNpb25zGICAgKyCqLcJDBgAIAA",
+			wantLastID: "",
+			wantErr:    false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotCursor, gotLastID, err := parsePythonCursor(tt.token)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("parsePythonCursor() error = %v, wantErr = %v", err, tt.wantErr)
+			}
+			if tt.wantErr {
+				return
+			}
+
+			if gotLastID != tt.wantLastID {
+				t.Errorf("parsePythonCursor() gotLastID = %q, want %q", gotLastID, tt.wantLastID)
+			}
+
+			if gotCursor.String() != tt.wantCursor {
+				t.Errorf("parsePythonCursor() gotCursor.String() = %q, want %q", gotCursor.String(), tt.wantCursor)
+			}
+		})
+	}
+}